Compare commits

...

74 Commits

Author SHA1 Message Date
Dan Finneran
296c9c1580 Unintentionally created duplicate functions that can't be used
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-08-12 12:17:20 +00:00
Daniel Finneran
d8877072d4 Merge pull request #1219 from kube-vip/v1-release
Version ONE DOT ZERO
2025-07-31 16:42:23 +01:00
Dan Finneran
89a8dc7de1 default setting to 0
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Dan Finneran
704c346f5e naming convention lint fix
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Dan Finneran
65061c5cd9 Adds an annotation to set the return path filter for a macvlan nic
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Daniel Finneran
32233918b4 Version ONE DOT ZERO 2025-07-31 12:34:37 +01:00
Daniel Finneran
76169da60f Merge pull request #1216 from kube-vip/dependabot/go_modules/github.com/docker/docker-28.3.3incompatible
Bump github.com/docker/docker from 28.3.0+incompatible to 28.3.3+incompatible
2025-07-31 12:33:44 +01:00
Daniel Finneran
9bcf1413f0 Merge pull request #1211 from kube-vip/dependabot/github_actions/anchore/sbom-action-0.20.4
Bump anchore/sbom-action from 0.20.1 to 0.20.4
2025-07-31 12:33:28 +01:00
Daniel Finneran
8e428e875f Merge pull request #1203 from kube-vip/dependabot/docker/alpine-3.22.1
Bump alpine from 3.22.0 to 3.22.1
2025-07-31 12:33:14 +01:00
dependabot[bot]
2fbecc25e5 Bump alpine from 3.22.0 to 3.22.1
Bumps alpine from 3.22.0 to 3.22.1.

---
updated-dependencies:
- dependency-name: alpine
  dependency-version: 3.22.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 15:17:21 +00:00
Daniel Finneran
02e77271d0 Merge pull request #1202 from kube-vip/dependabot/docker/golang-1.24.5-alpine3.22 2025-07-30 16:16:18 +01:00
Daniel Finneran
3d61888e58 Merge pull request #1192 from kube-vip/dependabot/go_modules/sigs.k8s.io/kind-0.29.0 2025-07-30 16:15:52 +01:00
dependabot[bot]
efe75f491b Bump sigs.k8s.io/kind from 0.27.0 to 0.29.0
Bumps [sigs.k8s.io/kind](https://github.com/kubernetes-sigs/kind) from 0.27.0 to 0.29.0.
- [Release notes](https://github.com/kubernetes-sigs/kind/releases)
- [Commits](https://github.com/kubernetes-sigs/kind/compare/v0.27.0...v0.29.0)

---
updated-dependencies:
- dependency-name: sigs.k8s.io/kind
  dependency-version: 0.29.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 13:55:46 +00:00
Daniel Finneran
000c139004 Merge pull request #1194 from kube-vip/dependabot/go_modules/github.com/vishvananda/netlink-1.3.1 2025-07-30 14:50:38 +01:00
Daniel Finneran
c39b84f0a9 Merge pull request #1193 from kube-vip/dependabot/go_modules/github.com/osrg/gobgp/v3-3.37.0 2025-07-30 14:50:25 +01:00
dependabot[bot]
ee958addaa Bump github.com/docker/docker
Bumps [github.com/docker/docker](https://github.com/docker/docker) from 28.3.0+incompatible to 28.3.3+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v28.3.0...v28.3.3)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-version: 28.3.3+incompatible
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 13:48:44 +00:00
Daniel Finneran
8fe53351f8 Merge pull request #1215 from kube-vip/endpointslice 2025-07-30 14:47:26 +01:00
Daniel Finneran
bc9d860d83 Merge pull request #1213 from kube-vip/egress_internal 2025-07-30 14:47:07 +01:00
Dan Finneran
332a23e543 Bump k8s api and endpointslices as default
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-29 13:36:32 +00:00
Dan Finneran
b20713b50f linting issue with port out of bounds.
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:18:47 +00:00
Dan Finneran
94e96581ef typo and add SCTP
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:13:20 +00:00
Dan Finneran
61be6d0b6a fighting with merge conflicts
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:08:53 +00:00
Dan Finneran
be22805a7d sigh
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:08:22 +00:00
Dan Finneran
25f6253286 Adds the internal egress functionality
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:01:00 +00:00
dependabot[bot]
f3e9fb6ea9 Bump github.com/osrg/gobgp/v3 from 3.36.0 to 3.37.0
Bumps [github.com/osrg/gobgp/v3](https://github.com/osrg/gobgp) from 3.36.0 to 3.37.0.
- [Release notes](https://github.com/osrg/gobgp/releases)
- [Changelog](https://github.com/osrg/gobgp/blob/master/.goreleaser.yml)
- [Commits](https://github.com/osrg/gobgp/compare/v3.36.0...v3.37.0)

---
updated-dependencies:
- dependency-name: github.com/osrg/gobgp/v3
  dependency-version: 3.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-28 15:17:29 +00:00
Patryk Strusiewicz-Surmacki
0f3dda02c4 Refactoring services-watcher code
Co-authored-by: Cellebyte <marcel.fest@live.de>
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-28 17:14:09 +02:00
Dan Finneran
a2873b5465 Adds the internal egress functionality
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 13:57:50 +00:00
dependabot[bot]
202d45e5ab Bump anchore/sbom-action from 0.20.1 to 0.20.4
Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.20.1 to 0.20.4.
- [Release notes](https://github.com/anchore/sbom-action/releases)
- [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md)
- [Commits](https://github.com/anchore/sbom-action/compare/v0.20.1...v0.20.4)

---
updated-dependencies:
- dependency-name: anchore/sbom-action
  dependency-version: 0.20.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-28 03:11:16 +00:00
Daniel Finneran
f5e4612c03 Merge pull request #1210 from kube-vip/fix/ci 2025-07-27 22:14:39 +01:00
Cellebyte
de888c501c split ci into two files
Signed-off-by: Cellebyte <marcel.fest@live.de>
2025-07-27 23:02:09 +02:00
Cellebyte
47bc83c248 Don't execute the e2e-tests in parallel
Signed-off-by: Cellebyte <marcel.fest@live.de>
2025-07-27 23:02:09 +02:00
Patryk Strusiewicz-Surmacki
1cf637c569 Moved endpoint-related code from pkg/manager to pkg/endpoints
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:23 +02:00
Patryk Strusiewicz-Surmacki
ae2571e241 Refactoring endpoint-watcher code
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:23 +02:00
Patryk Strusiewicz-Surmacki
86f5e9b8b2 Removed insecure client use in E2E tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:00 +02:00
Patryk Strusiewicz-Surmacki
ce61ff085a Use insecure client only in E2E tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-21 17:54:12 +02:00
dependabot[bot]
b816e154cf Bump golang from 1.24.4-alpine3.22 to 1.24.5-alpine3.22
Bumps golang from 1.24.4-alpine3.22 to 1.24.5-alpine3.22.

---
updated-dependencies:
- dependency-name: golang
  dependency-version: 1.24.5-alpine3.22
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-14 03:15:14 +00:00
Patryk Strusiewicz-Surmacki
ac1238c337 Rebase
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-10 17:50:15 +02:00
Patryk Strusiewicz-Surmacki
10dbf2c0ef Refactoring endpoint-watcher code
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-10 17:50:15 +02:00
Jan-Philipp Litza
cf68f8639c Add VIPs as deprecated
This causes IPv6 source address selection according to RFC 3484 to
ignore VIPs when choosing which address the host should use for egress
communication.

Signed-off-by: Jan-Philipp Litza <janphilipp@litza.de>
2025-07-09 10:54:28 +02:00
Daniel Finneran
b114c11b0f Merge pull request #1191 from kube-vip/thebsdbox-patch-1
Update README.md
2025-07-03 09:59:49 +01:00
Patryk Strusiewicz-Surmacki
42b7a8152b Basic e2e tests for BGP mode
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-01 12:36:33 +02:00
mushrushu
f7821c7fb3 fixed vars formatting & log statement
Signed-off-by: mushrushu <mushrushu@outlook.com>
2025-07-01 12:34:43 +02:00
mushrushu
42478905d0 added zebra integration and its configuration
Signed-off-by: mushrushu <mushrushu@outlook.com>
2025-07-01 12:34:43 +02:00
dependabot[bot]
8d55bd3b63 Bump github.com/vishvananda/netlink from 1.3.0 to 1.3.1
Bumps [github.com/vishvananda/netlink](https://github.com/vishvananda/netlink) from 1.3.0 to 1.3.1.
- [Release notes](https://github.com/vishvananda/netlink/releases)
- [Commits](https://github.com/vishvananda/netlink/compare/v1.3.0...v1.3.1)

---
updated-dependencies:
- dependency-name: github.com/vishvananda/netlink
  dependency-version: 1.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-30 03:20:14 +00:00
Daniel Finneran
16247fc3a3 Update README.md
Adds health of project badges! (looking healthy) 😄
2025-06-27 14:20:14 +01:00
Marcel Fest
b56b80cd30 Bump version to v0.9.2
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-24 19:02:31 +02:00
dependabot[bot]
ba25e0e583 Bump github.com/prometheus/client_golang from 1.21.0 to 1.22.0
Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.21.0 to 1.22.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.21.0...v1.22.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-version: 1.22.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-24 18:59:16 +02:00
dependabot[bot]
8f1fe355fc Bump github.com/onsi/gomega from 1.36.2 to 1.37.0
Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.36.2 to 1.37.0.
- [Release notes](https://github.com/onsi/gomega/releases)
- [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md)
- [Commits](https://github.com/onsi/gomega/compare/v1.36.2...v1.37.0)

---
updated-dependencies:
- dependency-name: github.com/onsi/gomega
  dependency-version: 1.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-24 16:29:41 +02:00
dependabot[bot]
649d9bf0ef Bump golang.org/x/sys from 0.32.0 to 0.33.0
Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.32.0 to 0.33.0.
- [Commits](https://github.com/golang/sys/compare/v0.32.0...v0.33.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sys
  dependency-version: 0.33.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-23 09:20:35 +02:00
dependabot[bot]
a5108a69aa Bump anchore/sbom-action from 0.19.0 to 0.20.1
Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.19.0 to 0.20.1.
- [Release notes](https://github.com/anchore/sbom-action/releases)
- [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md)
- [Commits](https://github.com/anchore/sbom-action/compare/v0.19.0...v0.20.1)

---
updated-dependencies:
- dependency-name: anchore/sbom-action
  dependency-version: 0.20.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-20 17:53:00 +02:00
dependabot[bot]
c74a496299 Bump go.etcd.io/etcd/client/v3 from 3.5.18 to 3.6.1
Bumps [go.etcd.io/etcd/client/v3](https://github.com/etcd-io/etcd) from 3.5.18 to 3.6.1.
- [Release notes](https://github.com/etcd-io/etcd/releases)
- [Commits](https://github.com/etcd-io/etcd/compare/v3.5.18...v3.6.1)

---
updated-dependencies:
- dependency-name: go.etcd.io/etcd/client/v3
  dependency-version: 3.6.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-17 12:06:43 +02:00
dependabot[bot]
b74c274466 Bump alpine from 3.21.3 to 3.22.0
Bumps alpine from 3.21.3 to 3.22.0.

---
updated-dependencies:
- dependency-name: alpine
  dependency-version: 3.22.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-17 11:57:04 +02:00
Dan Finneran
56b3867e57 typo fix 2025-06-17 11:56:43 +02:00
Dan Finneran
66d237bfbc adds kernel module check for nftables 2025-06-17 11:56:43 +02:00
Patryk Strusiewicz-Surmacki
68071b214e Fixed BGP clear deadlock
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-06-17 11:55:49 +02:00
Patryk Strusiewicz-Surmacki
a82ca5576b Updated go version to 1.24.4
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-06-17 09:53:57 +02:00
Patryk Strusiewicz-Surmacki
81dd386b4e Fixed service-tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
Co-authored-by: Marcel Fest <marcel@fest4.eu>
2025-06-10 22:26:02 +02:00
Patryk Strusiewicz-Surmacki
b61a74396d Moved configuredRoutes map to serviceContext struct
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-06-10 22:26:02 +02:00
Daniel Finneran
105fbc522a Merge pull request #1163 from p-strusiewiczsurmacki-mobica/rt-tests
Added ARP/NDP services tests and Routing Table services tests
2025-06-10 11:34:15 -07:00
Dan Finneran
2b52c39242 Small change to reduce pointless errors 2025-06-10 13:31:06 +02:00
Patryk Strusiewicz-Surmacki
25d39bca09 Added TODO for insecure discovery client
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-06-06 16:18:38 +02:00
Marcel Fest
644226321e fix typo
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
3928dda541 move lock for cleanRoutes feature at start of kube-vip
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Patryk Strusiewicz-Surmacki
d9a7f413a6 Moved code to use sync.Maps
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Patryk Strusiewicz-Surmacki
eb300bb634 Fixed deadlock
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
c30fd9e7be fix tests + refactor func
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
2fc969b848 Move the check into the function
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
47884088ec remove AI code
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
70e1212396 remove redundant condition
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
d947c2abcc refactor syncServices to be lockable at all
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
e2efb64aea Use the mutext in callbacks as well
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Marcel Fest
01279d45e3 Added function to create ip array from statu LB ingress field
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2025-06-05 17:26:55 +02:00
Patryk Strusiewicz-Surmacki
f2a7cad218 Added ARP/NDP services tests and Routing Table services tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-05-23 11:59:14 +02:00
Nathan Przybyszewski
98163341d3 Replace invalid verb "endpoints" with "update"
Signed-off-by: Nathan Przybyszewski <172687074+nprzy@users.noreply.github.com>
2025-05-01 10:28:43 +02:00
72 changed files with 5642 additions and 2371 deletions

View File

@@ -26,6 +26,6 @@ jobs:
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.19.0
uses: anchore/sbom-action@v0.20.4
with:
format: cyclonedx-json

51
.github/workflows/ci-pull-request.yaml vendored Normal file
View File

@@ -0,0 +1,51 @@
name: For each PR
on:
pull_request:
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
strategy:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests

View File

@@ -1,4 +1,4 @@
name: For each commit and PR
name: For each commit
on:
push:
pull_request:
@@ -43,40 +43,6 @@ jobs:
go-version-file: go.mod
- name: Run tests
run: make integration-tests
e2e-tests:
runs-on: ubuntu-latest
name: E2E ARP tests
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
# - name: Run Control plane tests
# run: make e2e-tests
- name: Run Control plane tests v1.29.0 onwards
run: make e2e-tests129
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.2-alpine3.20 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.2-alpine3.20 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.21.3
FROM alpine:3.22.1
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v0.9.1
VERSION := v1.0.0
BUILD := `git rev-parse HEAD`
@@ -133,10 +133,18 @@ integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests:
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests129:
V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
service-tests:
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
@@ -163,3 +171,9 @@ kind-quick:
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
mkdir -p bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin

View File

@@ -4,7 +4,7 @@ High Availability and Load-Balancing
![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
## Overview
Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services

View File

@@ -93,6 +93,11 @@ func init() {
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
@@ -123,7 +128,7 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")

88
go.mod
View File

@@ -1,92 +1,107 @@
module github.com/kube-vip/kube-vip
go 1.24.2
go 1.24.4
require (
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.3.3+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/golang/protobuf v1.5.4
github.com/google/go-cmp v0.7.0
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.5.8
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.36.3
github.com/osrg/gobgp/v3 v3.36.0
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.21.0
github.com/prometheus/client_golang v1.22.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.0
go.etcd.io/etcd/api/v3 v3.5.21
go.etcd.io/etcd/client/pkg/v3 v3.5.21
go.etcd.io/etcd/client/v3 v3.5.18
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.13.0
golang.org/x/sys v0.32.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.32.3
k8s.io/apimachinery v0.32.3
k8s.io/client-go v0.32.3
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.27.0
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k-sone/critbitgo v1.4.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.2 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -111,26 +126,32 @@ require (
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/oauth2 v0.25.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.31.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
google.golang.org/grpc v1.69.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
@@ -139,5 +160,6 @@ require (
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
)

207
go.sum
View File

@@ -1,17 +1,29 @@
al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -23,8 +35,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
@@ -35,6 +55,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -44,8 +66,9 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
@@ -76,15 +99,12 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 h1:+9C/TgFfcCmZBV7Fjb3kQCGlkpFrhtvFDgbdQHB9RaA=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962/go.mod h1:H3K1Iu/utuCfa10JO+GsmKUYSWi7ug57Rk6GaDRHaaQ=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
@@ -95,6 +115,8 @@ github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
@@ -127,8 +149,9 @@ github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrb
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -162,8 +185,8 @@ github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuri
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
@@ -175,29 +198,42 @@ github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE9
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU=
github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/osrg/gobgp/v3 v3.36.0 h1:6KbNDyvSbN2GAIiVMykAgLUsvcSldNPiCCP5KzV0VP4=
github.com/osrg/gobgp/v3 v3.36.0/go.mod h1:ldZ/eydK80FuAmTGYBV23tWhTRJtIk2tw5NSYhPeqVk=
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -205,8 +241,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
@@ -215,13 +251,14 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
@@ -237,6 +274,10 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@@ -249,9 +290,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -265,22 +305,32 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/etcd/api/v3 v3.5.21 h1:A6O2/JDb3tvHhiIz3xf9nJ7REHvtEFJJ3veW3FbCnS8=
go.etcd.io/etcd/api/v3 v3.5.21/go.mod h1:c3aH5wcvXv/9dqIw2Y810LDXJfhSYdHQ0vxmP3CCHVY=
go.etcd.io/etcd/client/pkg/v3 v3.5.21 h1:lPBu71Y7osQmzlflM9OfeIV2JlmpBjqBNlLtcoBqUTc=
go.etcd.io/etcd/client/pkg/v3 v3.5.21/go.mod h1:BgqT/IXPjK9NkeSDjbzwsHySX3yIle2+ndz28nVsjUs=
go.etcd.io/etcd/client/v3 v3.5.18 h1:nvvYmNHGumkDjZhTHgVU36A9pykGa2K4lAJ0yY7hcXA=
go.etcd.io/etcd/client/v3 v3.5.18/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E=
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -292,16 +342,16 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -322,20 +372,22 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -362,16 +414,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -380,8 +432,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -390,12 +442,12 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA=
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -416,16 +468,16 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU=
k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
@@ -446,9 +498,12 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.27.0 h1:PQ3f0iAWNIj66LYkZ1ivhEg/+Zb6UPMbO+qVei/INZA=
sigs.k8s.io/kind v0.27.0/go.mod h1:RZVFmy6qcwlSWwp6xeIUv7kXCPF3i8MXsEXxW/J+gJY=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 h1:nbCitCK2hfnhyiKo6uf2HxUPTCodY6Qaf85SbDIaMBk=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -40,7 +40,6 @@ func (e *Entry) Check() bool {
switch {
case utils.FileExists(adminConfigPath):
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
return false

View File

@@ -8,7 +8,6 @@ import (
"strings"
//nolint
"github.com/golang/protobuf/ptypes/any"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/osrg/gobgp/v3/pkg/server"
@@ -142,7 +141,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
Safi: api.Family_SAFI_UNICAST,
},
Nlri: nlri,
Pattrs: []*any.Any{originAttr, nhAttr},
Pattrs: []*anypb.Any{originAttr, nhAttr},
}
} else {
//nolint
@@ -160,13 +159,13 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
Family: v6Family,
NextHops: []string{"::"}, // gobgp will fill this
Nlris: []*any.Any{nlri},
Nlris: []*anypb.Any{nlri},
})
path = &api.Path{
Family: v6Family,
Nlri: nlri,
Pattrs: []*any.Any{originAttr, mpAttr},
Pattrs: []*anypb.Any{originAttr, mpAttr},
}
}
return

View File

@@ -9,10 +9,11 @@ import (
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
func NewBGPServer(c *Config) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -28,13 +29,25 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
b = &Server{
s: gobgp.NewBgpServer(),
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
}
return
}
// Start starts the BGP server
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: c.AS,
RouterId: c.RouterID,
Asn: b.c.AS,
RouterId: b.c.RouterID,
ListenPort: -1,
},
}); err != nil {
@@ -52,12 +65,23 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
return
}
for _, p := range c.Peers {
for _, p := range b.c.Peers {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
}); err != nil {
log.Error(err.Error())
return
}
}
return
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
@@ -121,10 +122,24 @@ type Config struct {
KeepaliveInterval uint64
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -55,7 +55,8 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, intfMgr)
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}

View File

@@ -151,10 +151,13 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
}
run := &runConfig{
@@ -281,7 +284,7 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
})
}
func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
@@ -289,7 +292,7 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) e
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},

View File

@@ -84,7 +84,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
go func() {
err = sm.NodeWatcher(lb, c.Port)
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
if err != nil {
log.Error("Error watching node labels", "err", err)
}
@@ -257,11 +257,11 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server, name string, serviceInstances *[]*Instance) {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
ctxArp, cancelArp := context.WithCancel(context.Background())
ctxArp, cancelArp := context.WithCancel(ctx)
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
@@ -318,7 +318,7 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
for i := range cluster.Network {
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(serviceInstances, r) < 1 {
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
@@ -379,20 +379,3 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.Remove(arpInstance)
}
func CountRouteReferences(serviceInstances *[]*Instance, route *netlink.Route) int {
cnt := 0
for _, instance := range *serviceInstances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

99
pkg/egress/egress.go Normal file
View File

@@ -0,0 +1,99 @@
package egress
import (
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
IPv6 := false
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" {
return nftables.DeleteSNAT(IPv6, serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

165
pkg/endpoints/endpoints.go Normal file
View File

@@ -0,0 +1,165 @@
package endpoints
import (
"context"
"fmt"
"net"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
)
type Processor struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
// There are local endpoints available on the node
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
// There are no local endpoints
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = endpoints[0]
return
}
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == *lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
activeEndpointAnnotation := kubevip.ActiveEndpoint
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}

View File

@@ -0,0 +1,125 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type BGP struct {
generic
bgpServer *bgp.Server
}
func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
return &BGP{
generic: generic,
bgpServer: bgpServer,
}
}
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
}
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
}
}
}
}

View File

@@ -0,0 +1,118 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableRoutingTable {
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
}
return &generic
}
type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
} else {
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
}
return endpoints, nil
}
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -0,0 +1,178 @@
package endpoints
import (
"context"
"errors"
"fmt"
"net"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
}
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
}
}
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
return rt.getAllEndpoints(service, id)
}
func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -0,0 +1,137 @@
package providers
import (
"context"
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
log "log/slog"
)
type Endpoints struct {
label string
//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
endpoints *v1.Endpoints
}
func NewEndpoints() Provider {
return &Endpoints{
label: "endpoints",
}
}
func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
}
ep.endpoints = eps
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -1,4 +1,4 @@
package manager
package providers
import (
"context"
@@ -13,17 +13,24 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type endpointslicesProvider struct {
type Endpointslices struct {
label string
endpoints *discoveryv1.EndpointSlice
}
func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
}
}
func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
@@ -31,9 +38,9 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
@@ -43,7 +50,7 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
return rw, nil
}
func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
cancel()
@@ -53,7 +60,7 @@ func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel co
return nil
}
func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
@@ -61,7 +68,7 @@ func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
return result, nil
}
func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
@@ -91,11 +98,11 @@ func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config
return localEndpoints, nil
}
func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -105,10 +112,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -123,10 +130,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
return nil
}
func (ep *endpointslicesProvider) getLabel() string {
func (ep *Endpointslices) GetLabel() string {
return ep.label
}
func (ep *endpointslicesProvider) getProtocol() string {
func (ep *Endpointslices) GetProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -0,0 +1,22 @@
package providers
import (
"context"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
watchtools "k8s.io/client-go/tools/watch"
)
type Provider interface {
CreateRetryWatcher(context.Context, *kubernetes.Clientset,
*v1.Service) (*watchtools.RetryWatcher, error)
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
GetProtocol() string
}

View File

@@ -1,4 +1,4 @@
package cluster
package instance
import (
"fmt"
@@ -6,36 +6,27 @@ import (
"strconv"
"strings"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/vip"
)
const (
// Hardware address of the host that has the VIP
HWAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
)
// Instance defines an instance of everything needed to manage vips
type Instance struct {
// Virtual IP / Load Balancer configuration
VIPConfigs []*kubevip.Config
// cluster instances
Clusters []*Cluster
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCP bool
@@ -68,7 +59,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[ServiceInterface] // If the service has a specific interface defined, then use it
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
@@ -157,8 +148,6 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
//log.Info("new instance", "svc", *svc, "interface", svcInterface)
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
@@ -196,9 +185,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[HWAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[RequestedIP]
instance.DHCPHostname = svc.Annotations[LoadbalancerHostname]
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
}
configPorts := make([]kubevip.Port, 0)
@@ -241,7 +230,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
for _, vipConfig := range instance.VIPConfigs {
c, err := InitCluster(vipConfig, false, intfMgr, arpMgr)
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
@@ -253,7 +242,6 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -371,6 +359,28 @@ func (i *Instance) startDHCP() error {
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
@@ -397,12 +407,32 @@ func (i *Instance) startDHCP() error {
return nil
}
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
// If the service has no status, return empty
lbStatusAddresses := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
// TODO: Handle hostname if needed
}
return lbStatusAddresses
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[LoadbalancerIPAnnotation]; annotationAvailable {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
for _, ip := range ips {
@@ -439,3 +469,14 @@ func FetchServiceAddresses(s *v1.Service) []string {
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
return nil
}

View File

@@ -0,0 +1,49 @@
package kubevip
const (
// Hardware address of the host that has the VIP
HwAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
// The host that has the VIP
VipHost = "kube-vip.io/vipHost"
// Enable Egress on a service
Egress = "kube-vip.io/egress"
// Enable internal Egress
EgressInternal = "kube-vip.io/egress-internal"
// Egress should be IPv6
EgressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
EgressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
ActiveEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
)

View File

@@ -7,6 +7,7 @@ import (
"math/bits"
"os"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
@@ -500,6 +501,34 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
env = os.Getenv(zebraEnable)
if env != "" {
result, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.BGPConfig.Zebra.Enabled = result
}
env = os.Getenv(zebraURL)
if env != "" {
c.BGPConfig.Zebra.URL = env
}
env = os.Getenv(zebraVersion)
if env != "" {
u64, err := strconv.ParseUint(env, 10, 32)
if err != nil {
return err
}
c.BGPConfig.Zebra.Version = uint32(u64)
}
env = os.Getenv(zebraSoftwareName)
if env != "" {
c.BGPConfig.Zebra.SoftwareName = env
}
// Enable the load-balancer
env = os.Getenv(lbEnable)
if env != "" {
@@ -578,13 +607,13 @@ func ParseEnvironment(c *Config) error {
c.K8sConfigFile = env
}
env = os.Getenv(enableEndpointSlices)
env = os.Getenv(enableEndpoints)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableEndpointSlices = b
c.EnableEndpoints = b
}
env = os.Getenv(mirrorDestInterface)
@@ -618,5 +647,25 @@ func ParseEnvironment(c *Config) error {
c.HealthCheckPort = int(i)
}
env = os.Getenv(enableUPNP)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableUPNP = b
}
if env = os.Getenv(egressClean); env == "" {
env = os.Getenv(strings.ToUpper(egressClean))
}
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EgressClean = b
}
return nil
}

View File

@@ -110,6 +110,15 @@ const (
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
zebraURL = "zebra_url"
// zebraVersion specifies Zebra API Version
zebraVersion = "zebra_version"
// zebraSoftwareName specifies Software Name for Zebra
zebraSoftwareName = "zebra_software_name"
// mpbgpNexthop defines MPBGP mode
mpbgpNexthop = "mpbgp_nexthop"
// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
@@ -203,8 +212,8 @@ const (
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
enableEndpointSlices = "enable_endpointslices"
// enableEndpoints enables use of Endpoints instead of EndpointSlices
enableEndpoints = "enable_endpoints"
// mirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
@@ -219,4 +228,10 @@ const (
// healthCheckPort, if set to non-zero will be the port the health check will listen on
healthCheckPort = "health_check_port"
// enableUPNP enables UPNP functions
enableUPNP = "enable_upnp"
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
)

View File

@@ -100,7 +100,7 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
{
APIGroups: []string{""},
Resources: []string{"services", "endpoints"},
Verbs: []string{"list", "get", "watch", "endpoints"},
Verbs: []string{"list", "get", "watch", "update"},
},
{
APIGroups: []string{""},
@@ -521,10 +521,10 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, prometheus...)
if c.EnableEndpointSlices {
if c.EnableEndpoints {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: enableEndpointSlices,
Value: strconv.FormatBool(c.EnableEndpointSlices),
Name: enableEndpoints,
Value: strconv.FormatBool(c.EnableEndpoints),
})
}

View File

@@ -161,8 +161,8 @@ type Config struct {
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
EnableEndpoints bool `yaml:"enableEndpoints"`
// MirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
@@ -180,6 +180,12 @@ type Config struct {
// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
HealthCheckPort int `yaml:"healthCheckPort"`
// EnableUPNP, enables UPNP functions
EnableUPNP bool `yaml:"enableUPNP"`
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

View File

@@ -1,36 +0,0 @@
package manager
const (
// The host that has the VIP
vipHost = "kube-vip.io/vipHost"
// Enable egress on a service
egress = "kube-vip.io/egress"
// Egress should be IPv6
egressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
egressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
egressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
egressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
egressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
activeEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
flushContrack = "kube-vip.io/flush-conntrack"
upnpEnabled = "kube-vip.io/forwardUPNP"
)

View File

@@ -7,7 +7,6 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
@@ -17,15 +16,13 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -42,11 +39,6 @@ type Manager struct {
// Manager services
// service bool
// Keeps track of all running instances
serviceInstances []*cluster.Instance
// UPNP functionality
upnp bool
// BGP Manager, this is a singleton that manages all BGP advertisements
bgpServer *bgp.Server
@@ -56,6 +48,8 @@ type Manager struct {
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
@@ -177,6 +171,32 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// }
// }
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
var bgpServer *bgp.Server
if config.EnableBGP {
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
@@ -194,8 +214,12 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
intfMgr: networkinterface.NewManager(),
arpMgr: arp.NewManager(config),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
@@ -248,16 +272,12 @@ func (sm *Manager) Start() error {
}
if sm.config.EnableARP || sm.config.EnableWireguard {
// Before starting the leader Election enable any additional functionality
upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
if upnpEnabled {
sm.upnp = true
if sm.config.EnableUPNP {
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
sm.upnp = false
sm.config.EnableUPNP = false
} else {
for _, c := range clients {
ip, err := c.GetExternalIPAddress()
@@ -265,12 +285,11 @@ func (sm *Manager) Start() error {
log.Error("unable to find IGD2 Gateway address", "err", err)
}
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.refreshUPNPForwards()
go sm.svcProcessor.RefreshUPNPForwards()
}
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
@@ -315,66 +334,3 @@ func (sm *Manager) parseAnnotations() error {
}
return nil
}
func (sm *Manager) serviceInterface() string {
svcIf := sm.config.Interface
if sm.config.ServicesInterface != "" {
svcIf = sm.config.ServicesInterface
}
return svcIf
}
func (sm *Manager) startTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", sm.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) stopTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) findServiceInstance(svc *v1.Service) *cluster.Instance {
log.Debug("finding service", "UID", svc.UID)
for i := range sm.serviceInstances {
log.Debug("saved service", "instance", i, "UID", svc.UID)
if sm.serviceInstances[i].ServiceSnapshot.UID == svc.UID {
log.Debug("found service instance", "UID", svc.UID)
return sm.serviceInstances[i]
}
}
log.Debug("service instance not found", "UID", svc.UID)
return nil
}
// Refresh UPNP Port Forwards for all Service Instances registered in the SM
func (sm *Manager) refreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(sm.serviceInstances))
for i := range sm.serviceInstances {
sm.upnpMap(context.TODO(), sm.serviceInstances[i])
if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", sm.serviceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -2,7 +2,6 @@ package manager
import (
"context"
"os"
"syscall"
"time"
@@ -83,7 +82,7 @@ func (sm *Manager) startARP(id string) error {
}
// This will tidy any dangling kube-vip iptables rules
if os.Getenv("EGRESS_CLEAN") != "" {
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
@@ -91,7 +90,7 @@ func (sm *Manager) startARP(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -126,7 +125,7 @@ func (sm *Manager) startARP(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
@@ -134,12 +133,10 @@ func (sm *Manager) startARP(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here

View File

@@ -19,8 +19,15 @@ func (sm *Manager) startBGP() error {
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
@@ -36,9 +43,8 @@ func (sm *Manager) startBGP() error {
"peer": peerDescription,
}).Set(metricValue)
}
})
if err != nil {
return err
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
// use a Go context so we can tell the leaderelection code when we
@@ -99,7 +105,7 @@ func (sm *Manager) startBGP() error {
}
}
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
return err
}

View File

@@ -3,14 +3,13 @@ package manager
import (
"context"
"fmt"
"os"
"strconv"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
@@ -40,17 +39,10 @@ func (sm *Manager) startTableMode(id string) error {
}()
}
egressCleanEnv := os.Getenv("EGRESS_CLEAN")
if egressCleanEnv != "" {
egressClean, err := strconv.ParseBool(egressCleanEnv)
if err != nil {
log.Warn("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
}
if egressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
// Shutdown function that will wait on this signal, unless we call it ourselves
@@ -92,7 +84,7 @@ func (sm *Manager) startTableMode(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -126,7 +118,7 @@ func (sm *Manager) startTableMode(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -134,12 +126,10 @@ func (sm *Manager) startTableMode(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
@@ -156,7 +146,7 @@ func (sm *Manager) startTableMode(id string) error {
})
} else {
log.Info("beginning watching services without leader election")
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
}
@@ -167,6 +157,8 @@ func (sm *Manager) startTableMode(id string) error {
}
func (sm *Manager) cleanRoutes() error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
@@ -177,21 +169,17 @@ func (sm *Manager) cleanRoutes() error {
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = sm.countRouteReferences(&routes[i]) > 0
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] failed to delete", "route", routes[i], "err", err)
} else {
log.Debug("[route] deleted", "route", routes[i])
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}
func (sm *Manager) countRouteReferences(route *netlink.Route) int {
return cluster.CountRouteReferences(&sm.serviceInstances, route)
}

View File

@@ -56,7 +56,7 @@ func (sm *Manager) startWireguard(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -91,7 +91,7 @@ func (sm *Manager) startWireguard(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -99,12 +99,10 @@ func (sm *Manager) startWireguard(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")

View File

@@ -4,5 +4,12 @@ import "github.com/prometheus/client_golang/prometheus"
// PrometheusCollector defines a service watch event counter.
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
collectors := []prometheus.Collector{}
if sm.svcProcessor != nil {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
}

View File

@@ -57,7 +57,8 @@ func (sm *Manager) annotationsWatcher() error {
// they're as needed
log.Warn(err.Error())
rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},

View File

@@ -1,526 +0,0 @@
package manager
import (
"context"
"fmt"
"net"
"strings"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type epProvider interface {
createRetryWatcher(context.Context, *Manager,
*v1.Service) (*watchtools.RetryWatcher, error)
getAllEndpoints() ([]string, error)
getLocalEndpoints(string, *kubevip.Config) ([]string, error)
getLabel() string
updateServiceAnnotation(string, string, *v1.Service, *Manager) error
loadObject(runtime.Object, context.CancelFunc) error
getProtocol() string
}
type endpointsProvider struct {
label string
endpoints *v1.Endpoints
}
func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
}
ep.endpoints = eps
return nil
}
func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.getLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.getLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *endpointsProvider) getLabel() string {
return ep.label
}
func (ep *endpointsProvider) getProtocol() string {
return ""
}
func (sm *Manager) watchEndpoint(ctx context.Context, id string, service *v1.Service, provider epProvider) error {
log.Info("watching", "provider", provider.getLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderContext, cancel := context.WithCancel(ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.createRetryWatcher(leaderContext, sm, service)
if err != nil {
cancel()
return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-ctx.Done():
log.Debug("context cancelled", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-sm.shutdownChan:
log.Debug("shutdown called", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
var lastKnownGoodEndpoint string
for event := range ch {
activeEndpointAnnotation := activeEndpoint
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err = provider.loadObject(event.Object, cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
}
if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = activeEndpointIPv6
}
// Build endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
}
}
if sm.config.EnableRoutingTable {
instance := sm.findServiceInstance(service)
if err != nil {
log.Error("failed to find the instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
}
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", provider.getLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
continue
}
// if we haven't populated one, then do so
if lastKnownGoodEndpoint != "" {
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
if sm.config.EnableRoutingTable {
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, service.Annotations); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
log.Warn(" existing endpoint has been removed, restarting leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
leaderElectionActive = false
}
// Set our active endpoint to an existing one
lastKnownGoodEndpoint = endpoints[0]
}
} else {
lastKnownGoodEndpoint = endpoints[0]
}
if !leaderElectionActive && sm.config.EnableServicesElection {
go func() {
leaderContext, cancel = context.WithCancel(ctx)
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if leaderContext.Err() != context.Canceled {
leaderElectionActive = true
err := sm.StartServicesLeaderElection(leaderContext, service)
if err != nil {
log.Error(err.Error())
}
leaderElectionActive = false
} else {
leaderElectionActive = false
break
}
}
}()
}
// There are local endpoints available on the node
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be added per node
if sm.config.EnableRoutingTable {
instance := sm.findServiceInstance(service)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !isNetworkConfigured(service.UID, cluster.Network[i]) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
storeConfiguredNetwork(string(service.UID), cluster.Network[i])
leaderElectionActive = true
}
}
}
}
}
}
// If BGP mode is enabled - hosts should be added per node
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !isNetworkConfigured(service.UID, cluster.Network[i]) {
network := cluster.Network[i]
if err != nil {
log.Error("error formatting address with subnet mask", "err", err)
}
log.Debug("attempting to advertise BGP service", "provider", provider.getLabel(), "ip", network.CIDR())
err = sm.bgpServer.AddHost(network.CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", provider.getLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
provider.getLabel(), "ip", network.CIDR(), "service name", service.Name, "namespace", service.Namespace)
storeConfiguredNetwork(string(service.UID), cluster.Network[i])
leaderElectionActive = true
}
}
}
}
}
}
}
} else {
// There are no local endpoints
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be deleted
if sm.config.EnableRoutingTable {
if errs := sm.clearRoutes(service); len(errs) == 0 {
delete(configuredNetworks, string(service.UID))
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
// If BGP mode is enabled - routes should be deleted
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err = sm.bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] deleting BGP host", "provider", provider.getLabel(), "ip", network.CIDR(), "err", err)
} else {
log.Info("[endpoint] deleted BGP host", "provider",
provider.getLabel(), "ip", network.CIDR(), "service name", service.Name, "namespace", service.Namespace)
delete(configuredNetworks[string(service.UID)], cluster.Network[i].IP())
leaderElectionActive = false
}
}
}
}
}
}
// If there are no local endpoints, and we had one then remove it and stop the leaderElection
if lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, service.Annotations); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
lastKnownGoodEndpoint = "" // reset endpoint
if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
cancel() // stop services watcher
}
leaderElectionActive = false
}
}
// Set the service accordingly
if service.Annotations[egress] == "true" {
service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
}
log.Debug("watcher", "provider",
provider.getLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", lastKnownGoodEndpoint, "active leader election", leaderElectionActive)
case watch.Deleted:
// When no-leader-elecition mode
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// find all existing local endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
// Delete all routes in routing table mode
if sm.config.EnableRoutingTable {
sm.clearRoutes(service)
}
// Delete all hosts in BGP mode
if sm.config.EnableBGP {
sm.clearBGPHosts(service)
}
}
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.getLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}
func (sm *Manager) clearRoutes(service *v1.Service) []error {
errs := []error{}
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if sm.countRouteReferences(route) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
}
}
}
}
return errs
}
func (sm *Manager) clearBGPHosts(service *v1.Service) {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := sm.bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", service.Name, "namespace", service.Namespace)
}
}
}
}
}
func storeConfiguredNetwork(svcUID string, network vip.Network) {
if _, exists := configuredNetworks[svcUID]; !exists {
configuredNetworks[svcUID] = make(map[string]vip.Network)
}
configuredNetworks[svcUID][network.IP()] = network
}

View File

@@ -1,408 +0,0 @@
package manager
import (
"context"
"fmt"
"reflect"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// TODO: Fix the naming of these contexts
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancer map[string]context.Context
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancerCancel map[string]func()
// activeService keeps track of services that already have a leaderElection in place
var activeService map[string]bool
// watchedService keeps track of services that are already being watched
var watchedService map[string]bool
// configuredNetworks keeps track of networks that has been configured on the node
var configuredNetworks map[string]map[string]vip.Network
func init() {
// Set up the caches for monitoring existing active or watched services
activeServiceLoadBalancerCancel = make(map[string]func())
activeServiceLoadBalancer = make(map[string]context.Context)
activeService = make(map[string]bool)
watchedService = make(map[string]bool)
configuredNetworks = make(map[string]map[string]vip.Network)
}
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := sm.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := sm.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if sm.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
sm.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", sm.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
break
}
// Select loadbalancer class filtering function
lbClassFilterFunc := sm.lbClassFilter
if sm.config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = sm.lbClassFilterLegacy
}
// Check the loadBalancer class
if lbClassFilterFunc(svc) {
break
}
svcAddresses := cluster.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
break
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := sm.findServiceInstance(svc)
originalService := []string{}
shouldGarbageCollect := true
if i != nil {
originalService = cluster.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(sm.config.Interface, addr, sm.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", sm.config.Interface)
}
}
}
// This service has been modified, but it was also active..
if activeService[string(svc.UID)] {
if i != nil {
if !reflect.DeepEqual(originalService, svcAddresses) {
// Calls the cancel function of the context
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
activeServiceLoadBalancerCancel[string(svc.UID)]()
<-activeServiceLoadBalancer[string(svc.UID)].Done()
log.Warn("(svcs) waiting for load balancer to finish")
}
err = sm.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
activeService[string(svc.UID)] = false
watchedService[string(svc.UID)] = false
delete(activeServiceLoadBalancer, string(svc.UID))
delete(configuredNetworks, string(svc.UID))
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if !activeService[string(svc.UID)] {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", cluster.FetchServiceAddresses(svc))
activeServiceLoadBalancer[string(svc.UID)], activeServiceLoadBalancerCancel[string(svc.UID)] = context.WithCancel(ctx)
if sm.config.EnableServicesElection || // Service Election
((sm.config.EnableRoutingTable || sm.config.EnableBGP) && // Routing table mode or BGP
(!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !watchedService[string(svc.UID)] {
// background the endpoint watcher
if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
watchedService[string(svc.UID)] = true
}
} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
watchedService[string(svc.UID)] = true
} else {
go func() {
for {
select {
case <-activeServiceLoadBalancer[string(svc.UID)].Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}
activeService[string(svc.UID)] = true
}
case watch.Deleted:
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
if activeService[string(svc.UID)] {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
break
}
// If no leader election is enabled, delete routes here
if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
sm.config.EnableRoutingTable && hasConfiguredNetworks(svc.UID) {
if errs := sm.clearRoutes(svc); len(errs) == 0 {
delete(configuredNetworks, string(svc.UID))
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = sm.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
activeServiceLoadBalancerCancel[string(svc.UID)]()
}
activeService[string(svc.UID)] = false
watchedService[string(svc.UID)] = false
}
if sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
if sm.config.EnableBGP {
sm.clearBGPHosts(svc)
} else if sm.config.EnableRoutingTable {
sm.clearRoutes(svc)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if sm.config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
return false
}
func hasConfiguredNetworks(serviceUID types.UID) bool {
_, ok := configuredNetworks[string(serviceUID)]
if !ok {
return false
}
return len(configuredNetworks[string(serviceUID)]) > 0
}
func isNetworkConfigured(serviceUID types.UID, network vip.Network) bool {
_, exists := configuredNetworks[string(serviceUID)]
if !exists {
return false
}
_, exists = configuredNetworks[string(serviceUID)][network.IP()]
return exists
}

View File

@@ -32,7 +32,3 @@ func (m *Manager) Get(intf netlink.Link) *Link {
m.interfaces[intf.Attrs().Name] = result
return result
}
func (m *Manager) Delete(intf netlink.Link) {
delete(m.interfaces, intf.Attrs().Name)
}

458
pkg/nftables/nftables.go Normal file
View File

@@ -0,0 +1,458 @@
package nftables
import (
"errors"
"fmt"
"log/slog"
"net"
"strconv"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"golang.org/x/sys/unix"
)
const (
NatTable = "kube_vip_%s"
SNatChain = "kube_vip_snat_%s"
)
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
conn, err := nftables.New()
if err != nil {
return err
}
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Look up the table
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
if t == nil {
// If it doesn't exist then create it
slog.Debug("[egress]", "Creating Table", tableName)
conn.AddTable(GetTable(IPv6))
}
}
slog.Debug("[egress]", "Creating Chain for service", service, "IPv6", IPv6)
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddChain(GetSNatChain(IPv6, service))
conn.Flush()
// Create our nftables rule
rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
if err != nil {
return err
}
slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
conn.AddRule(rule) // Add the rule
err = conn.Flush() // Commit the rule to nftables
if err != nil {
return err
}
return conn.CloseLasting() // Close out any remaining netlink communication
}
func DeleteSNAT(IPv6 bool, service string) error {
conn, err := nftables.New()
if err != nil {
return err
}
var chainName = fmt.Sprintf(SNatChain, service)
slog.Info("[egress]", "Looking for", chainName)
chain, err := conn.ListChain(GetTable(IPv6), chainName)
if err != nil {
return err
}
if chain != nil {
slog.Info("[egress]", "Deleting chain", chainName)
conn.DelChain(chain)
return conn.Flush()
}
return fmt.Errorf("unable to find chain [%s]", chainName)
}
func GetTable(IPv6 bool) *nftables.Table {
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Default to IPv4
table := &nftables.Table{
Family: nftables.TableFamilyIPv4,
Name: tableName,
}
// Move to IPv6 if needed
if IPv6 {
table.Family = nftables.TableFamilyIPv6
}
return table
}
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
var chainName = fmt.Sprintf(SNatChain, service)
policy := nftables.ChainPolicyAccept
return &nftables.Chain{
Name: chainName,
Table: GetTable(IPv6),
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityNATSource,
Policy: &policy,
}
}
func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
if IPv6 {
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
}
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
}
// ClearTable will remove the original tables and create new empty ones
func ClearTable(conn *nftables.Conn) error {
tableName := fmt.Sprintf(NatTable, "v6")
if t, err := FilterTable(conn, tableName, false); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(true))
tableName = fmt.Sprintf(NatTable, "v4")
if t, err := FilterTable(conn, tableName, true); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(false))
return nil
}
// Create our nftables rule
func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
// Validate pod IP
if net.ParseIP(podIP) == nil {
return nil, errors.New("ip is invalid")
}
// Validate vip IP
if net.ParseIP(vipIP) == nil {
return nil, errors.New("output_ip is not a valid ip")
}
// Get the kube-vip table
table := GetTable(IPv6)
// Create our rule
rule := &nftables.Rule{
Table: table,
Exprs: []expr.Any{},
}
// Set the correct chain
rule.Chain = GetSNatChain(IPv6, service)
// Create a set for our original/source address
set := &nftables.Set{
Table: table,
Anonymous: true,
Constant: true,
KeyType: nftables.TypeIPAddr,
Interval: false,
}
if IPv6 {
set.KeyType = nftables.TypeIP6Addr
} else {
set.KeyType = nftables.TypeIPAddr
}
// Create an element using our pod IP
elements := []nftables.SetElement{}
if IPv6 {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
} else {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
}
// Add the elements to the set
err := conn.AddSet(set, elements)
if err != nil {
return nil, err
}
// Create the expression using the set
expression := []expr.Any{}
payload := &expr.Payload{
OperationType: expr.PayloadLoad,
Base: expr.PayloadBaseNetworkHeader,
DestRegister: 1,
SourceRegister: 0,
}
// Set the length of the data based upon the type of IP version being used
if IPv6 {
payload.Offset = 8
payload.Len = 16
} else {
payload.Offset = 12
payload.Len = 4
}
lookup := &expr.Lookup{
SourceRegister: 1,
DestRegister: 0,
SetID: set.ID,
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, lookup)
// Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
// If we filter on ports protocols then parse them
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
// Create an element using our pod IP
tcpElements := []nftables.SetElement{}
udpElements := []nftables.SetElement{}
sctpElements := []nftables.SetElement{}
tcpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
udpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
sctpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
for _, fixedPort := range fixedPorts {
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 2 { // Ensure we have two elements { proto:port }
// parse the port to a number
port, err := strconv.Atoi(data[1])
if err != nil {
slog.Error("[egress]", "unable to process port", data[1])
continue
}
// Ensure the port is within the valid range for uint16
if port < 0 || port > 65535 {
slog.Error("[egress]", "port out of range for uint16", data[1])
continue
}
switch data[0] {
case "tcp":
//nolint:gosec
tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "udp":
//nolint:gosec
udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "sctp":
//nolint:gosec
sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
default:
slog.Error("[egress]", "unknown protocol", data[0])
}
}
}
// Add TCP Ports
if len(tcpElements) != 0 {
err = conn.AddSet(tcpSet, tcpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_TCP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: tcpSet.Name,
SetID: tcpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add UDP ports
if len(udpElements) != 0 {
err = conn.AddSet(udpSet, udpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_UDP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: udpSet.Name,
SetID: udpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add SCTP Ports
if len(sctpElements) != 0 {
err = conn.AddSet(sctpSet, sctpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_SCTP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: sctpSet.Name,
SetID: sctpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
}
// Parse which CIDRs we will not SNAT for
for _, cidr := range ignoreCIDR {
start, end, err := nftables.NetFirstAndLastIP(cidr)
if err != nil {
return nil, err
}
expression = []expr.Any{}
payload := &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
}
notEqualRange := &expr.Range{
Op: expr.CmpOpNeq,
Register: 1,
}
if IPv6 {
payload.Len = 16
payload.Offset = 24
notEqualRange.FromData = start.To16()
notEqualRange.ToData = end.To16()
} else {
payload.Offset = 16
payload.Len = 4
notEqualRange.FromData = start.To4()
notEqualRange.ToData = end.To4()
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, notEqualRange)
// // Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
}
// Final expression to the rule is the SNAT to the VIP address
expression = []expr.Any{}
immediate := &expr.Immediate{
Register: 1,
}
nat := &expr.NAT{
Type: expr.NATTypeSourceNAT,
RegAddrMin: 1,
RegAddrMax: 1,
RegProtoMin: 0,
RegProtoMax: 0,
Random: false,
FullyRandom: false,
Persistent: false,
Prefix: false,
}
if IPv6 {
immediate.Data = net.ParseIP(vipIP).To16()
nat.Family = unix.NFPROTO_IPV6
} else {
immediate.Data = net.ParseIP(vipIP).To4()
nat.Family = unix.NFPROTO_IPV4
}
// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
// Add expressions
expression = append(expression, immediate)
expression = append(expression, nat)
rule.Exprs = append(rule.Exprs, expression...)
return rule, nil
}

View File

@@ -0,0 +1,36 @@
package servicecontext
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func New(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

36
pkg/services/context.go Normal file
View File

@@ -0,0 +1,36 @@
package services
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func NewContext(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"bufio"
@@ -11,6 +11,8 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -21,7 +23,7 @@ const (
defaultServiceCIDR = "10.96.0.0/12"
)
func (sm *Manager) iptablesCheck() error {
func (p *Processor) iptablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
@@ -48,6 +50,31 @@ func (sm *Manager) iptablesCheck() error {
return nil
}
func (p *Processor) nftablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
var queue, ct bool
for scanner.Scan() {
line := strings.Fields(scanner.Text())
switch line[0] {
case "nft_queue":
queue = true
case "nft_ct":
ct = true
}
}
if !queue || !ct {
return fmt.Errorf("missing nftables modules -> nft_ct [%t] -> ntf_queue [%t]", ct, queue)
}
return nil
}
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
cidrs := strings.Split(sourceCidrs, ",")
isV6 := vip.IsIPv6(ip)
@@ -107,26 +134,27 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations map[string]string) error {
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
}
if discoverErr != nil {
log.Warn("autodiscover CIDR", "err", discoverErr)
}
if sm.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
if p.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
} else {
if discoverErr == nil {
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
@@ -141,8 +169,8 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
podCidr = defaultPodCIDR
}
if sm.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
if p.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
} else {
if discoverErr == nil {
serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
@@ -180,7 +208,30 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
// Use the internal egress implementation
if internalEgress != "" {
// Create an array of CIDRs that we wont SNAT to.
ignoreCIDRs := []string{
podCidr,
serviceCidr,
}
// Add any specifically denied networks
if deniedNetworks != "" {
networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
ignoreCIDRs = append(ignoreCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
return nil
}
i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
@@ -279,12 +330,12 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
return nil
}
func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}
@@ -307,83 +358,3 @@ func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
return
}
func (sm *Manager) TeardownEgress(podIP, vipIP, namespace string, annotations map[string]string) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
protocol := iptables.ProtocolIPv4
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import "testing"

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"context"
@@ -22,14 +22,14 @@ func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
if err != nil {
return err
}
for _, instance := range sm.serviceInstances {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
@@ -44,9 +44,9 @@ func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) erro
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", sm.config.NodeName)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
@@ -54,9 +54,9 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
Name: serviceLease,
Namespace: service.Namespace,
},
Client: sm.clientSet.CoordinationV1(),
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: sm.config.NodeName,
Identity: p.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
@@ -69,7 +69,16 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
activeService[string(service.UID)] = true
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
if svcCtx == nil {
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
svcCtx.IsActive = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
Lock: lock,
@@ -80,32 +89,32 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := sm.syncServices(ctx, service); err != nil {
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", sm.config.NodeName)
if activeService[string(service.UID)] {
if err := sm.deleteService(service.UID); err != nil {
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
activeService[string(service.UID)] = false
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == sm.config.NodeName {
if identity == p.config.NodeName {
// I just got the lock
return
}

344
pkg/services/processor.go Normal file
View File

@@ -0,0 +1,344 @@
package services
import (
"context"
"fmt"
log "log/slog"
"reflect"
"sync"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
config *kubevip.Config
lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
svcMap sync.Map
// Keeps track of all running instances
ServiceInstances []*instance.Instance
mutex sync.Mutex
bgpServer *bgp.Server
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
}
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
}
return &Processor{
config: config,
lbClassFilter: lbClassFilterFunc,
ServiceInstances: []*instance.Instance{},
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return true, nil
}
svcAddresses := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
return true, nil
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := instance.FindServiceInstance(svc, p.ServiceInstances)
var originalService []string
shouldGarbageCollect := true
if i != nil {
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
if i != nil {
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
newService := instance.FetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
err = p.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
p.svcMap.Delete(svc.UID)
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
if svcCtx == nil {
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.IsActive = true
}
return false, nil
}
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil && svcCtx.IsActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
}
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
if p.config.EnableBGP {
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
} else if p.config.EnableRoutingTable {
endpoints.ClearRoutes(svc, &p.ServiceInstances)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return true, nil
}
func (p *Processor) Stop() {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
}
func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
svcCtx, ok := p.svcMap.Load(uid)
if !ok {
return nil, nil
}
ctx, ok := svcCtx.(*servicecontext.Context)
if !ok {
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
}
return ctx, nil
}
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"context"
@@ -17,62 +17,93 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync")
type ServiceInstanceAction string
const (
ActionDelete ServiceInstanceAction = "delete"
ActionAdd ServiceInstanceAction = "add"
ActionNone ServiceInstanceAction = "none"
)
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
foundInstance := false
newServiceAddresses := cluster.FetchServiceAddresses(svc)
newServiceUID := svc.UID
ingressIPs := []string{}
for _, ingress := range svc.Status.LoadBalancer.Ingress {
ingressIPs = append(ingressIPs, ingress.IP)
}
shouldBreake := false
for x := range sm.serviceInstances {
if shouldBreake {
break
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
for _, newServiceAddress := range newServiceAddresses {
log.Debug("service", "isDHCP", sm.serviceInstances[x].IsDHCP, "newServiceAddress", newServiceAddress)
if sm.serviceInstances[x].ServiceSnapshot.UID == newServiceUID {
// If the found instance's DHCP configuration doesn't match the new service, delete it.
if (sm.serviceInstances[x].IsDHCP && newServiceAddress != "0.0.0.0") ||
(!sm.serviceInstances[x].IsDHCP && newServiceAddress == "0.0.0.0") ||
(!sm.serviceInstances[x].IsDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, newServiceAddress)) ||
(len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc)) ||
(sm.serviceInstances[x].IsDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, sm.serviceInstances[x].DHCPInterfaceIP)) {
if err := sm.deleteService(newServiceUID); err != nil {
return err
}
shouldBreake = true
break
}
foundInstance = true
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
}
// This instance wasn't found, we need to add it to the manager
if !foundInstance && len(newServiceAddresses) > 0 {
if err := sm.addService(ctx, svc); err != nil {
return err
}
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
return nil
}
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
addresses := instance.FetchServiceAddresses(svc)
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCP {
if address != "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
return ActionDelete
}
}
if !instance.IsDHCP {
if address == "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
return ActionDelete
}
}
if !comparePortsAndPortStatuses(svc) {
return ActionDelete
}
}
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
return ActionNone
}
}
if len(addresses) > 0 {
log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
return ActionAdd // If no matching instance is found, we need to add a new service instance
}
return ActionNone
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
if len(svc.Status.LoadBalancer.Ingress) == 0 {
return false
}
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
if len(portsStatus) != len(svc.Spec.Ports) {
return false
@@ -85,20 +116,24 @@ func comparePortsAndPortStatuses(svc *v1.Service) bool {
return true
}
func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
newService, err := cluster.NewInstance(svc, sm.config, sm.intfMgr, sm.arpMgr)
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(newService.VIPConfigs[x], sm.bgpServer, svc.Name, &sm.serviceInstances)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
sm.upnpMap(ctx, newService)
p.upnpMap(ctx, newService)
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
@@ -106,8 +141,8 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !sm.config.DisableServiceUpdates {
if err := sm.updateStatus(newService); err != nil {
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
@@ -116,26 +151,26 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}()
}
sm.serviceInstances = append(sm.serviceInstances, newService)
p.ServiceInstances = append(p.ServiceInstances, newService)
if !sm.config.DisableServiceUpdates {
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := sm.updateStatus(newService); err != nil {
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs := cluster.FetchServiceAddresses(svc)
serviceIPs := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[flushContrack] == "true" {
if svc.Annotations[kubevip.FlushContrack] == "true" {
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -143,26 +178,35 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
// We will need to modify the iptables rules
err = sm.iptablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
if p.config.EgressWithNftables {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.nftablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.iptablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
var podIP string
errList := []error{}
// Should egress be IPv6
if svc.Annotations[egressIPv6] == "true" {
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[activeEndpointIPv6] != "" {
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[activeEndpointIPv6]
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err = sm.configureEgress(serviceIP, podIP, svc.Namespace, svc.Annotations)
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
@@ -170,13 +214,13 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
}
}
} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[activeEndpoint]
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[activeEndpointIPv6]
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err = sm.configureEgress(serviceIP, podIPs, svc.Namespace, svc.Annotations)
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
@@ -184,13 +228,13 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
}
if len(errList) == 0 {
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
provider = providers.NewEndpointslices()
}
err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -203,23 +247,23 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
return nil
}
func (sm *Manager) deleteService(uid types.UID) error {
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
sm.mutex.Lock()
defer sm.mutex.Unlock()
p.mutex.Lock()
defer p.mutex.Unlock()
var updatedInstances []*cluster.Instance
var serviceInstance *cluster.Instance
var updatedInstances []*instance.Instance
var serviceInstance *instance.Instance
found := false
for x := range sm.serviceInstances {
log.Debug("service lookup", "target UID", uid, "found UID ", sm.serviceInstances[x].ServiceSnapshot.UID)
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if sm.serviceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, sm.serviceInstances[x])
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = sm.serviceInstances[x]
serviceInstance = p.ServiceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
@@ -239,66 +283,67 @@ func (sm *Manager) deleteService(uid types.UID) error {
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
for _, vip := range cluster.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
for _, vip := range cluster.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
}
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("error finding VIP Interface: %v", err)
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("error deleting DHCP Link : %v", err)
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
for i := range serviceInstance.VIPConfigs {
if serviceInstance.VIPConfigs[i].EnableBGP {
sm.clearBGPHosts(serviceInstance.ServiceSnapshot)
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[activeEndpoint] != "" {
log.Info("egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := sm.TeardownEgress(serviceInstance.ServiceSnapshot.Annotations[activeEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, serviceInstance.ServiceSnapshot.Annotations)
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
if err != nil {
log.Error("egress teardown", "err", err)
log.Error("[service] egress teardown", "err", err)
}
}
}
}
// Update the service array
sm.serviceInstances = updatedInstances
p.ServiceInstances = updatedInstances
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(sm.serviceInstances))
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
if !isUPNPEnabled(s.ServiceSnapshot) {
// Skip services missing the annotation
return
}
if !sm.upnp {
if !p.config.EnableUPNP {
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
return
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
@@ -308,10 +353,10 @@ func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
for _, vip := range cluster.FetchServiceAddresses(s.ServiceSnapshot) {
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.ConnectionClient.GetServiceClient().Location)
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
@@ -351,7 +396,7 @@ func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (sm *Manager) updateStatus(i *cluster.Instance) error {
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
@@ -363,7 +408,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -374,13 +419,13 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if sm.config.EnableARP {
if p.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[cluster.HWAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[cluster.RequestedIP] = i.DHCPInterfaceIP
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
@@ -389,7 +434,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
@@ -408,7 +453,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
for _, c := range i.VIPConfigs {
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
@@ -438,7 +483,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
@@ -451,5 +496,21 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -0,0 +1,94 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.Ctx.Done():
log.Debug("context cancelled", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-p.shutdownChan:
log.Debug("shutdown called", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
var lastKnownGoodEndpoint string
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
if restart {
continue
} else if err != nil {
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
}
case watch.Deleted:
if err := epProcessor.Delete(service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}

View File

@@ -0,0 +1,183 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := p.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := p.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if p.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
p.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-p.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := p.AddOrModify(ctx, event, serviceFunc)
if restart {
break
}
if err != nil {
return fmt.Errorf("add/modify service error: %w", err)
}
case watch.Deleted:
restart, err := p.Delete(event)
if restart {
break
}
if err != nil {
return fmt.Errorf("delete service error: %w", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
return true
}
return false
}
func (p *Processor) serviceInterface() string {
svcIf := p.config.Interface
if p.config.ServicesInterface != "" {
svcIf = p.config.ServicesInterface
}
return svcIf
}
func (p *Processor) startTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (p *Processor) stopTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}

View File

@@ -2,8 +2,8 @@ package vip
import (
"fmt"
"math"
"net"
"os"
"slices"
"strconv"
"strings"
@@ -36,6 +36,7 @@ type Network interface {
DeleteIP() (bool, error)
DeleteRoute() error
UpdateRoutes() (bool, error)
IsSet() (bool, error)
IP() string
CIDR() string
IPisLinkLocal() bool
@@ -62,6 +63,7 @@ type network struct {
link *networkinterface.Link
ports []v1.ServicePort
serviceName string
enableSecurity bool
ignoreSecurity bool
dnsName string
@@ -81,7 +83,8 @@ type network struct {
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled bool, intfMgr *networkinterface.Manager) ([]Network, error) {
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -117,6 +120,12 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
if iface == "lo" && !loGlobalScope {
// set host scope on loopback, otherwise global scope will be used by default
result.address.Scope = unix.RT_SCOPE_HOST
@@ -139,6 +148,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
networks = append(networks, result)
@@ -157,6 +168,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
// we're able to resolve store this as the initial IP
@@ -167,6 +180,9 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
networks = append(networks, result)
}
@@ -298,7 +314,7 @@ func (configurator *network) AddIP(precheck bool) (bool, error) {
exists := false
var err error
if precheck {
if exists, err = configurator.isSet(); err != nil {
if exists, err = configurator.IsSet(); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
}
}
@@ -319,7 +335,7 @@ func (configurator *network) AddIP(precheck bool) (bool, error) {
}
func (configurator *network) configureIPTables() error {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
}
@@ -468,7 +484,7 @@ func (configurator *network) DeleteIP() (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
result, err := configurator.isSet()
result, err := configurator.IsSet()
if err != nil {
return false, errors.Wrap(err, "ip check in DeleteIP failed")
}
@@ -479,9 +495,10 @@ func (configurator *network) DeleteIP() (bool, error) {
}
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
return false, fmt.Errorf("could not delete IP %q from interface %q: %w", configurator.address.IPNet.String(), configurator.link.Intf.Attrs().Name, err)
return false, errors.Wrap(err, "could not delete ip")
}
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
}
@@ -588,7 +605,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
}
// isSet - Check to see if VIP is set
func (configurator *network) isSet() (result bool, err error) {
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
if configurator.address == nil {
@@ -633,7 +650,13 @@ func (configurator *network) SetIP(ip string) error {
}
if configurator.address != nil && configurator.IsDNS() {
addr.ValidLft = defaultValidLft
} else {
addr.ValidLft = math.MaxInt
}
// set address as deprecated so it isn't used as source address according to RFC 3484
addr.PreferedLft = 0
configurator.address = addr
return nil
}

View File

@@ -135,9 +135,7 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
Halen: m.hardwareAddressLength,
}
target := ethernetBroadcast
for i := 0; i < len(target); i++ { //nolint
ll.Addr[i] = target[i]
}
copy(ll.Addr[:], target)
b, err := m.bytes()
if err != nil {

13
test.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/bash
rm -f logs.txt
for i in $(seq 1 100);
do
echo "RUN $i"
GOMAXPROCS=4 make e2e-tests129
if [[ "$?" -ne 0 ]]; then
echo "FAILED AT RUN $i"
break
fi
done

View File

@@ -0,0 +1,3 @@
[global.config]
as = {{ .AS }}
router-id = "1.1.1.1"

683
testing/e2e/e2e_bgp_test.go Normal file
View File

@@ -0,0 +1,683 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"text/template"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
api "github.com/osrg/gobgp/v3/api"
)
const (
goBGPAS uint32 = 65500
kubevipAS uint32 = 65501
goBGPPort uint32 = 50051
defaultFixedNexthopv6 = "fc00:1000:1000:1000::100"
defaultFixedNexthopv4 = "172.18.0.100"
)
var _ = Describe("kube-vip BGP mode", Ordered, func() {
if Mode == ModeBGP {
var (
logger log.Logger
imagePath string
k8sImagePath string
configPath string
kubeVIPBGPManifestTemplate *template.Template
goBGPConfigTemplate *template.Template
tempDirPath string
v129 bool
localIPv4 string
localIPv6 string
curDir string
networkInterface string
bgpKill chan any
)
BeforeAll(func() {
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
if configPath == "" {
configPath = "/etc/kubernetes/admin.conf"
}
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
_, v129 = os.LookupEnv("V129")
var err error
curDir, err = os.Getwd()
Expect(err).NotTo(HaveOccurred())
templateBGPPath := filepath.Join(curDir, "kube-vip-bgp.yaml.tmpl")
kubeVIPBGPManifestTemplate, err = template.New("kube-vip-bgp.yaml.tmpl").ParseFiles(templateBGPPath)
Expect(err).NotTo(HaveOccurred())
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).NotTo(HaveOccurred())
v4addr, _, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv4 = v4addr.String()
v6addr, _, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv6 = v6addr.String()
goBGPConfig := &e2e.BGPPeerValues{
AS: goBGPAS,
}
bgpKill = make(chan any)
goBGPConfigPath := filepath.Join(filepath.Join(curDir, "bgp"), "config.toml.tmpl")
goBGPConfigTemplate, err = template.New("config.toml.tmpl").ParseFiles(goBGPConfigPath)
Expect(err).ToNot(HaveOccurred())
goBGPConfigPath = filepath.Join(tempDirPath, "config.toml")
f, err := os.OpenFile(goBGPConfigPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
Expect(err).ToNot(HaveOccurred())
defer f.Close()
err = goBGPConfigTemplate.Execute(f, goBGPConfig)
Expect(err).ToNot(HaveOccurred())
go startGoBGP(goBGPConfigPath, bgpKill)
})
AfterAll(func() {
close(bgpKill)
})
Describe("kube-vip IPv4 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv4Family, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip IPv6 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv6Family, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - auto_sourceif nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
_, containerIP = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
containerIP, _ = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
}
})
func testBGP(offset uint, lbFamily string, afiFamily api.Family_Afi, svcFamily []corev1.IPFamily, svcName string,
trafficPolicy corev1.ServiceExternalTrafficPolicy, client kubernetes.Interface, numberOfServices int, gobgpClient api.GobgpApiClient, expectedNexthop string) {
lbAddress := e2e.GenerateVIP(lbFamily, offset)
routeCheckFamily := &api.Family{
Afi: afiFamily,
Safi: api.Family_SAFI_UNICAST,
}
testServiceBGP(svcName, lbAddress, trafficPolicy, client, svcFamily, numberOfServices, gobgpClient, routeCheckFamily, expectedNexthop)
}
func setupEnv(tempDirPath, cpVIP, clusterName *string, manifestValues *e2e.KubevipManifestValues,
localIPv4, localIPv6, imagePath, configPath, k8sImagePath, clusterAddrFamily, bgpClientAddrFamily string, peerAddrFamily []string, client *kubernetes.Interface,
gobgpPeers *[]*e2e.BGPPeerValues, v129 bool, kubeVIPBGPManifestTemplate *template.Template, gobgpClient *api.GobgpApiClient,
logger log.Logger, nodesNumber int, mpbgpnexthop, clusterNameSuffix string) (string, string) {
var err error
*tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).ToNot(HaveOccurred())
*cpVIP = e2e.GenerateVIP(clusterAddrFamily, SOffset.Get())
var clusterIPFamily kindconfigv1alpha4.ClusterIPFamily
var podSubnet, serviceSubnet string
switch clusterAddrFamily {
case e2e.IPv6Family:
clusterIPFamily = kindconfigv1alpha4.IPv6Family
case e2e.DualstackFamily:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
case e2e.DualstackFamilyIPv6:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
podSubnet = "fd00:10:244::/56,10.244.0.0/16"
serviceSubnet = "fd00:10:96::/112,10.96.0.0/16"
default:
clusterIPFamily = kindconfigv1alpha4.IPv4Family
}
networking := &kindconfigv1alpha4.Networking{
IPFamily: clusterIPFamily,
}
if podSubnet != "" && serviceSubnet != "" {
networking.PodSubnet = podSubnet
networking.ServiceSubnet = serviceSubnet
}
kvPeers := []*e2e.BGPPeerValues{}
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv4,
AS: goBGPAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv6,
AS: goBGPAS,
IPFamily: e2e.IPv6Family,
})
}
kvPeersStr := []string{}
for _, p := range kvPeers {
kvPeersStr = append(kvPeersStr, p.String())
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: *cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
BGPAS: kubevipAS,
BGPPeers: strings.Join(kvPeersStr, ","),
MPBGPNexthop: mpbgpnexthop,
MPBGPNexthopIPv4: defaultFixedNexthopv4,
MPBGPNexthopIPv6: defaultFixedNexthopv6,
}
By(manifestValues.BGPPeers)
*clusterName, *client = prepareCluster(*tempDirPath, clusterNameSuffix, k8sImagePath, v129, kubeVIPBGPManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
container := fmt.Sprintf("%s-control-plane", *clusterName)
containerIPv4, containerIPv6, err := GetContainerIPs(container)
Expect(err).ToNot(HaveOccurred())
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv4,
AS: kubevipAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv6,
AS: kubevipAS,
IPFamily: e2e.IPv6Family,
})
}
if bgpClientAddrFamily == e2e.IPv6Family {
*gobgpClient, err = newGoBGPClient(localIPv6, goBGPPort)
} else {
*gobgpClient, err = newGoBGPClient(localIPv4, goBGPPort)
}
Expect(err).ToNot(HaveOccurred())
for _, p := range *gobgpPeers {
if slices.Contains(peerAddrFamily, p.IPFamily) {
peerCtx := context.TODO()
Eventually(peerCtx, func() error {
_, err = (*gobgpClient).AddPeer(context.TODO(), &api.AddPeerRequest{
Peer: &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: p.IP,
PeerAsn: uint32(p.AS),
},
AfiSafis: []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
},
},
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
},
},
},
},
})
return err
}, "120s", "100ms").Should(Succeed())
}
}
return containerIPv4, containerIPv6
}
func testServiceBGP(svcName, lbAddress string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
client kubernetes.Interface, serviceAddrFamily []corev1.IPFamily, numberOfServices int,
gobgpClient api.GobgpApiClient, gobgpFamily *api.Family, expectedNexthop string) {
lbAddresses := vip.Split(lbAddress)
services := []string{}
for i := range numberOfServices {
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
}
for _, svc := range services {
createTestService(svc, dsNamespace, dsName, lbAddress,
client, corev1.IPFamilyPolicyPreferDualStack, serviceAddrFamily, trafficPolicy)
}
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
for i := range numberOfServices {
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
if i < numberOfServices-1 {
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
}
}
for _, addr := range lbAddresses {
checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 0)
}
}
func GetContainerIPs(containerName string) (string, string, error) {
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
panic(err)
}
containers, err := cli.ContainerList(context.Background(), container.ListOptions{})
if err != nil {
return "", "", fmt.Errorf("failed to list containers: %w", err)
}
for _, c := range containers {
for _, n := range c.Names {
if n[1:] == containerName {
fmt.Println(n)
for _, n := range c.NetworkSettings.Networks {
return n.IPAddress, n.GlobalIPv6Address, nil
}
}
}
}
return "", "", nil
}
func newGoBGPClient(address string, port uint32) (api.GobgpApiClient, error) {
grpcOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
target := net.JoinHostPort(address, strconv.Itoa(int(port)))
conn, err := grpc.NewClient(target, grpcOpts...)
if err != nil {
return nil, fmt.Errorf("failed to connect to GoBGP server %q: %w", target, err)
}
return api.NewGobgpApiClient(conn), nil
}
func checkGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedPaths int) []*api.Destination {
var paths []*api.Destination
Eventually(func() error {
var err error
paths, err = getGoBGPPaths(ctx, client, family, prefixes)
if err != nil {
return err
}
if len(paths) != expectedPaths {
return fmt.Errorf("expected %d paths, but found %d", expectedPaths, len(paths))
}
return nil
}, "120s").ShouldNot(HaveOccurred())
return paths
}
func getGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix) ([]*api.Destination, error) {
pathCtx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
stream, err := client.ListPath(pathCtx, &api.ListPathRequest{
TableType: api.TableType_GLOBAL,
Family: family,
Name: "",
Prefixes: prefixes,
SortType: api.ListPathRequest_PREFIX,
})
if err != nil {
return nil, err
}
rib := make([]*api.Destination, 0)
for {
r, err := stream.Recv()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
rib = append(rib, r.Destination)
}
return rib, nil
}
func startGoBGP(config string, kill chan any) {
By("starting GoBGP server")
cmd := exec.Command("../../bin/gobgpd", "-f", config)
go cmd.Run()
<-kill
By("stopping GoBGP server")
err := cmd.Process.Kill()
Expect(err).ToNot(HaveOccurred())
}

583
testing/e2e/e2e_rt_test.go Normal file
View File

@@ -0,0 +1,583 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"text/template"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
)
var _ = Describe("kube-vip routing table mode", func() {
if Mode == ModeRT {
var (
logger log.Logger
imagePath string
k8sImagePath string
configPath string
kubeVIPRoutingTableManifestTemplate *template.Template
tempDirPath string
v129 bool
)
BeforeEach(func() {
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
if configPath == "" {
configPath = "/etc/kubernetes/admin.conf"
}
_, v129 = os.LookupEnv("V129")
curDir, err := os.Getwd()
Expect(err).NotTo(HaveOccurred())
templateRoutingTablePath := filepath.Join(curDir, "kube-vip-routing-table.yaml.tmpl")
kubeVIPRoutingTableManifestTemplate, err = template.New("kube-vip-routing-table.yaml.tmpl").ParseFiles(templateRoutingTablePath)
Expect(err).NotTo(HaveOccurred())
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).NotTo(HaveOccurred())
})
Describe("kube-vip IPv4 control-plane routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv4Family,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups IPv4 address and route on control-plane node", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
checkIPAddress(cpVIP, container, true)
e2e.CheckRoutePresence(cpVIP, container, true)
}
})
})
Describe("kube-vip IPv6 control-plane routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv6Family,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups IPv6 address and route on control-plane node", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
checkIPAddress(cpVIP, container, true)
e2e.CheckRoutePresence(cpVIP, container, true)
}
})
})
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv4 primary", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv6, localIPv6Net, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv6,
ipnet: localIPv6Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups DualStack addresses and routes on control-plane nodes", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
addresses := vip.Split(cpVIP)
for _, addr := range addresses {
checkIPAddress(addr, container, true)
e2e.CheckRoutePresence(addr, container, true)
}
}
})
})
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv6 primary", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv4, localIPv4Net, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv4,
ipnet: localIPv4Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups DualStack addresses and routes on control-plane nodes", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
addresses := vip.Split(cpVIP)
for _, addr := range addresses {
checkIPAddress(addr, container, true)
e2e.CheckRoutePresence(addr, container, true)
}
}
})
})
Describe("kube-vip IPv4 services routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv4Family,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip IPv6 services routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv6Family,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services routing table mode functionality - IPv4 primary", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
EnableEndpointslices: "true",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 and IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services routing table mode functionality - IPv6 primary", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
EnableEndpointslices: "true",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 and IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
}
})
func testServiceRT(svcName, lbAddress, leaseName, leaseNamespace, clusterName string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
client kubernetes.Interface, serviceElection bool, ipFamily []corev1.IPFamily, numberOfServices int) {
lbAddresses := vip.Split(lbAddress)
services := []string{}
for i := range numberOfServices {
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
}
for _, svc := range services {
createTestService(svc, dsNamespace, dsName, lbAddress,
client, corev1.IPFamilyPolicyPreferDualStack, ipFamily, trafficPolicy)
}
var container string
if serviceElection {
container = e2e.GetLeaseHolder(leaseName, leaseNamespace, client)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
for _, addr := range lbAddresses {
e2e.CheckRoutePresence(addr, container, true)
}
for i := range numberOfServices {
expected := i < numberOfServices-1
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
for _, addr := range lbAddresses {
e2e.CheckRoutePresence(addr, container, expected)
}
}
}
type san struct {
ip *net.IP
ipnet *net.IPNet
}

View File

@@ -4,15 +4,41 @@
package e2e_test
import (
"log"
"os"
"sync"
"testing"
"github.com/kube-vip/kube-vip/testing/e2e"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
"github.com/kube-vip/kube-vip/testing/e2e"
const (
ModeEnv = "TEST_MODE"
ModeARP = "arp"
ModeRT = "rt"
ModeBGP = "bgp"
)
var (
SOffset *e2e.SecureOffset
ConfigMtx *sync.Mutex
Mode string
)
func TestE2E(t *testing.T) {
mode := os.Getenv(ModeEnv)
if mode == "" {
Mode = ModeARP
} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
log.Fatal("invalid", "mode", mode)
os.Exit(1)
} else {
Mode = mode
}
SOffset = e2e.NewOffset(5)
ConfigMtx = &sync.Mutex{}
RegisterFailHandler(Fail)
RunSpecs(t, "E2E Suite")
}

File diff suppressed because it is too large Load Diff

View File

@@ -268,15 +268,6 @@ func (c *Cluster) Delete() {
Expect(c.provider.Delete(c.Name, "")).To(Succeed())
}
func startKubeletForEtcd(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
)
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
)
}
func createCerts(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm",

View File

@@ -60,7 +60,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
test.vip = e2e.GenerateIPv4VIP()
test.vip = e2e.GenerateVIP(e2e.IPv4Family, 5)
test.logger.Printf("Selected VIP %s", test.vip)
test.currentDir, err = os.Getwd()

View File

@@ -6,22 +6,34 @@ package e2e
import (
"bufio"
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"net"
"os/exec"
"strings"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/vip"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
coordinationv1 "k8s.io/api/coordination/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/cluster"
)
const (
IPv4Family = "IPv4"
IPv6Family = "IPv6"
DualstackFamily = "DualStack"
DualstackFamilyIPv6 = "DualStackIPv6"
)
func EnsureKindNetwork() {
By("checking if the Docker \"kind\" network exists")
cmd := exec.Command("docker", "inspect", "kind")
@@ -58,53 +70,47 @@ func EnsureKindNetwork() {
Eventually(session).Should(gexec.Exit(0))
}
func GenerateIPv6VIP() string {
func GenerateVIP(family string, offset uint) string {
if family == DualstackFamily || family == DualstackFamilyIPv6 {
return fmt.Sprintf("%s,%s", GenerateVIP(IPv4Family, offset), GenerateVIP(IPv6Family, offset))
}
cidrs := getKindNetworkSubnetCIDRs()
for _, cidr := range cidrs {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if cidr != "" {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if ip.To4() == nil {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
if ip.To4() == nil && family == IPv6Family {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
return net.IP(chosenVIP).String()
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
return net.IP(chosenVIP).String()
} else if ip.To4() != nil && family == IPv4Family {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
return net.IP(chosenVIP).String()
}
}
}
Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
Fail("Could not find any " + family + " CIDRs in the Docker \"kind\" network")
return ""
}
func GenerateIPv4VIP() string {
cidrs := getKindNetworkSubnetCIDRs()
for _, cidr := range cidrs {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if ip.To4() != nil {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-5)
return net.IP(chosenVIP).String()
}
}
Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
return ""
}
func GenerateDualStackVIP() string {
return GenerateIPv4VIP() + "," + GenerateIPv6VIP()
func GenerateDualStackVIP(offset uint) string {
return GenerateVIP(IPv4Family, offset) + "," + GenerateVIP(IPv6Family, offset)
}
func getKindNetworkSubnetCIDRs() []string {
@@ -133,30 +139,88 @@ func getKindNetworkSubnetCIDRs() []string {
return cidrs
}
func CheckIPAddressPresence(ip string, container string) bool {
cmdOut := new(bytes.Buffer)
func CheckIPAddressPresence(ip string, container string, expected bool) bool {
family := "-4"
if vip.IsIPv6(ip) {
family = "-6"
}
cmd := exec.Command(
"docker", "exec", container, "ip", family, "addr", "show", "dev", "eth0",
)
cmdOut := new(bytes.Buffer)
cmdErr := new(bytes.Buffer)
cmd.Stdout = cmdOut
Eventually(cmd.Run(), "20s").Should(Succeed())
return strings.Contains(cmdOut.String(), ip)
cmd.Stderr = cmdErr
if err := cmd.Run(); err != nil {
return false
}
return strings.Contains(cmdOut.String(), ip) == expected
}
func CheckRoutePresence(ip string, container string) bool {
cmdOut := new(bytes.Buffer)
func CheckIPAddressPresenceByLease(name, namespace, ip string, client kubernetes.Interface, expected bool) bool {
container := GetLeaseHolder(name, namespace, client)
if container == "" {
return false
}
return CheckIPAddressPresence(ip, container, expected)
}
func CheckRoutePresence(ip string, container string, expected bool) bool {
family := "-4"
if vip.IsIPv6(ip) {
family = "-6"
}
cmd := exec.Command(
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
)
cmd.Stdout = cmdOut
Eventually(cmd.Run(), "20s").Should(Succeed())
return strings.Contains(cmdOut.String(), ip)
result := false
Eventually(func() bool {
cmdOut := new(bytes.Buffer)
cmdErr := new(bytes.Buffer)
cmd := exec.Command(
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
)
cmd.Stdout = cmdOut
cmd.Stderr = cmdErr
cmd.Run()
result = strings.Contains(cmdOut.String(), ip) == expected
return result
}, "60s", "1s").Should(BeTrue())
return result
}
func GetLeaseHolder(name, namespace string, client kubernetes.Interface) string {
var lease *coordinationv1.Lease
Eventually(func() error {
var err error
lease, err = client.CoordinationV1().Leases(namespace).Get(context.TODO(), name, metav1.GetOptions{})
return err
}, "300s").ShouldNot(HaveOccurred())
Expect(lease).ToNot(BeNil())
return *lease.Spec.HolderIdentity
}
type SecureOffset struct {
value int
mu sync.Mutex
}
func NewOffset(value int) *SecureOffset {
return &SecureOffset{
value: value,
}
}
func (so *SecureOffset) Get() uint {
so.mu.Lock()
defer so.mu.Unlock()
so.value++
return uint(so.value - 1)
}

View File

@@ -0,0 +1,82 @@
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
spec:
containers:
- name: kube-vip
args:
- manager
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: lb_class_legacy_handling
value: "false"
- name: lb_class_name
- name: prometheus_server
value: :2112
- name: disable_service_updates
value: "false"
- name: vip_arp
value: "false"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: vip_leaderelection
value: "false"
- name: vip_interface
value: lo
- name: bgp_enable
value: "true"
- name: bgp_hold_time
value: "3"
- name: bgp_keepalive_interval
value: "1"
- name: bgp_sourceif
value: "eth0"
- name: bgp_routerid
value: "2.2.2.2"
- name: bgp_as
value: "{{ .BGPAS }}"
- name: bgp_peers
value: "{{ .BGPPeers }}"
- name: mpbgp_nexthop
value: "{{ .MPBGPNexthop }}"
- name: mpbgp_ipv4
value: "{{ .MPBGPNexthopIPv4 }}"
- name: mpbgp_ipv6
value: "{{ .MPBGPNexthopIPv6 }}"
- name: address
value: "{{ .ControlPlaneVIP }}"
- name: cp_enable
value: "{{ .ControlPlaneEnable }}"
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "{{ .ConfigPath }}"
name: kubeconfig

View File

@@ -12,6 +12,8 @@ spec:
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: vip_arp
value: "false"
- name: vip_interface
@@ -27,7 +29,7 @@ spec:
- name: vip_retryperiod
value: "1"
- name: cp_enable
value: "true"
value: "{{ .ControlPlaneEnable }}"
- name: vip_nodename
valueFrom:
fieldRef:
@@ -36,10 +38,14 @@ spec:
value: '32,128'
- name: cp_namespace
value: kube-system
- name: svc_enable
value: 'false'
- name: vip_routingtable
value: 'true'
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:

View File

@@ -12,6 +12,8 @@ spec:
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: vip_arp
value: "true"
- name: vip_interface
@@ -32,6 +34,12 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:

View File

@@ -3,8 +3,47 @@
package e2e
import (
"fmt"
"net"
)
type KubevipManifestValues struct {
ControlPlaneVIP string
ImagePath string
ConfigPath string
ControlPlaneVIP string
ImagePath string
ConfigPath string
SvcEnable string
SvcElectionEnable string
EnableEndpointslices string
ControlPlaneEnable string
BGPAS uint32
BGPPeers string
MPBGPNexthop string
MPBGPNexthopIPv4 string
MPBGPNexthopIPv6 string
}
type BGPPeerValues struct {
IP string
AS uint32
MPBGP string
IPFamily string
}
func (pv *BGPPeerValues) String() string {
tmpIP := pv.IP
ip := net.ParseIP(tmpIP)
if ip == nil {
return ""
}
if ip.To4() == nil {
tmpIP = fmt.Sprintf("[%s]", tmpIP)
}
if pv.MPBGP != "" {
return fmt.Sprintf("%s:%d::false/mpbgp_nexthop=%s", tmpIP, pv.AS, pv.MPBGP)
}
return fmt.Sprintf("%s:%d::false", tmpIP, pv.AS)
}

View File

@@ -38,11 +38,14 @@ func main() {
flag.BoolVar(&t.LeaderActive, "leaderActive", false, "Perform a test on the active leader")
flag.BoolVar(&t.LocalDeploy, "localDeploy", false, "Perform a test on the active leader")
flag.BoolVar(&t.Egress, "egress", false, "Perform an egress test")
flag.BoolVar(&t.EgressInternal, "egressInternal", false, "Perform an egress test, using the internal functionality")
flag.BoolVar(&t.EgressIPv6, "egressIPv6", false, "Perform an egress test")
flag.BoolVar(&t.DualStack, "dualStack", false, "Perform an dual stack test")
flag.BoolVar(&t.RetainCluster, "retain", false, "Retain the cluster")
flag.StringVar(&t.KindVersionImage, "kindImage", "", "The image to use for the kind nodes e.g. (kindest/node:v1.21.14)")
flag.BoolVar(&existing, "existing", false, "Use an existing cluster")
flag.BoolVar(&t.IPv6, "ipv6", false, "Perform an IPv6-only test")
flag.StringVar(&t.DockerNIC, "networkInterface", "br-", "Selects networking interface to use")
flag.BoolVar(&t.Cilium, "cilium", false, "Use cilium as a CNI")
@@ -50,9 +53,6 @@ func main() {
slog.Infof("🔬 beginning e2e tests, image: [%s] DualStack [%t]", t.ImagePath, t.DualStack)
// if !t.ignoreDualStack {
// t.Dualstack = true
// }
t.Affinity = os.Getenv("NODE_TOLERATE")
t.DeploymentName = "kube-vip-deploy"
@@ -134,7 +134,7 @@ func main() {
}
}
t.StartServiceTest(ctx, clientset)
errs := t.StartServiceTest(ctx, clientset)
os.Exit(len(errs))
}
}

View File

@@ -3,11 +3,13 @@ package deployment
import (
"bytes"
"fmt"
"net"
"os/exec"
"strings"
"time"
"github.com/gookit/slog"
"github.com/vishvananda/netlink"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/cluster"
"sigs.k8s.io/kind/pkg/cmd"
@@ -16,6 +18,11 @@ import (
var provider *cluster.Provider
const (
defaultIPv4Pool = "172.18.100.10-172.18.100.30"
defaultIPv6Pool = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
)
type kubevipManifestValues struct {
ControlPlaneVIP string
ImagePath string
@@ -39,13 +46,12 @@ func (config *TestConfig) CreateKind() error {
},
}
if config.IPv6 {
// Change Networking Family
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
}
if config.DualStack {
// Change Networking Family
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
} else {
if config.IPv6 || !config.Egress && config.EgressIPv6 {
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
}
}
if config.Cilium {
@@ -140,11 +146,52 @@ func (config *TestConfig) CreateKind() error {
}
}
globalRange := "172.18.100.10-172.18.100.30"
if config.IPv6 {
globalRange = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
ipv4Pool := defaultIPv4Pool
ipv6Pool := defaultIPv6Pool
var ipv4Net *net.IPNet
var ipv6Net *net.IPNet
links, err := netlink.LinkList()
if err != nil {
return fmt.Errorf("netlink: failed to list links: %w", err)
}
for _, link := range links {
if strings.Contains(link.Attrs().Name, config.DockerNIC) {
_, ipv4Net, err = getNetwork(link, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to get IPv4 network: %w", err)
}
if ipv4Net == nil {
return fmt.Errorf("failed to find IPv4 network on the docker interface")
}
ipv4Pool = generateRange(ipv4Net)
_, ipv6Net, err = getNetwork(link, netlink.FAMILY_V6)
if err != nil {
return fmt.Errorf("failed to get IPv6 network: %w", err)
}
if ipv6Net == nil {
return fmt.Errorf("failed to find IPv6 network on the docker interface")
}
ipv6Pool = generateRange(ipv6Net)
}
}
globalRange := ipv4Pool
if config.IPv6 {
globalRange = ipv6Pool
}
if config.DualStack {
globalRange = fmt.Sprintf("%s,%s", ipv4Pool, ipv6Pool)
}
slog.Infof("Cloud provider's range: %s", globalRange)
cmd := exec.Command("kubectl", "create", "configmap", "--namespace", "kube-system", "kubevip", "--from-literal", "range-global="+globalRange)
if _, err := cmd.CombinedOutput(); err != nil {
return err
@@ -163,6 +210,43 @@ func (config *TestConfig) CreateKind() error {
return nil
}
func getNetwork(link netlink.Link, family int) (*net.IP, *net.IPNet, error) {
addrs, err := netlink.AddrList(link, family)
if err != nil {
return nil, nil, fmt.Errorf("netlink: failed to get addresses for link %q: %w", link.Attrs().Name, err)
}
if len(addrs) > 0 {
for _, a := range addrs {
if a.Scope == int(netlink.SCOPE_UNIVERSE) {
ip, cidr, err := net.ParseCIDR(a.IPNet.String())
if err != nil {
return nil, nil, fmt.Errorf("failed to parse CIDR: %w", err)
}
return &ip, cidr, nil
}
}
}
return nil, nil, nil
}
func generateRange(network *net.IPNet) string {
if network.IP.To4() != nil {
parts := strings.Split(network.IP.String(), ".")
p1 := fmt.Sprintf("%s.%s.100.10", parts[0], parts[1])
p2 := fmt.Sprintf("%s.%s.100.100", parts[0], parts[1])
return fmt.Sprintf("%s-%s", p1, p2)
}
if network.IP.To16() != nil {
parts := strings.Split(network.IP.String(), ":")
p1 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0010", parts[0], parts[1], parts[2], parts[3])
p2 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0100", parts[0], parts[1], parts[2], parts[3])
return fmt.Sprintf("%s-%s", p1, p2)
}
return ""
}
func DeleteKind() error {
slog.Info("🧽 deleting Kind cluster")
return provider.Delete("services", "")

View File

@@ -18,13 +18,14 @@ import (
// service defines the settings for a new service
type Service struct {
name string
egress bool // enable egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
name string
egress bool // enable egress
egressInternal bool // enable Internal egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
}
type Deployment struct {
@@ -155,7 +156,7 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
Containers: []v1.Container{
{
Name: "kube-vip-web",
Image: "plndr/e2e:0.0.1",
Image: "docker.io/plndr/e2e:0.0.1",
Ports: []v1.ContainerPort{
{
Name: "http",
@@ -238,6 +239,11 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
"kube-vip.io/egress": "true",
}
}
if s.egressInternal {
svc.Annotations["kube-vip.io/egress-internal"] = "true"
}
if s.egressIPv6 {
svc.Annotations["kube-vip.io/egress-ipv6"] = "true"
}
@@ -276,7 +282,7 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
slog.Fatal(err)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
@@ -297,7 +303,6 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// slog.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
slog.Fatalf("unable to parse Kubernetes services from API watcher")

View File

@@ -7,9 +7,11 @@ import (
"fmt"
"net"
"net/http"
"strings"
"time"
"github.com/gookit/slog"
"github.com/vishvananda/netlink"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
@@ -24,12 +26,14 @@ import (
// 1. Create a deployment
// 2. Expose the deployment
func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kubernetes.Clientset) {
func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kubernetes.Clientset) []error {
var err error
var errs []error
if config.Simple {
err = config.SimpleDeployment(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
@@ -37,6 +41,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
err = config.MultipleDeployments(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
@@ -45,6 +50,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
err = config.Failover(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
@@ -53,41 +59,72 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
err = config.Failover(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.LocalDeploy {
// Failover tests
err = config.LocalDeployment(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.Egress {
// Failover tests
err = config.EgressDeployment(ctx, clientset)
// Egress test
err = config.EgressDeployment(ctx, clientset, false)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.Egress && config.EgressInternal {
// Egress test
err = config.EgressDeployment(ctx, clientset, true)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.EgressIPv6 {
// Failover tests
err = config.Egressv6Deployment(ctx, clientset)
// Egress v6 tests
err = config.Egressv6Deployment(ctx, clientset, false)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.EgressIPv6 && config.EgressInternal {
// Egress v6 tests
err = config.Egressv6Deployment(ctx, clientset, true)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.DualStack {
// Failover tests
// Dualstack tests
err = config.DualStackDeployment(ctx, clientset)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
slog.Infof("🏆 Testing Complete [%d] passed", config.SuccessCounter)
const testComplete = "🏆 Testing Complete [%d] passed / [%d] failed"
if len(errs) > 0 {
slog.Errorf(testComplete, config.SuccessCounter, len(errs))
return errs
}
slog.Infof(testComplete, config.SuccessCounter, len(errs))
return errs
}
func httpTest(address string) error {
@@ -147,7 +184,7 @@ func leaderFailover(ctx context.Context, name, leaderNode *string, clientset *ku
slog.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
@@ -230,7 +267,7 @@ func podFailover(ctx context.Context, name, leaderNode *string, clientset *kuber
slog.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
@@ -282,43 +319,62 @@ func podFailover(ctx context.Context, name, leaderNode *string, clientset *kuber
return nil
}
func tcpServer(IPv6 bool, egressAddress *string, timeout int) bool {
func tcpServer(egressAddress *string, timeout int, network string) bool {
var listen net.Listener
var err error
if !IPv6 {
listen, err = net.Listen("tcp", ":12345") //nolint
} else {
listen, err = net.Listen("tcp6", ":12346") //nolint
port := ":12345"
if network == "tcp6" {
port = ":12346"
}
listen, err = net.Listen(network, port) //nolint
if err != nil {
slog.Error(err)
}
// close listener
go func() {
time.Sleep(time.Second * time.Duration(timeout))
listen.Close()
}()
for {
conn, err := listen.Accept()
if err != nil {
return false
// slog.Fatal(err)
}
remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
// If it is IPv6 expand the address
// if IPv6 {
// addr, _ := netip.ParseAddr(remoteAddress)
// remoteAddress = addr.StringExpanded()
// }
if remoteAddress == *egressAddress {
slog.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
return true
srvChan := make(chan any)
finishChan := make(chan any)
go func() {
defer func() {
err = listen.Close()
if err != nil {
slog.Error(err)
}
close(finishChan)
}()
select {
case <-srvChan:
return
case <-time.After(time.Second * time.Duration(timeout)):
return
}
slog.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
go handleRequest(conn)
}
}()
result := false
go func() {
defer close(srvChan)
for {
conn, err := listen.Accept()
if err != nil {
result = false
return
}
remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
if remoteAddress == *egressAddress {
slog.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
result = true
return
}
slog.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
go handleRequest(conn)
}
}()
<-finishChan
return result
}
func handleRequest(conn net.Conn) {
@@ -339,34 +395,32 @@ func handleRequest(conn net.Conn) {
conn.Close()
}
func GetLocalIPv4() string {
addrs, err := net.InterfaceAddrs()
func GetLocalIP(ifName string, family int) (*net.IP, *net.IPNet, error) {
links, err := netlink.LinkList()
if err != nil {
return ""
return nil, nil, fmt.Errorf("netlink: failed to list links: %w", err)
}
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
return ipnet.IP.String()
for _, link := range links {
if strings.Contains(link.Attrs().Name, ifName) {
ip, ipnet, err := getNetwork(link, family)
if err != nil {
return nil, nil, fmt.Errorf("failed to get IPv4 address: %w", err)
}
if ip == nil {
return nil, nil, fmt.Errorf("failed to find IPv4 address on the interface %q", ifName)
}
return ip, ipnet, nil
}
}
return ""
return nil, nil, nil
}
func GetLocalIPv6() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
return ""
}
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() == nil && !ipnet.IP.IsLinkLocalUnicast() {
if ipnet.IP.To16() != nil {
return ipnet.IP.String()
}
}
}
return ""
func GetLocalIPv4(ifName string) (*net.IP, *net.IPNet, error) {
return GetLocalIP(ifName, netlink.FAMILY_V4)
}
func GetLocalIPv6(ifName string) (*net.IP, *net.IPNet, error) {
return GetLocalIP(ifName, netlink.FAMILY_V6)
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net"
"time"
"github.com/gookit/slog"
v1 "k8s.io/api/core/v1"
@@ -33,6 +32,7 @@ type TestConfig struct {
LocalDeploy bool
DualStack bool
Egress bool
EgressInternal bool
EgressIPv6 bool
RetainCluster bool
SkipHostnameChange bool
@@ -45,6 +45,9 @@ type TestConfig struct {
// Cilium config
Cilium bool
// Docker config
DockerNIC string
}
func (config *TestConfig) SimpleDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
@@ -305,8 +308,8 @@ func (config *TestConfig) LocalDeployment(ctx context.Context, clientset *kubern
return nil
}
func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
// pod Failover tests
func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
// egress test
var err error
defer func() error {
@@ -322,14 +325,10 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
return nil
}() //nolint
slog.Infof("🧪 ---> egress IP re-write (local policy) <---")
slog.Infof("🧪 ---> egress IP re-write (local policy, internal: %t) <---", internal)
var egress string
var found bool
timeout := 30
// Set up a local listener
go func() {
found = tcpServer(false, &egress, timeout)
}()
deploy := Deployment{
name: config.DeploymentName,
@@ -339,7 +338,11 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
}
// Find this machines IP address
deploy.address = GetLocalIPv4()
addr, _, err := GetLocalIPv4(config.DockerNIC)
if err != nil {
return fmt.Errorf("unable to detect local IP address: %w", err)
}
deploy.address = addr.String()
if deploy.address == "" {
return fmt.Errorf("unable to detect local IP address")
}
@@ -358,30 +361,34 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
timeout: 30,
}
if internal {
svc.egressInternal = true
}
_, lbAddresses, err := svc.CreateService(ctx, clientset)
if err != nil {
return err
}
egress = lbAddresses[0]
for i := 1; i < 15; i++ {
if found {
slog.Infof("🕵️ egress has correct IP address")
config.SuccessCounter++
break
}
time.Sleep(time.Second * 1)
if len(lbAddresses) < 1 {
return fmt.Errorf("no loadbalancer address found")
}
if !found {
egress = lbAddresses[0]
found = tcpServer(&egress, timeout, "tcp4")
if found {
slog.Infof("🕵️ egress has correct IP address")
config.SuccessCounter++
} else {
return fmt.Errorf("😱 No traffic found from loadbalancer address ")
}
return err
return nil
}
func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset) error {
// pod Failover tests
func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
// egress v6 test
var err error
defer func() error {
@@ -397,14 +404,10 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
return nil
}() //nolint
slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy) <---")
slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy, internal: %t) <---", internal)
var egress string
var found bool
timeout := 60
// Set up a local listener
go func() {
found = tcpServer(true, &egress, timeout)
}()
timeout := 30
deploy := Deployment{
name: config.DeploymentName,
@@ -414,9 +417,13 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
}
// Find this machines IP address
deploy.address = GetLocalIPv6()
addr, _, err := GetLocalIPv6(config.DockerNIC)
if err != nil {
return fmt.Errorf("unable to detect local IP address: %w", err)
}
deploy.address = addr.String()
if deploy.address == "" {
slog.Fatalf("Unable to detect local IP address")
return fmt.Errorf("unable to detect local IP address")
}
slog.Infof("📠 found local address [%s]", deploy.address)
// Create a deployment that connects back to this machines IP address
@@ -430,20 +437,23 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
name: config.ServiceName,
egress: true,
egressIPv6: true,
testHTTP: false,
timeout: timeout,
testDualstack: true,
}
if internal {
svc.egressInternal = true
}
_, lbAddresses, err := svc.CreateService(ctx, clientset)
if err != nil {
return err
}
for x := range lbAddresses {
ip := net.ParseIP(lbAddresses[x])
// if ip == nil {
// return errors.New("invalid address")
// }
if ip == nil {
return fmt.Errorf("invalid address")
}
if ip.To4() == nil {
// use brackets for IPv6 address
egress = lbAddresses[x]
@@ -451,20 +461,20 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
}
}
for i := 1; i < timeout; i++ {
if found {
slog.Infof("🕵️ egress has correct IP address")
config.SuccessCounter++
break
}
time.Sleep(time.Second * 1)
if egress == "" {
return fmt.Errorf("no loadbalancer address found")
}
if !found {
slog.Error("😱 No traffic found from loadbalancer address ")
found = tcpServer(&egress, timeout, "tcp6")
if found {
slog.Infof("🕵️ egress has correct IP address")
config.SuccessCounter++
} else {
return fmt.Errorf("😱 No traffic found from loadbalancer address ")
}
return err
return nil
}
func (config *TestConfig) DualStackDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {