Compare commits

...

99 Commits

Author SHA1 Message Date
Marcel Fest
c925fb7cd9 fix: dhcp panic and leak potential
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-09-18 14:20:22 +02:00
Marcel Fest
d6129c8299 fix(vip): synchronize shared datapath state
Serialize the interface link cache, route tracker and address configuration so
concurrent Services cannot corrupt shared state or deadlock on nested address
locks.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-09-18 14:20:22 +02:00
Marcel Fest
3d73039cce feat(lease): record VIP ownership in lease annotations (#1767)
* feat(lease): record VIP ownership in lease annotations

Encode the instance name, address protocol and an ordered VIP list on the
Kubernetes lease so a restarting instance can tell which tagged addresses it
still owns. Only the current holder publishes the annotation.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
Signed-off-by: Cellebyte <marcel.fest@live.de>
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix: flip flop on ordering when reading and writing the annotation

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix: address feedback from review

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

---------

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
Signed-off-by: Cellebyte <marcel.fest@live.de>
2026-09-18 14:20:19 +02:00
Patryk Strusiewicz-Surmacki
a60f80b1b2 Refactored and fixed metrics code
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2026-09-16 18:01:07 +02:00
Marcel Fest
d99186480c chore: bump kube-vip to v1.2.4
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-09-16 12:13:24 +02:00
dependabot[bot]
d557211b55 build(deps): bump golang.org/x/sys from 0.47.0 to 0.48.0
Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.47.0 to 0.48.0.
- [Commits](https://github.com/golang/sys/compare/v0.47.0...v0.48.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sys
  dependency-version: 0.48.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-14 13:38:41 +02:00
dependabot[bot]
e4eb5e8e7f build(deps): bump github.com/onsi/ginkgo/v2 from 2.32.1 to 2.32.2
Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.32.1 to 2.32.2.
- [Release notes](https://github.com/onsi/ginkgo/releases)
- [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md)
- [Commits](https://github.com/onsi/ginkgo/compare/v2.32.1...v2.32.2)

---
updated-dependencies:
- dependency-name: github.com/onsi/ginkgo/v2
  dependency-version: 2.32.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-14 13:09:08 +02:00
dependabot[bot]
a7b068f9d2 build(deps): bump golang.org/x/sync from 0.22.0 to 0.23.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.22.0 to 0.23.0.
- [Commits](https://github.com/golang/sync/compare/v0.22.0...v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-version: 0.23.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-14 13:08:26 +02:00
dependabot[bot]
a6bf5280d4 build(deps): bump anchore/sbom-action from 0.24.0 to 0.24.2
Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.24.0 to 0.24.2.
- [Release notes](https://github.com/anchore/sbom-action/releases)
- [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md)
- [Commits](https://github.com/anchore/sbom-action/compare/v0.24.0...v0.24.2)

---
updated-dependencies:
- dependency-name: anchore/sbom-action
  dependency-version: 0.24.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-13 22:24:58 +02:00
Cellebyte
61a52e8f2a fix: function calls
Signed-off-by: Cellebyte <marcel.fest@live.de>
2026-09-13 22:02:01 +02:00
Cellebyte
7e2dd0d262 fix: add manager interface at the right place
Signed-off-by: Cellebyte <marcel.fest@live.de>
2026-09-13 22:02:01 +02:00
Cellebyte
7a085cb3ce fix(bgp-cp): mem-leak on stacking defers and never exit
Signed-off-by: Cellebyte <marcel.fest@live.de>
2026-09-13 22:02:01 +02:00
Marcel Fest
1f5c135fd1 use ipv6 in error message 2026-09-13 22:01:14 +02:00
Patryk Strusiewicz-Surmacki
b4771c5319 Fixed DHCP concurrency exit issues
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2026-09-13 22:01:14 +02:00
Marcel Fest
3771ccee29 Add suggestion for IPv4/IPv6 swap 2026-09-13 21:47:08 +02:00
dependabot[bot]
ee64dceb36 build(deps): bump github.com/onsi/gomega from 1.42.1 to 1.43.0
Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.42.1 to 1.43.0.
- [Release notes](https://github.com/onsi/gomega/releases)
- [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md)
- [Commits](https://github.com/onsi/gomega/compare/v1.42.1...v1.43.0)

---
updated-dependencies:
- dependency-name: github.com/onsi/gomega
  dependency-version: 1.43.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 22:34:59 +02:00
dependabot[bot]
38fabeba9e build(deps): bump golang from 1.27.0-alpine3.23 to 1.27.1-alpine3.23
Bumps golang from 1.27.0-alpine3.23 to 1.27.1-alpine3.23.

---
updated-dependencies:
- dependency-name: golang
  dependency-version: 1.27.1-alpine3.23
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 22:10:09 +02:00
dependabot[bot]
7ef1899567 build(deps): bump github.com/google/go-containerregistry
Bumps [github.com/google/go-containerregistry](https://github.com/google/go-containerregistry) from 0.21.9 to 0.22.1.
- [Release notes](https://github.com/google/go-containerregistry/releases)
- [Commits](https://github.com/google/go-containerregistry/compare/v0.21.9...v0.22.1)

---
updated-dependencies:
- dependency-name: github.com/google/go-containerregistry
  dependency-version: 0.22.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 22:09:14 +02:00
dependabot[bot]
637c3da47e build(deps): bump sigs.k8s.io/kind from 0.32.0 to 0.33.0
Bumps [sigs.k8s.io/kind](https://github.com/kubernetes-sigs/kind) from 0.32.0 to 0.33.0.
- [Release notes](https://github.com/kubernetes-sigs/kind/releases)
- [Changelog](https://github.com/kubernetes-sigs/kind/blob/main/RELEASE.md)
- [Commits](https://github.com/kubernetes-sigs/kind/compare/v0.32.0...v0.33.0)

---
updated-dependencies:
- dependency-name: sigs.k8s.io/kind
  dependency-version: 0.33.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 22:05:48 +02:00
Maximilian Rink
e666a0cdd1 fix(e2e): harden nightly etcd readiness
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-09 18:07:48 +02:00
Maximilian Rink
9a2142c028 ci: bound nightly etcd stabilization
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-09 18:07:48 +02:00
Maximilian Rink
0606e9477c ci: surface etcd suite failures and harden nightly artifacts
Review follow-ups: step-level continue-on-error with a run-summary warning
instead of a silently-neutral job, ginkgo JSON report so the log artifact
is never empty, always-upload coverage with if-no-files-found=error,
read-only workflow permissions, GOMAXPROCS=4 for target consistency.

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-09 18:07:48 +02:00
Maximilian Rink
8a968618bf ci: add coverage output and nightly workflow with etcd e2e suite
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-09 18:07:48 +02:00
dependabot[bot]
5cd138a85b build(deps): bump github.com/osrg/gobgp/v4 from 4.8.0 to 4.9.0
Bumps [github.com/osrg/gobgp/v4](https://github.com/osrg/gobgp) from 4.8.0 to 4.9.0.
- [Release notes](https://github.com/osrg/gobgp/releases)
- [Commits](https://github.com/osrg/gobgp/compare/v4.8.0...v4.9.0)

---
updated-dependencies:
- dependency-name: github.com/osrg/gobgp/v4
  dependency-version: 4.9.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 17:59:06 +02:00
dependabot[bot]
246a786fe2 build(deps): bump google.golang.org/grpc from 1.83.1 to 1.83.2
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.83.1 to 1.83.2.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.83.1...v1.83.2)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-version: 1.83.2
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-09 17:57:42 +02:00
Maximilian Rink
6ee3024bc6 fix(e2e): guard etcd suite teardown after setup failure
Register the cluster handle before setup assertions so partial Kind creation can be cleaned up, while keeping teardown safe when setup fails before registration.

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-09 17:23:54 +02:00
dependabot[bot]
2bc2df53fc build(deps): bump github.com/stretchr/testify from 1.11.1 to 1.12.1
Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.11.1 to 1.12.1.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.11.1...v1.12.1)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-version: 1.12.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-04 14:16:39 +02:00
Marcel Fest
618904dea3 fix(ci): deflake debouncer cancellation test and patch CVE-2026-56854
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-09-04 13:53:04 +02:00
Maximilian Rink
b116d5a469 chore(iptables): remove unused ListByID, Stats, StructuredStats, ParseStat (#1728)
* test(endpoints): follow WireGuard clear refactor

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* chore(iptables): remove unused ListByID, Stats, StructuredStats, ParseStat

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

---------

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-09-04 09:10:19 +02:00
Daniel Finneran
6cbf5aaeda Merge pull request #1723 from MaxRink/fix/fix-dhcp-robustness
fix(vip): harden DHCP client lifecycle and reply parsing
2026-09-03 15:22:21 +01:00
Daniel Finneran
dc453f07fc Merge pull request #1720 from MaxRink/fix/fix-dad-skip
fix(vip): apply per-call DAD skip instead of leaking it into persistent state
2026-09-03 15:21:04 +01:00
Daniel Finneran
14b2f51aba Merge pull request #1718 from MaxRink/fix/fix-svcctx-races
fix(servicecontext): synchronize access to shared context fields
2026-09-03 15:20:14 +01:00
Daniel Finneran
01e8fbc3e3 Merge pull request #1716 from MaxRink/fix/fix-debouncer-deadlock
fix(debouncer): prevent deadlock when forwarding to a terminating object
2026-09-03 15:18:39 +01:00
Daniel Finneran
4504649c91 Merge pull request #1710 from MaxRink/fix/fix-cluster-stop
fix(cluster): make Cluster.Stop concurrency-safe and lock Processor.Stop
2026-09-03 15:17:13 +01:00
Daniel Finneran
47f4e75183 Merge pull request #1709 from MaxRink/fix/fix-route-tracker
fix(route): insert route tracker entry only after AddRoute succeeds
2026-09-03 15:16:16 +01:00
Daniel Finneran
f589a18bd9 Merge pull request #1713 from MaxRink/fix/fix-mpbgp-family
fix(bgp): validate MP-BGP fixed-address family
2026-09-02 17:28:58 +01:00
Daniel Finneran
d79f3ddb52 Merge pull request #1714 from MaxRink/fix/fix-endpoints-locality
fix(endpoints): only use Hostname for locality when NodeName is absent
2026-09-02 17:27:19 +01:00
Daniel Finneran
1b25ce7d0e Merge pull request #1712 from MaxRink/fix/fix-annotations-nodelist
fix(manager): handle empty node list in annotations watcher
2026-09-02 17:26:22 +01:00
Daniel Finneran
96c4406d63 Merge pull request #1711 from MaxRink/fix/fix-bgp-peers
fix(bgp): honour configured peer port and keep source interface on MP-BGP fallback
2026-09-02 17:25:43 +01:00
Daniel Finneran
a51944b89d Merge pull request #1707 from MaxRink/fix/ipv6-cp-vip-e2e-timeout
test(e2e): raise IPv6 control-plane VIP reachability timeout for parallel runs
2026-09-02 17:24:42 +01:00
Daniel Finneran
a36dc36947 Merge pull request #1739 from MaxRink/fix/wireguard-clear-test
test(endpoints): follow WireGuard clear refactor
2026-09-01 18:21:42 +03:00
Daniel Finneran
47b546073a Merge pull request #1703 from MaxRink/fix/makefile-go-version
fix: align GO_VERSION with go.mod toolchain requirement
2026-09-01 10:56:39 +03:00
Justin Cichra
2b126dceed Fix bgp_attach_ip_to_interface not applied to Service VIPs (#1744)
shouldAddServiceIP() checks BGPAttachIPToInterface on a per-service
config struct. NewInstance() builds that struct fresh for each Service,
copying over most fields from the global config, missing this one. As
a result the flag has no effect regardless of its value, and BGP-mode
Service VIPs are never bound to the interface.

Copy the field at both construction sites where it's built, and add
a regression test covering the propagation.

Signed-off-by: Justin Cichra <jrcichra@yahoo.com>
2026-08-31 00:20:08 +02:00
Maximilian Rink
110d34b844 ci: retrigger flaky e2e leg
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:18 +02:00
Maximilian Rink
5388fed92b fix: align GO_VERSION with go.mod toolchain requirement
go.mod requires go >= 1.26.4 but unit-tests-docker ran golang:1.25.6
with GOTOOLCHAIN=local and failed before running any test. Bump to
1.26.6 and stop leaving a root-owned coverage.out behind on Linux
hosts while preserving the test exit status.

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:17 +02:00
Maximilian Rink
487859e76f test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:17 +02:00
Maximilian Rink
0e57deef3d test(e2e): raise IPv6 control-plane VIP reachability timeout for parallel runs
The IPv6 control-plane VIP reachability check used a 5s Eventually
window; under the 4-process parallel e2e run on a 4-vCPU runner NDP
convergence for the IPv6 VIP can exceed that, causing intermittent
timeouts while sibling specs pass. Raise it to a load-tolerant window
matching the IPv4 case; the reachability assertion is unchanged.

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:14 +02:00
Maximilian Rink
b25badd185 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:14 +02:00
Maximilian Rink
77fa726c99 test(bgp): cover peer transport options generically
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:12 +02:00
Maximilian Rink
920a0182cb fix(bgp): honour configured peer port and keep source interface on MP-BGP fallback
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:11 +02:00
Maximilian Rink
8d043de910 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:11 +02:00
Maximilian Rink
b0b12cfc13 test(vip): fix errcheck lint in DHCPv4 backoff test
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:01 +02:00
Maximilian Rink
b034c81bef fix(vip): harden DHCP client lifecycle and reply parsing
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:00 +02:00
Maximilian Rink
ca9640227a test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:21:00 +02:00
Maximilian Rink
5276f0123f ci: retrigger flaky e2e leg
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:20:58 +02:00
Maximilian Rink
c2e5be6d6a test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-30 23:20:58 +02:00
Maximilian Rink
a8293f66e4 fix(route): insert route tracker entry only after AddRoute succeeds
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 03:03:04 +02:00
Maximilian Rink
9d72f43f62 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 03:03:02 +02:00
Maximilian Rink
3c3096d89d fix(bgp): validate MP-BGP fixed-address family
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 03:01:29 +02:00
Maximilian Rink
5d5c893501 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 03:01:28 +02:00
Maximilian Rink
d417a0c8e8 fix(manager): handle empty node list in annotations watcher
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:08:11 +02:00
Maximilian Rink
825ffdb20c test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:08:09 +02:00
Maximilian Rink
fe3a379f2d fix(debouncer): prevent deadlock when forwarding to a terminating object
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:06:38 +02:00
Maximilian Rink
c42a207c26 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:06:36 +02:00
Maximilian Rink
d6e5753464 fix(vip): apply per-call DAD skip instead of leaking it into persistent state
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:05:25 +02:00
Maximilian Rink
5131b92810 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:05:23 +02:00
Maximilian Rink
e1fd9ac3e3 fix(cluster): make Cluster.Stop concurrency-safe and lock Processor.Stop
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:02:16 +02:00
Maximilian Rink
1831a05525 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:02:14 +02:00
Maximilian Rink
19198d47fa test(endpoints): cover provider locality parity
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:00:53 +02:00
Maximilian Rink
1af388ff9f fix(endpoints): only use Hostname for locality when NodeName is absent
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:00:52 +02:00
Maximilian Rink
d1ff5f2952 test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-28 00:00:50 +02:00
Maximilian Rink
54881a117c fix(endpoints): preserve nil context guard
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-27 23:59:48 +02:00
Maximilian Rink
426a409a5d fix(servicecontext): synchronize access to shared context fields
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-27 23:59:46 +02:00
Maximilian Rink
5871bec56c test(endpoints): follow WireGuard clear refactor
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-27 23:59:45 +02:00
Daniel Finneran
b7f3379514 Merge pull request #1686 from Dessalines39394/fix/star-history-chart
docs: fix broken star history chart in README
2026-08-25 19:01:39 +03:00
Marcel Fest
a15745c442 fix: pr introduced order regression 2026-08-25 13:15:21 +02:00
Marcel Fest
b684eed5a4 fix(egress): prevent stale active-endpoint overwrite (#1701)
Make the endpoint watcher the sole active-endpoint annotation writer so a Service snapshot captured during startup cannot replace a newer endpoint. Preserve intentionally empty snapshots across endpoint-to-zero-to-same transitions, select the cleared annotation from the configured egress family, reject malformed or wrong-family endpoints, and preserve the other family in dual-stack state.

Fixes #1673.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-24 18:55:11 +02:00
Marcel Fest
0bdd6a9015 refactor/endpoints for different modes (#1699)
* fix(endpoints): recompute and withdraw only when no longer referenced

Track EndpointSlice/Endpoints deletions in provider state instead of stopping
the watcher or clearing unconditionally. Recompute the aggregate endpoint set
after each deletion and only withdraw local routing/BGP/DNAT state once no
tracked object still references it, matching the existing behaviour for a
Modified event that reduces endpoints to zero.

Remove the now-unreachable per-mode delete()/deleteAction() hooks, which acted
on stale pre-deletion state and could withdraw prematurely while another
object still referenced the same address. Fix the WireGuard control-plane
endpoint watcher to drop deleted objects from its cache instead of reloading
them as still current.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* feat(worker): support global leader election for BGP and WireGuard

Wire the existing global leader-election machinery into BGP.StartServices and
WireGuard.StartServices so both modes can run with a single node advertising
all services, matching the option already available for ARP and RoutingTable.

WireGuard has no multipath mechanism, so it must always run some form of
leader election (per-service or global); remove its no-election path and the
redundant services watcher previously started from OnStartedLeading, which
would have run a second time alongside the global election path.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(services): drop unnecessary election guard on route teardown

route.Manager.Delete is reference-counted per route and safe to call even
when nothing is tracked for it, exactly like the existing BGP host teardown.
Remove the leader-election guard on the RoutingTable branch of deleteService
so both branches read identically and route cleanup runs consistently across
all leader-election configurations.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(endpoints): treat an unknown ready condition as ready

Deferring a nil Serving to Ready matched the Serving godoc but stopped
half way: per the EndpointConditions.Ready godoc a nil Ready is an unknown
state that consumers should interpret as ready, which is what kube-proxy
does. An endpoint with both conditions unset, common in manually managed
EndpointSlices for external workloads, was therefore skipped and never
selected as a local endpoint.

Only skip the endpoint when the resolved condition is explicitly false,
and cover the condition permutations in the provider tests.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(endpoints): reconcile fully on endpoint object deletion

Delete only mirrored part of AddOrModify's zero-endpoint handling, so a
deleted EndpointSlice left the datapath inconsistent in three ways.

In ARP global-leader mode the VIP clusters were left running with no
backends, because the cluster stop that AddOrModify performs was missing.
When endpoints remained, lastKnownGoodEndpoint was never revisited, so a
last known endpoint that lived in the deleted slice stayed stale: egress
rules were neither torn down nor moved and the active-endpoint annotations
were not refreshed. The EndpointSlice controller can delete a whole slice
during consolidation without touching the survivor, so no follow-up event
is guaranteed to repair this. In WireGuard mode processInstance never ran,
leaving DNAT rules pointing at removed addresses.

Extract the shared decisions into shouldProcessInstance and
handleNoEndpoints and use them from both paths, and run the same
lastKnownGoodEndpoint and annotation reconciliation after a deletion.
handleNoEndpoints also guards against a missing instance, which the
deletion path can legitimately hit. The getEndpoints error is now wrapped
with the provider label like every other error here.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* docs(bgp): call out the services election behaviour change

Enabling leader election in BGP mode now runs a global services leader.
Nothing in the configuration changed for existing deployments, but a user
who enabled it for the control plane previously got the no-leader path,
where every node advertised the service VIP over ECMP, and now gets a
single advertising node.

Record this in the changelog and warn once on startup when this path is
taken, pointing at the settings that restore the previous datapath.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* refactor(endpoints): deduplicate the control plane endpoint watcher

The watch.Deleted case duplicated the Added/Modified body except for the
provider mutation, which meant maintaining the log message and the DNAT
update in two places. Pick the mutation from the event type and share the
rest.

Also state the single-object invariant on the Endpoints provider's
DeleteObject, so its asymmetry with the slice provider is not mistaken for
an oversight.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(endpoints): only ever consider active endpoints

GetLocalEndpoints skipped endpoints whose conditions said they were not
serving, but GetAllEndpoints applied no filtering at all. A service using
externalTrafficPolicy: Cluster therefore advertised backends that had
already been marked as not ready, and the two policies disagreed about
which endpoints back the same service.

Extract the condition evaluation into isServing and apply it on both
paths, and assert in the tests that Cluster and Local agree.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* refactor(endpoints): reconcile every endpoint event through one path

Now that a deletion recomputes the remaining endpoints, AddOrModify and
Delete differed only in how the event mutated the provider. Keeping two
near-identical bodies is what let their behaviour drift apart in the first
place, so fold them into a single Reconcile that dispatches the mutation
in applyEvent.

Deletions consequently pick up the parts of the add path they were still
missing: the IPv6 egress guard, the leader election start, and the
endpointless opt-in branch, which now keeps an opted-in Cluster service up
after its last EndpointSlice disappears instead of tearing it down.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

---------

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-24 18:53:48 +02:00
Maximilian Rink
b864abf27d fix(worker): honor context cancellation in cleanRoutingTable worker (#1726)
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-24 16:02:08 +02:00
Maximilian Rink
6fa38027e2 fix(wireguard): guard nil service context in endpoint clear (#1708)
The endpoint worker's delete() path calls clear(nil, nil, service), and
clear() unconditionally dereferenced svcCtx.LeaderCancel, panicking on
every service or endpoint deletion in WireGuard mode (reached via the
endpoint watcher on a Deleted event). Guard the dereference with a nil
check on svcCtx. Adds a regression test that reproduces the panic.

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
2026-08-24 16:00:57 +02:00
dependabot[bot]
b478234d29 build(deps): bump google.golang.org/grpc from 1.83.0 to 1.83.1 (#1731)
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.83.0 to 1.83.1.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.83.0...v1.83.1)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-version: 1.83.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-24 09:35:27 +02:00
dependabot[bot]
03241ee27f build(deps): bump k8s.io/client-go from 0.36.3 to 0.36.4 (#1733)
Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.36.3 to 0.36.4.
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.36.3...v0.36.4)

---
updated-dependencies:
- dependency-name: k8s.io/client-go
  dependency-version: 0.36.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-24 09:35:20 +02:00
dependabot[bot]
6e8b391685 build(deps): bump golang from 1.26.6-alpine3.23 to 1.27.0-alpine3.23 (#1729)
Bumps golang from 1.26.6-alpine3.23 to 1.27.0-alpine3.23.

---
updated-dependencies:
- dependency-name: golang
  dependency-version: 1.27.0-alpine3.23
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-24 09:35:06 +02:00
Maximilian Rink
e5ff483a23 fix: address review findings from #1698 (#1702)
* fix(e2e/services): pass metrics address as container arg

The prometheusHTTPServer env var is never read by kube-vip; environment
parsing only knows prometheus_server and ignores empty values, so every
parallel DaemonSet still tried to bind the :2112 flag default on the
shared host network. Pass the address as --prometheusHTTPServer instead,
where an empty value really disables the metrics server.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* fix(egress): restore CIDR fallback and cluster-wide SNAT exclusion

Auto-discovery dropped the kube-controller-manager fallback, breaking
clusters whose CNI does its own IPAM without Node PodCIDR allocation.
Fall back to the --cluster-cidr/--service-cluster-ip-range flags when
the ServiceCIDR or Node APIs yield nothing.

Per-node PodCIDR discovery combined with getSameFamilyCidr also
narrowed the SNAT exclusion to the local node's pod CIDR, so egress
traffic to pods on other nodes was SNAT'd to the VIP. Exclude every
same-family pod CIDR in both the nftables and iptables paths.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* test(e2e/services): fail failover tests when the VIP never recovers

leaderFailover and podFailover returned nil when the watch drained
without any post-kill update passing httpTest, so an unreachable VIP
after failover passed CI. Keep the warn-and-continue polling, but turn
the drain path into an error.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* test(e2e/services): require sustained quiet window in lease error check

A single quiet 2 s interval can coincide with the pause between failing
election retries and produce a false pass for the #1664 guard. Require
three consecutive intervals without counter growth and reset the streak
whenever the counter grows.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* fix(svcs): log service deletion only when a tracked service is removed

deleteTrackedService logged '(svcs) deleted' for every watch event of
every non-LoadBalancer service, even when nothing was tracked. Move the
log inside the svcCtx guard.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* fix(svcs): skip API refresh when egress endpoints are unchanged

updateEgressConfiguration issued a Services Get on every no-op watch
event before short-circuiting. Compare the ActiveEndpoint annotations
first and only refresh the service from the API when they differ.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* test(e2e/services): drop dead StartServiceTest and restore run summary

StartServiceTest was dead code once main.go inlined the phases, and the
E2E_KEEP_LOGS-conditional temp-dir cleanup plus the 'Testing Complete'
summary silently disappeared with it. Reinstate both in run() and
collect every test result so parallel failures are all reported instead
of collapsing to the errgroup's first error.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* style(e2e): gofmt test files

gofmt -l flagged e2e_bgp_healthcheck_test.go (the Mode == ModeBGP block
body was never re-indented) and a trailing space in e2e_sigusr1_test.go.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* ci: write ginkgo JSON reports where the artifact upload finds them

Without --output-dir ginkgo resolves the report path relative to the
suite directory, so /tmp/kube-vip-test-report-*.json landed under
testing/e2e/tmp/ and the /tmp/kube-vip-test* artifact glob missed it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* ci: scope cancel-in-progress to PRs and restore tag builds

cancel-in-progress: true also cancelled in-progress main builds on
back-to-back merges; limit cancellation to pull_request runs. The
branches: [main] filter added in the refactor dropped the tag pushes
the bare push trigger previously covered, and the release workflow
only publishes images, so restore lint/test coverage for v* tags.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* ci: fail the validation job on unformatted Go code

make check only formats '*.go pkg cmd' and merely warns, which let
unformatted files under testing/ land. Add a blocking gofmt -l check
over the whole repo that prints the offending files.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* test(e2e/services): kill kube-vip pods in the test namespace during failover

Since the parallel-phase refactor the kube-vip DaemonSet runs in the
per-test namespace, but the failover killer still listed pods in
kube-system, so it never deleted a single pod and no failover was ever
induced. The lenient drain path masked this; the strict assertion
exposed it.

Target the test namespace, gate success on events observed after at
least one actual pod kill, extend the watch window to 60s to leave
convergence margin after the ~25s kill churn, and return distinct
errors for aborted watch, zero kills, and unreachable VIP. Detach test
cleanup from the errgroup context (context.WithoutCancel) and demote
cleanup-path Fatals to errors so one failing test no longer kills
sibling teardown.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

* test(e2e/bgp): raise route-destination timeout for parallel runs

With ginkgo --procs=4, sibling processes create kind clusters and load
images on the same runner; the 30s window for the first route check
overlaps that contention and flakes (observed >13s convergence even in
passing specs). Use 120s, matching the BGP health-check suite's
re-announcement timeout. Eventually returns on success, so passing
specs are unaffected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

---------

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-24 09:30:19 +02:00
Daniel Finneran
bee5cfe4a2 Merge pull request #1698 from kube-vip/refactor/testing
Refactor/testing to speedup CI in Pull Requests
2026-08-21 15:42:46 +01:00
Marcel Fest
81d54050c7 test(e2e/rt): reduce cluster size for four workers
Use one control-plane node for basic route checks and three for health-check scenarios so stopping one API server preserves quorum. Four Ginkgo workers now peak at eight control-plane containers instead of twelve.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:39 +02:00
Marcel Fest
5e9fcf642c fix: formatting
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:38 +02:00
Marcel Fest
83797e4da1 chore: remove obsolete versioned deployment examples
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:38 +02:00
Marcel Fest
ed28c49f48 test(e2e/services): isolate parallel egress watchers
Scope each egress DaemonSet to its test namespace, pin Kind's dual-stack Pod CIDRs, retain per-instance nftables tables, and grant the test service account ServiceCIDR discovery access.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:35 +02:00
Marcel Fest
42ba1fefb2 feat(egress): discover cluster CIDRs from Kubernetes APIs
Read ServiceCIDRs and Node PodCIDRs through the Processor client, retain a Service-only controller-manager fallback, and grant generated ClusterRoles get/list/watch access to ServiceCIDRs.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:26 +02:00
Marcel Fest
988eb0994a fix(services): tear down VIP when Service leaves LoadBalancer type
Base teardown on tracked UID state so LoadBalancer-to-ClusterIP transitions cancel the service context and remove the active instance even when the new object is ignored.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:25 +02:00
Marcel Fest
90a3892271 test(e2e): parallelize mode scenarios
Run ARP, routing-table, and BGP modes on separate runners, shard scenario resources by Ginkgo worker, and cap routing-table mode at two workers for Kind stability.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:24 +02:00
Marcel Fest
c553663654 fix(egress): stabilize endpoint-driven SNAT reconciliation
Wait for ready backends, isolate selectors and nftables instances, tolerate transient failover traffic, refresh late endpoint state, and preserve local traffic-policy coverage.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:23 +02:00
Marcel Fest
b6151a4454 test(e2e/services): isolate parallel test phases
Run independent service scenarios in dedicated namespaces, add per-namespace RBAC and cleanup barriers, make goroutine cleanup cancellation-safe, and parameterize CI execution with phase timing.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:22 +02:00
Marcel Fest
6f69d4511f test(e2e): harden cluster readiness checks
Replace fixed waits with polling, remove obsolete serialization, and align the BGP health-check mode guard with the rest of the BGP suite.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-21 16:07:20 +02:00
Marcel Fest
5e2220fd4d Fix/watch err (#1697)
* refactor(errors): centralize fatal error handling

Detect wrapped PanicError values consistently and preserve their underlying causes when adding fatal context. Apply the helpers to manager, cluster, and IPVS error paths.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(watchers): restart after terminal watch failures

Propagate fatal endpoint watcher failures through the owning service watcher so kube-vip releases leadership instead of remaining active with a stale watch. Treat terminal service, node, and annotation watch failures as errors while preserving clean context cancellation.

Return exhausted authorization failures to RetryWatcher, safely decode watch error objects, and replace direct go-spew diagnostics with structured logging.

Fixes #1685

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(services): replace state after traffic policy changes

Recreate the service context and instance as one generation when a Service change requires teardown. Ignore delayed leadership cleanup from superseded contexts so it cannot remove replacement state.

This prevents a stale Cluster-policy endpoint watcher from winning the service lease after externalTrafficPolicy changes to Local.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(cli): return command errors to container runtime

Propagate manager and service command failures through Cobra so the process exits with status 1. Show usage for invocation errors while keeping runtime failures concise.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* refactor(logging): use structured errors

Replace direct stdout error output with slog records for command failures and traffic mirror qdisc lookup failures.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(watchers): continue after endpoint deletion

Keep EndpointSlice watchers active when an individual endpoint object is deleted so replacement objects can be observed and service traffic can recover.

Assisted-by: GitHub-Copilot:unspecified
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

---------

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-20 13:03:08 +02:00
Vedant Madane
4e13a81af0 docs: link upgrade guide from README (#1696)
Point operators at the in-place upgrade docs on kube-vip.io.

Fixes #499

Signed-off-by: Vedant Madane <vedantnm@gmail.com>
2026-08-20 08:13:33 +02:00
Marcel Fest
a19500b116 fix: logrus required by our test-cases, now only transient because of containerd (#1694)
* fix: logrus required by our test-cases, now only transient because of containerd

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix: duplicate log module

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(race): between service cancelling lease and lease cancelling service

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

* fix(race): again

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

---------

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2026-08-19 17:05:49 +02:00
Dessalines39394
ea916c5a31 docs: point star history chart to a working provider
The star history chart in the README was broken because it relied on a data source that is now restricted by GitHub's stargazer API limits. Point the chart and its link to an alternative provider that uses a different data source requiring no API token, so the chart renders correctly again.
2026-08-12 12:52:55 +00:00
119 changed files with 6410 additions and 2202 deletions

13
.github/suggestion-comment.md vendored Normal file
View File

@@ -0,0 +1,13 @@
I'll help you add a suggestion. Unfortunately, I can't directly add a suggestion to an existing comment through the API. However, here's what I recommend:
**Option 1: Reply with a suggestion**
Create a new comment with a suggested fix:
```suggestion
failed to get an IPv6 address after %d attempt(s), giving up, error: %s
```
**Option 2: Edit your existing comment**
Update your comment to include the suggestion details pointing out that line 284 in the error message says "IPv4" but should say "IPv6" since this is the DHCPv6Client.
Would you like me to create a new reply comment with the suggestion instead?

View File

@@ -26,6 +26,6 @@ jobs:
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.24.0
uses: anchore/sbom-action@v0.24.2
with:
format: cyclonedx-json

View File

@@ -1,18 +1,33 @@
name: For each PR
on:
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
timeout-minutes: 120
env:
GINKGO_PROCS: ${{ matrix.ginkgo-procs }}
strategy:
fail-fast: false
max-parallel: 3
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
include:
- mode: arp
ginkgo-procs: 4
- mode: rt
ginkgo-procs: 4
- mode: bgp
ginkgo-procs: 4
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
@@ -25,21 +40,22 @@ jobs:
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
if: matrix.mode == 'arp'
- name: Run ARP mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-arp
if: matrix.mode== 'arp'
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-arp.json" make e2e-tests-arp
if: matrix.mode == 'arp'
- name: Run RT mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-rt
if: matrix.mode== 'rt'
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-rt.json" make e2e-tests-rt
if: matrix.mode == 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
if: matrix.mode == 'bgp'
- name: Run BGP mode tests
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-bgp
if: matrix.mode== 'bgp'
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-bgp.json" make e2e-tests-bgp
if: matrix.mode == 'bgp'
- name: Change log directory permissions
run: sudo chmod -R 755 /tmp/kube-vip-test*
if: matrix.mode== 'bgp' && always()
if: matrix.mode == 'bgp' && always()
- name: Save logs
uses: actions/upload-artifact@v7
with:
@@ -49,10 +65,11 @@ jobs:
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
timeout-minutes: 60
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
- name: Checkout code
uses: actions/checkout@v7
- name: Install Go

View File

@@ -1,11 +1,19 @@
name: For each commit
on:
push:
branches: [main]
tags: ['v*']
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
validation:
runs-on: ubuntu-latest
name: Checks and linters
timeout-minutes: 30
steps:
- name: Init
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
@@ -20,11 +28,20 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Verify gofmt
run: |
unformatted=$(gofmt -l .)
if [ -n "$unformatted" ]; then
echo "The following files are not gofmt-formatted:"
echo "$unformatted"
exit 1
fi
- name: All checks
run: make check
unit-tests:
runs-on: ubuntu-latest
name: Unit tests
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
@@ -37,6 +54,7 @@ jobs:
integration-tests:
name: Integration tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v7
@@ -49,10 +67,11 @@ jobs:
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Build image with iptables
- name: Build image
run: make dockerx86Action
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master

111
.github/workflows/nightly-e2e.yaml vendored Normal file
View File

@@ -0,0 +1,111 @@
name: Nightly e2e
on:
schedule:
- cron: '30 2 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
unit-coverage:
runs-on: ubuntu-latest
name: Unit tests with coverage
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Install Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Run tests
run: make unit-tests
- name: Summarize coverage
if: always()
run: |
if test -f coverage.out; then
echo "### Unit coverage" >> "$GITHUB_STEP_SUMMARY"
go tool cover -func=coverage.out | tail -1 >> "$GITHUB_STEP_SUMMARY"
else
echo "### Unit coverage: report missing" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload coverage
uses: actions/upload-artifact@v7
with:
name: unit-coverage
path: coverage.out
if-no-files-found: error
if: always()
etcd-e2e:
runs-on: ubuntu-latest
name: Etcd E2E tests
timeout-minutes: 60
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v7
- name: Install Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Prepare Etcd artifacts
if: always()
run: |
mkdir -p /tmp/kube-vip-etcd-artifacts
: > /tmp/kube-vip-etcd-artifacts/suite.log
printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
- name: Run Etcd tests
id: etcd
# Scheduled failures are tolerated only during the initial stabilization window.
# The enforcement step below makes manual runs and later schedules blocking.
continue-on-error: true
shell: bash
run: |
set +e
set -o pipefail
DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
exit_code=${PIPESTATUS[0]}
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
exit "$exit_code"
- name: Summarize Etcd suite
if: always()
env:
OUTCOME: ${{ steps.etcd.outcome }}
EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
run: |
echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
"${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
> /tmp/kube-vip-etcd-artifacts/result.json
- name: Save logs
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: etcd-e2e-logs
path: |
/tmp/kube-vip-etcd-artifacts
/tmp/kube-vip-test*
if-no-files-found: warn
if: always()
- name: Enforce Etcd result
if: always()
env:
EVENT_NAME: ${{ github.event_name }}
OUTCOME: ${{ steps.etcd.outcome }}
run: |
if test "$OUTCOME" = success; then
exit 0
fi
if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
exit 0
fi
echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
exit 1

1
.gitignore vendored
View File

@@ -3,6 +3,7 @@ kube-vip
.vscode
bin
testing/e2e/etcd/certs
coverage.out
pkg/etcd/etcd.pid
pkg/etcd/etcd-data
testing/e2e/e2e.test

View File

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- Propagate `bgp_attach_ip_to_interface` into per-service config so it attaches BGP-mode Service VIPs to the interface as configured.
- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
- Reintroduce BGP config via node annotations. Fixes #1488.
@@ -50,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added support in ipoib interfaces in ARP mode. Fixes #694
### Changed
- BGP mode now honours `enable_leader_election` for services: a single global services leader advertises the service VIPs instead of every node advertising them. Deployments that enabled `enable_leader_election` for the control plane and relied on ECMP/multipath for services must unset it (or switch to `enable_service_election`) to keep the previous datapath. kube-vip logs a warning on startup when this path is taken.
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
- wireguard.go now manages a complete wireguard interface on the current network namespace
- manager_wireguard.go uses the new wireguard.go implementation

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.6-alpine3.23 as dev
FROM golang:1.27.1-alpine3.23 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.6-alpine3.23 as dev
FROM golang:1.27.1-alpine3.23 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.2.3
VERSION := v1.2.4
BUILD := `git rev-parse HEAD`
@@ -16,10 +16,14 @@ TARGETOS=linux
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
DOCKERTAG ?= $(VERSION)
REPOSITORY ?= docker.io/plndr
GO_VERSION := 1.25.6
GO_VERSION := $(word 2,$(shell grep '^go ' go.mod))
K8S_VERSION ?= v1.35.0
GINKGO_ARGS ?=
GINKGO_PROCS ?=
GINKGO_PARALLEL := $(if $(GINKGO_PROCS),--procs=$(GINKGO_PROCS),-p)
BUILDX_CACHE_FLAGS ?=
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker e2e-tests-etcd
all: check install
@@ -77,17 +81,17 @@ docker:
# This will build a local docker image (x86 only), use make dockerLocal for all architectures
dockerx86Local:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) $(BUILDX_CACHE_FLAGS) .
@echo New Multi Architecture Docker image created
dockerx86Action:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
@echo New Multi Architecture Docker image created
dockerx86ActionIPTables:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
@echo New Multi Architecture Docker image created
dockerLocal:
@@ -129,22 +133,25 @@ manifest-test:
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
unit-tests:
go test -race ./...
go test -race -coverprofile=coverage.out -covermode=atomic ./...
unit-tests-docker:
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) make unit-tests
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) sh -c "make unit-tests; status=$$?; chmod 666 coverage.out 2>/dev/null || true; exit $$status"
integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests-arp: get-whoami
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests-rt: get-whoami
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests-bgp: get-whoami get-gobgp
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests-etcd: get-whoami
GOMAXPROCS=4 K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e/etcd
e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp

View File

@@ -18,6 +18,8 @@ The idea behind `kube-vip` is a small self-contained Highly-Available option for
**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
For upgrading an existing install in place (static Pod or DaemonSet), see the [upgrade guide](https://kube-vip.io/docs/upgrade/).
## Features
Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
@@ -130,7 +132,7 @@ Additionally it is now relatively easy and quick to develop with [skaffold](http
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)
[![Star History Chart](https://star-history.dera.page/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.dera.page/#kube-vip/kube-vip&type=date)
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_shield)

View File

@@ -4,17 +4,14 @@ import (
"context"
"fmt"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
log "log/slog"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
@@ -52,8 +49,9 @@ var (
)
var kubeVipCmd = &cobra.Command{
Use: "kube-vip",
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
Use: "kube-vip",
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
SilenceErrors: true,
}
func init() {
@@ -187,11 +185,16 @@ func init() {
}
// Execute - starts the command parsing process
func Execute() {
if err := kubeVipCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
func Execute() int {
cmd, err := kubeVipCmd.ExecuteC()
if err != nil {
log.Error("command failed", "err", err)
if cmd == kubeVipCmd {
_ = cmd.Usage()
}
return 1
}
return 0
}
var kubeVipVersion = &cobra.Command{
@@ -215,26 +218,24 @@ var kubeVipSample = &cobra.Command{
var kubeVipService = &cobra.Command{
Use: "service",
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
cmd.SilenceUsage = true
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
return fmt.Errorf("loading config file: %w", err)
}
}
// parse environment variables, these will overwrite anything loaded from config file
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing env", "err", err)
return
return fmt.Errorf("parsing environment: %w", err)
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
return fmt.Errorf("validating configuration: %w", err)
}
// Change RTN_UNSPEC to default type
@@ -246,8 +247,7 @@ var kubeVipService = &cobra.Command{
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
if err := initConfig.CheckInterface(); err != nil {
log.Error("checking interface", "err", err)
return
return fmt.Errorf("checking interface: %w", err)
}
// User Environment variables as an option to make manifest clearer
@@ -260,8 +260,7 @@ var kubeVipService = &cobra.Command{
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
log.Error("checking subnet exists if vip_address defined", "err", err)
return
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
}
}
@@ -269,8 +268,7 @@ var kubeVipService = &cobra.Command{
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if err != nil {
log.Error("generating CIDR", "err", err)
return
return fmt.Errorf("generating CIDR: %w", err)
}
}
@@ -280,41 +278,39 @@ var kubeVipService = &cobra.Command{
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
return fmt.Errorf("new manager: %w", err)
}
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
if err != nil {
log.Error("manager start", "err", err)
return
return fmt.Errorf("manager start: %w", err)
}
return nil
},
}
var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
cmd.SilenceUsage = true
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
return fmt.Errorf("loading config file: %w", err)
}
}
// parse environment variables, these will overwrite anything loaded from config file
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
return fmt.Errorf("parsing environment: %w", err)
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
return fmt.Errorf("validating configuration: %w", err)
}
// Change RTN_UNSPEC to default type
@@ -329,8 +325,7 @@ var kubeVipManager = &cobra.Command{
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
log.Error("checking subnet exists if vip_address defined", "err", err)
return
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
}
}
@@ -338,8 +333,7 @@ var kubeVipManager = &cobra.Command{
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if err != nil {
log.Error("No interface is specified for kube-vip to bind to")
return
return fmt.Errorf("generating CIDR: %w", err)
}
}
@@ -353,12 +347,17 @@ var kubeVipManager = &cobra.Command{
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
metrics.RegisterPrometheusMetrics()
// start prometheus server
if initConfig.PrometheusHTTPServer != "" {
wg.Go(func() {
servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
if err := metrics.Serve(ctx, metrics.ServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
}); err != nil {
// Continue even if metrics server fails
log.Error("prometheus HTTP server", "err", err)
}
})
}
@@ -388,13 +387,11 @@ var kubeVipManager = &cobra.Command{
}
if mode == "" {
log.Error("no valid kube-vip mode detected, ensure a supported mode is configured")
return
return fmt.Errorf("no valid kube-vip mode detected, ensure a supported mode is configured")
}
if modesEnabled > 1 {
log.Error("multiple kube-vip modes detected, ensure only one mode is configured")
return
return fmt.Errorf("multiple kube-vip modes detected, ensure only one mode is configured")
}
// Provide configuration to output/logging
@@ -402,18 +399,15 @@ var kubeVipManager = &cobra.Command{
// End if nothing is enabled
if !initConfig.EnableServices && !initConfig.EnableControlPlane {
log.Error("no features are enabled")
return
return fmt.Errorf("no features are enabled")
}
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
log.Error("auto subnet discovery cannot be used outside ARP mode")
return
return fmt.Errorf("auto subnet discovery cannot be used outside ARP mode")
}
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
log.Error("auto subnet discovery cannot be used if VIP address was provided")
return
return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
}
// If we're using wireguard then all traffic goes through the wg0 interface
@@ -430,20 +424,17 @@ var kubeVipManager = &cobra.Command{
log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
if err != nil {
log.Error("adding link", "err", err)
return
return fmt.Errorf("adding link: %w", err)
}
l, err = netlink.LinkByName(initConfig.Interface)
if err != nil {
log.Error("finding link", "err", err)
return
return fmt.Errorf("finding link: %w", err)
}
}
}
err = netlink.LinkSetUp(l)
if err != nil {
log.Error("setting link UP", "err", err)
return
return fmt.Errorf("setting link UP: %w", err)
}
} else { // if we're not using Wireguard then we'll need to use an actual interface
@@ -453,8 +444,7 @@ var kubeVipManager = &cobra.Command{
defaultIF, err := vip.GetDefaultGatewayInterface()
if err != nil {
_ = cmd.Help()
log.Error("detecting interface", "err", err)
return
return fmt.Errorf("detecting interface: %w", err)
}
initConfig.Interface = defaultIF.Name
log.Info("kube-vip bind", "interface", initConfig.Interface)
@@ -470,8 +460,7 @@ var kubeVipManager = &cobra.Command{
}
// Perform a check on the state of the interface
if err := initConfig.CheckInterface(); err != nil {
log.Error("checking interface", "err", err)
return
return fmt.Errorf("checking interface: %w", err)
}
// User Environment variables as an option to make manifest clearer
@@ -483,81 +472,22 @@ var kubeVipManager = &cobra.Command{
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
return fmt.Errorf("new manager: %w", err)
}
metrics.RegisterPrometheusMetrics()
// Label metrics after the call to manager.New, as it may modify the node name
// if it was not set in the configuration.
metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
if err != nil {
log.Error("start manager", "err", err)
return
return fmt.Errorf("start manager: %w", err)
}
return nil
},
}
// PrometheusHTTPServerConfig defines the Prometheus server configuration.
type PrometheusHTTPServerConfig struct {
// Addr sets the http server address used to expose the metric endpoint
Addr string
}
func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerConfig) {
var err error
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
_, _ = w.Write([]byte(`<html>
<head><title>kube-vip</title></head>
<body>
<h1>kube-vip Metrics</h1>
<p><a href="` + "/metrics" + `">Metrics</a></p>
</body>
</html>`))
})
srv := &http.Server{
Addr: config.Addr,
Handler: mux,
ReadHeaderTimeout: 2 * time.Second,
}
wg := sync.WaitGroup{}
wg.Go(func() {
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("prometheus HTTP server", "err", err)
return
}
})
log.Info("prometheus HTTP server started")
<-ctx.Done()
// create prometheus shutdown context (independent of other contexts)
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
cancel()
}()
if err = srv.Shutdown(ctxShutDown); err != nil {
log.Error("shutting down prometheus HTTP server", "err", err)
return
}
if err == http.ErrServerClosed {
err = nil
}
log.Info("prometheus HTTP server stopped")
wg.Wait()
}
func GenerateCidrRange(address string, dnsMode string) (string, error) {
var cidrs []string

View File

@@ -1,74 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: ghcr.io/kube-vip/kube-vip:0.3.7
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
- --configMap
- plndr-configmap
- --arp
- --interface
- ens192
- --log
- "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
status: {}
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: lease-access
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configMap"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: lease-access
subjects:
- kind: User
name: system:serviceaccount:default:default
roleRef:
kind: Role
name: lease-access
apiGroup: rbac.authorization.k8s.io

View File

@@ -1,83 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: vip
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configmaps", "endpoints"]
verbs: ["watch", "get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role-bind
subjects:
- kind: ServiceAccount
name: vip
apiGroup: ""
roleRef:
kind: Role
name: vip-role
apiGroup: ""
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: ghcr.io/kube-vip/kube-vip:0.3.7
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
env:
- name: vip_interface
value: "ens192"
- name: vip_configmap
value: "plndr"
- name: vip_arp
value: "true"
- name: vip_loglevel
value: "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
serviceAccountName: vip
status: {}

View File

@@ -1,83 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: vip
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configmaps", "endpoints"]
verbs: ["watch", "get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role-bind
subjects:
- kind: ServiceAccount
name: vip
apiGroup: ""
roleRef:
kind: Role
name: vip-role
apiGroup: ""
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: plndr/kube-vip:0.1.4
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
env:
- name: vip_interface
value: "ens192"
- name: vip_configmap
value: "plndr"
- name: vip_arp
value: "true"
- name: vip_loglevel
value: "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
serviceAccountName: vip
status: {}

View File

@@ -1,55 +0,0 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
name: kube-vip-ds
namespace: kube-system
spec:
selector:
matchLabels:
name: kube-vip-ds
template:
metadata:
creationTimestamp: null
labels:
name: kube-vip-ds
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: vip_interface
value: eth0
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: svc_enable
value: "true"
- name: vip_startleader
value: "false"
- name: vip_addpeerstolb
value: "true"
- name: vip_localpeer
value: ip-172-20-40-207:172.20.40.207:10000
- name: vip_address
image: plndr/kube-vip:v0.3.5
imagePullPolicy: Always
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
- SYS_TIME
hostNetwork: true
serviceAccountName: kube-vip
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0

43
go.mod
View File

@@ -5,41 +5,40 @@ go 1.26.4
require (
github.com/cloudflare/ipvs v0.12.0
github.com/containernetworking/plugins v1.9.1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.5.2+incompatible
github.com/florianl/go-conntrack v0.7.0
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.21.9
github.com/google/go-containerregistry v0.22.1
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.7.1
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.32.1
github.com/onsi/gomega v1.42.1
github.com/osrg/gobgp/v4 v4.8.0
github.com/onsi/ginkgo/v2 v2.32.2
github.com/onsi/gomega v1.43.0
github.com/osrg/gobgp/v4 v4.9.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.24.1
github.com/sirupsen/logrus v1.9.4
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
github.com/vishvananda/netlink v1.3.1
github.com/stretchr/testify v1.12.1
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b
github.com/vishvananda/netns v0.0.5
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/pkg/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
go.uber.org/zap v1.28.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.83.0
google.golang.org/grpc v1.83.2
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.36.3
k8s.io/apimachinery v0.36.3
k8s.io/client-go v0.36.3
k8s.io/api v0.36.4
k8s.io/apimachinery v0.36.4
k8s.io/client-go v0.36.4
k8s.io/klog/v2 v2.140.0
sigs.k8s.io/kind v0.32.0
sigs.k8s.io/kind v0.33.0
sigs.k8s.io/yaml v1.6.0
)
@@ -55,6 +54,7 @@ require (
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
@@ -121,7 +121,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -132,15 +131,15 @@ require (
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/mod v0.38.0 // indirect
golang.org/x/net v0.57.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/mod v0.39.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.48.0 // indirect
golang.org/x/tools v0.49.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect

85
go.sum
View File

@@ -98,8 +98,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.21.9 h1:F+D4uZ3iA3DLMJLfhaqMdHJbzeqm/216WGQq2dokuLs=
github.com/google/go-containerregistry v0.21.9/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo=
github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84=
github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
@@ -144,8 +144,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -192,18 +192,18 @@ github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
github.com/osrg/gobgp/v4 v4.8.0 h1:CoprkOZ2nsH8aTHij6xWj18QOrsBwG9D00l1PGw7394=
github.com/osrg/gobgp/v4 v4.8.0/go.mod h1:bJbFm7T2nRANggShfl3I9h0UpPCzu4uAY5J/6dTdRvs=
github.com/osrg/gobgp/v4 v4.9.0 h1:pKOw914kwQ4I/lWNVTfEDosEN3FuqPGytMEInXxpTyQ=
github.com/osrg/gobgp/v4 v4.9.0/go.mod h1:bJbFm7T2nRANggShfl3I9h0UpPCzu4uAY5J/6dTdRvs=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
@@ -246,11 +246,11 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
@@ -265,8 +265,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b h1:XtEhFJO3IqjQWHJZ3bbNm7LtbDehriJK65KW+6lnw+Q=
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b/go.mod h1:lEui7SPMd9fgxzHVGRAvTxsBGCF6PRH81o2kLWLWHgw=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -307,46 +307,45 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
@@ -358,8 +357,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -373,12 +372,12 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w=
k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg=
k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM=
k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE=
k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg=
k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30=
k8s.io/api v0.36.4 h1:RxrvqCL6vgH5/+UnTeu1IIFqYmGfy0hnyrod1rn35Oo=
k8s.io/api v0.36.4/go.mod h1:S2B3orCFBDhrgyWbLeuKcT2QdHIpQesBkCYSlWtwUOw=
k8s.io/apimachinery v0.36.4 h1:PT2UzkupGuAx/+xT5XjiMJ1WGpY3fn9/hdAvjweRet4=
k8s.io/apimachinery v0.36.4/go.mod h1:p2I2dipt7JHG+quVwQ1d02d28O4GdDi77RByQ13MTpk=
k8s.io/client-go v0.36.4 h1:MDvfDNvMSt0Br94SK8neviVlwL9qifw9B26hJCpD1K0=
k8s.io/client-go v0.36.4/go.mod h1:pNK4WKELbwlEDvtbE8l22lEZL5THYF61H5EealokZmA=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg=
@@ -389,8 +388,8 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.32.0 h1:p9hscbj98u/qyrjVpjId86LI70nQmbSsipV7wCG10Xk=
sigs.k8s.io/kind v0.32.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/kind v0.33.0 h1:AjvDv3vOygb/VKLVQW87lfktIBzkxR8Ump9DjxC8+Lk=
sigs.k8s.io/kind v0.33.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw=

View File

@@ -1,6 +1,8 @@
package main
import (
"os"
"github.com/kube-vip/kube-vip/cmd"
)
@@ -14,5 +16,5 @@ func main() {
cmd.Release.Version = Version
cmd.Release.Build = Build
cmd.Execute()
os.Exit(cmd.Execute())
}

View File

@@ -25,6 +25,11 @@ const defaultBGPPort uint32 = 179
// AddPeer will add peers to the BGP configuration
func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
remotePort := defaultBGPPort
if peer.Port != 0 {
remotePort = uint32(peer.Port)
}
p := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer.Address,
@@ -50,7 +55,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
Transport: &api.Transport{
MtuDiscovery: true,
RemoteAddress: peer.Address,
RemotePort: defaultBGPPort,
RemotePort: remotePort,
},
}
@@ -85,6 +90,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
if err != nil {
log.Error("failed to get MP-BGP addresses, will not us MP-BGP for this host", "error", err)
b.setPeerSource(p)
} else {
p.AfiSafis = []*api.AfiSafi{
{
@@ -136,13 +142,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
}
}
} else {
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
b.setPeerSource(p)
}
if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
@@ -152,6 +152,16 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
return nil
}
func (b *Server) setPeerSource(p *api.Peer) {
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
}
func (b *Server) getPath(ip net.IP) *apiutil.Path {
isV6 := ip.To4() == nil

View File

@@ -0,0 +1,146 @@
package bgp
import (
"context"
"sync"
"testing"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v4/api"
gobgp "github.com/osrg/gobgp/v4/pkg/server"
)
func TestAddPeerConfiguresTransportOptions(t *testing.T) {
tests := []struct {
name string
newServer func(*testing.T) *Server
peer kubevip.BGPPeer
wantPort uint32
wantLocalAddr string
wantInterface string
}{
{
name: "configured remote port",
newServer: func(t *testing.T) *Server {
return newStartedTestBGPServer(t, kubevip.BGPConfig{
AS: 65000,
RouterID: "192.0.2.1",
Peers: []kubevip.BGPPeer{{Address: "192.0.2.10", AS: 65001}},
})
},
peer: kubevip.BGPPeer{Address: "192.0.2.10", AS: 65001, Port: 180},
wantPort: 180,
},
{
name: "configured source interface after MP-BGP fallback",
newServer: func(t *testing.T) *Server {
return newPeerTestServer(t, kubevip.BGPConfig{
AS: 65000,
RouterID: "192.0.2.1",
SourceIF: "lo",
MpbgpNexthop: "fixed",
Peers: []kubevip.BGPPeer{{Address: "192.0.2.20", AS: 65001}},
MpbgpIPv4: "",
MpbgpIPv6: "",
})
},
peer: kubevip.BGPPeer{Address: "192.0.2.20", AS: 65001},
wantInterface: "lo",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
server := tt.newServer(t)
if err := server.AddPeer(context.Background(), tt.peer); err != nil {
t.Fatalf("AddPeer() error = %v", err)
}
peer := listTestPeer(t, server, tt.peer.Address)
if peer.GetTransport() == nil {
t.Fatal("configured peer has no transport")
}
if tt.wantPort != 0 && peer.GetTransport().GetRemotePort() != tt.wantPort {
t.Fatalf("remote port = %d, want %d", peer.GetTransport().GetRemotePort(), tt.wantPort)
}
if tt.wantLocalAddr != "" && peer.GetTransport().GetLocalAddress() != tt.wantLocalAddr {
t.Fatalf("local address = %q, want %q", peer.GetTransport().GetLocalAddress(), tt.wantLocalAddr)
}
if tt.wantInterface != "" && peer.GetTransport().GetBindInterface() != tt.wantInterface {
t.Fatalf("bind interface = %q, want %q", peer.GetTransport().GetBindInterface(), tt.wantInterface)
}
})
}
}
func newStartedTestBGPServer(t *testing.T, config kubevip.BGPConfig) *Server {
t.Helper()
server, err := NewBGPServer(config, log.LevelError)
if err != nil {
t.Fatalf("NewBGPServer() error = %v", err)
}
go server.s.Serve()
if err := server.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: config.AS,
RouterId: config.RouterID,
ListenPort: -1,
},
}); err != nil {
server.s.Stop()
t.Fatalf("StartBgp() error = %v", err)
}
t.Cleanup(server.s.Stop)
return server
}
func listTestPeer(t *testing.T, server *Server, address string) *api.Peer {
t.Helper()
var got *api.Peer
if err := server.s.ListPeer(context.Background(), &api.ListPeerRequest{Address: address}, func(peer *api.Peer) {
got = peer
}); err != nil {
t.Fatalf("ListPeer() error = %v", err)
}
if got == nil {
t.Fatalf("ListPeer() returned no peer for %s", address)
}
return got
}
func newPeerTestServer(t *testing.T, cfg kubevip.BGPConfig) *Server {
t.Helper()
raw := startEmbeddedRawBGP(t)
return &Server{s: raw, c: &cfg, tracker: make(map[string]map[string]bool)}
}
func startEmbeddedRawBGP(t *testing.T) *gobgp.BgpServer {
t.Helper()
raw := gobgp.NewBgpServer()
go raw.Serve()
if err := raw.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: 65000,
RouterId: "192.0.2.1",
ListenPort: -1,
},
}); err != nil {
t.Fatalf("starting embedded BGP server: %v", err)
}
var stopOnce sync.Once
t.Cleanup(func() {
stopOnce.Do(func() {
if err := raw.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
t.Logf("stopping embedded BGP server: %v", err)
}
})
})
return raw
}

View File

@@ -15,6 +15,11 @@ import (
gobgp "github.com/osrg/gobgp/v4/pkg/server"
)
type BGPManager interface {
AddHost(ctx context.Context, addr string, object string) error
DelHost(ctx context.Context, addr string, object string) error
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"os"
"sync"
"time"
log "log/slog"
@@ -22,6 +23,7 @@ import (
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
stop chan bool
stopMutex sync.Mutex
Network []vip.Network
arpMgr *arp.Manager
routeMgr *route.Manager
@@ -93,6 +95,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
// Stop - Will stop the Cluster and release VIP if needed
func (cluster *Cluster) Stop() {
cluster.stopMutex.Lock()
defer cluster.stopMutex.Unlock()
// Close the stop channel, which will shut down the VIP (if needed)
if cluster.stop != nil {
close(cluster.stop)

View File

@@ -0,0 +1,32 @@
package cluster
import (
"sync"
"sync/atomic"
"testing"
)
func TestStopConcurrentDoesNotRaceOrPanic(t *testing.T) {
c := &Cluster{stop: make(chan bool)}
start := make(chan struct{})
var wg sync.WaitGroup
var panics atomic.Int64
for range 128 {
wg.Go(func() {
<-start
defer func() {
if recover() != nil {
panics.Add(1)
}
}()
c.Stop()
})
}
close(start)
wg.Wait()
if got := panics.Load(); got != 0 {
t.Fatalf("concurrent Stop panicked %d time(s)", got)
}
}

View File

@@ -29,14 +29,7 @@ import (
"k8s.io/client-go/kubernetes"
)
// BGPRouteManager allows to manage the routes announced by the BGP server.
type BGPRouteManager interface {
AddHost(ctx context.Context, addr string, object string) error
DelHost(ctx context.Context, addr string, object string) error
}
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager,
bgpServer BGPRouteManager, killFunc func()) error {
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager, bgpServer bgp.BGPManager, killFunc func()) error {
var err error
@@ -113,7 +106,7 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
err = em.NodeWatcher(ctx, lb, c.Port)
if err != nil {
log.Error("Error watching node labels", "err", err)
if errors.Is(err, &utils.PanicError{}) {
if utils.IsPanicError(err) {
killFunc()
return
}
@@ -287,7 +280,28 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
return nil
}
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer BGPRouteManager, vipCIDR string) {
func (cluster *Cluster) bgpHealthCheck(ctx context.Context, c *kubevip.Config) (bool, error) {
statusCode := 0
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
if err != nil {
return false, fmt.Errorf("building request %v: %w", req, err)
} else {
resp, err := cluster.healthCheckHTTPClient.Do(req)
if err != nil {
return false, fmt.Errorf("checking control-plane: %w", err)
}
defer resp.Body.Close()
statusCode = resp.StatusCode
}
healthy := statusCode == http.StatusOK
if !healthy {
return healthy, fmt.Errorf("wrong status code: %d", statusCode)
}
return healthy, nil
}
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer bgp.BGPManager, vipCIDR string) {
period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
consecutiveFailures := 0
@@ -304,24 +318,7 @@ func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Confi
)
for {
statusCode := 0
var healthErr error
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
if err != nil {
healthErr = err
} else {
resp, err := cluster.healthCheckHTTPClient.Do(req)
if err != nil {
healthErr = err
} else {
defer resp.Body.Close()
statusCode = resp.StatusCode
}
}
healthy := healthErr == nil && statusCode == http.StatusOK
healthy, healthErr := cluster.bgpHealthCheck(ctx, c)
if healthy {
consecutiveFailures = 0
if !routeAnnounced {
@@ -336,10 +333,7 @@ func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Confi
consecutiveFailures++
if healthErr != nil {
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
} else {
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "status", statusCode)
}
if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
@@ -399,7 +393,7 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, wg *sync.WaitGroup) error {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp bgp.BGPManager, name string, wg *sync.WaitGroup) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
@@ -429,7 +423,7 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
lbCancel()
return utils.NewPanicError(fmt.Sprintf("failed to set mask for subnet %q: %s", c.VIPSubnet, err.Error()))
return utils.WrapPanicError(err, "failed to set mask for subnet %q", c.VIPSubnet)
}
_, err := network.DeleteIP()
if err != nil {

View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/route"
@@ -28,7 +29,7 @@ func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
@@ -40,7 +41,7 @@ func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while unhealthy")
@@ -56,7 +57,7 @@ func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 3
startVipService(t, cfg, bgpManager)
@@ -78,7 +79,7 @@ func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
@@ -100,7 +101,7 @@ func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cancelContext, vipServiceDone := startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
cancelContext, vipServiceDone := startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
@@ -121,7 +122,7 @@ func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
bgpManager := newMockBGPRouteManager()
bgpManager.setAddErr(errTestAddHost)
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while AddHost errors")
@@ -137,7 +138,7 @@ func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
@@ -195,9 +196,8 @@ func (e *testError) Error() string { return e.msg }
// startVipService launches vipService in a goroutine with a mock network and
// registers a cleanup to cancel the context and wait for it to finish.
// Uses InitCluster so the real code parses certs for the BGP health check client.
func startVipService(t *testing.T, cfg *kubevip.Config, bgpManager *mockBGPRouteManager) (context.CancelFunc, <-chan struct{}) {
func startVipService(t *testing.T, cfg *kubevip.Config, bgpServer bgp.BGPManager) (context.CancelFunc, <-chan struct{}) {
t.Helper()
c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("InitCluster: %v", err)
@@ -208,7 +208,7 @@ func startVipService(t *testing.T, cfg *kubevip.Config, bgpManager *mockBGPRoute
done := make(chan struct{})
go func() {
_ = c.StartVipService(ctx, cfg, nil, bgpManager, func() {})
_ = c.StartVipService(ctx, cfg, nil, bgpServer, func() {})
close(done)
}()
@@ -247,14 +247,14 @@ func startRoutingTableVipService(t *testing.T, cfg *kubevip.Config, network *moc
}
func newRoutingTableConfig(url, caPath string) *kubevip.Config {
cfg := newTestConfig(url, caPath)
cfg := newBGPConfig(url, caPath)
cfg.EnableBGP = false
cfg.EnableRoutingTable = true
cfg.BackendHealthCheckInterval = 1
return cfg
}
func newTestConfig(url, caPath string) *kubevip.Config {
func newBGPConfig(url, caPath string) *kubevip.Config {
return &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{

View File

@@ -52,9 +52,8 @@ func (n *ns) add(name string, output chan<- watch.Event) *object {
return i
}
func (n *ns) del(name string) {
if _, exists := n.Load(name); exists {
n.Delete(name)
func (n *ns) del(name string, object *object) {
if n.CompareAndDelete(name, object) {
n.cnt.Add(-1)
}
}
@@ -119,35 +118,57 @@ func (d *debouncer) Start(ctx context.Context) error {
return fmt.Errorf("objects of type %T are not supported", v)
}
eventNs, exists := d.getNs(namespace)
if !exists {
// if not, create new map for the namespace
eventNs = d.addNs(namespace)
}
processEvent:
for {
eventNs, exists := d.getNs(namespace)
if !exists {
// if not, create new map for the namespace
eventNs = d.addNs(namespace)
}
// check if the object was previously reconciled
eventObject, exists := eventNs.get(name)
// check if the object was previously reconciled
eventObject, exists := eventNs.get(name)
// if not and the event is not of type 'Deleted', create new object
if !exists && tmp.Type != watch.Deleted {
eventObject = eventNs.add(name, d.output)
// if not and the event is not of type 'Deleted', create new object
if !exists && tmp.Type != watch.Deleted {
eventObject = eventNs.add(name, d.output)
wg.Go(func() {
// start deboucing events for this object
eventObject.start(debouncerCtx, d.debounceTime)
// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
eventObject = nil
eventNs.del(name)
// if namespace is empty, delete the namespace map
if eventNs.cnt.Load() == 0 {
d.delNs(namespace)
workerObject := eventObject
workerNs := eventNs
workerName := name
workerNamespace := namespace
workerObject.onStop = func() {
// Remove the object before its worker can become receiver-less.
workerNs.del(workerName, workerObject)
}
})
}
if eventObject != nil {
wg.Go(func() {
// start deboucing events for this object
workerObject.start(debouncerCtx, d.debounceTime)
// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
workerNs.del(workerName, workerObject)
// if namespace is empty, delete the namespace map
if workerNs.cnt.Load() == 0 {
d.delNs(workerNamespace, workerNs)
}
})
}
if eventObject == nil {
break processEvent
}
// pass the watch event to the debouncer object
eventObject.input <- tmp
select {
case eventObject.input <- tmp:
break processEvent
case <-eventObject.stopChan:
// The object stopped after the map lookup. Retry the event
// against the newly-created object instead of dropping it.
continue processEvent
case <-debouncerCtx.Done():
return nil
}
}
}
}
@@ -181,8 +202,8 @@ func (d *debouncer) addNs(namespace string) *ns {
return &n
}
func (d *debouncer) delNs(namespace string) {
d.namespaces.Delete(namespace)
func (d *debouncer) delNs(namespace string, ns *ns) {
d.namespaces.CompareAndDelete(namespace, ns)
}
type object struct {
@@ -190,6 +211,7 @@ type object struct {
output chan<- watch.Event
stopChan chan any
stopOnce sync.Once
onStop func()
}
func newObject(output chan<- watch.Event) *object {
@@ -247,6 +269,9 @@ func (o *object) start(ctx context.Context, debounceTime time.Duration) {
func (o *object) stop() {
o.stopOnce.Do(func() {
if o.onStop != nil {
o.onStop()
}
close(o.stopChan)
})
}

View File

@@ -0,0 +1,162 @@
package debouncer
import (
"context"
"runtime"
"testing"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
)
func TestStartReturnsWhenCancellationInterruptsObjectForwarding(t *testing.T) {
input := make(chan watch.Event)
d := &debouncer{
input: input,
output: make(chan watch.Event),
stopChan: make(chan any),
debounceTime: 200 * time.Millisecond,
}
// Leave the object without a receiver. This is the state reached when its
// worker exits on context cancellation just before Start forwards an event.
ns := d.addNs("default")
ns.add("example", d.output)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- d.Start(ctx) }()
event := watch.Event{
Type: watch.Modified,
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "example", Namespace: "default",
}},
}
sent := make(chan struct{})
go func() {
input <- event
close(sent)
}()
select {
case <-sent:
case <-time.After(250 * time.Millisecond):
cancel()
t.Fatal("debouncer did not receive the test event")
}
cancel()
// Nothing ever receives from the object, so Start can only return by
// abandoning the blocked forward when the context is cancelled. Receiving
// here instead would make the forward succeed and the assertion racy.
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("debouncer remained blocked forwarding an event after context cancellation")
}
}
func TestStartRecreatesObjectAfterDeletionWithoutCancellation(t *testing.T) {
previousProcs := runtime.GOMAXPROCS(1)
t.Cleanup(func() { runtime.GOMAXPROCS(previousProcs) })
input := make(chan watch.Event)
d, err := New(input, "200ms")
if err != nil {
t.Fatalf("failed to create debouncer: %s", err)
}
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
done := make(chan error, 1)
go func() { done <- d.Start(ctx) }()
service := func(eventType watch.EventType, resourceVersion string) watch.Event {
return watch.Event{
Type: eventType,
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "example",
Namespace: "default",
ResourceVersion: resourceVersion,
}},
}
}
send := func(event watch.Event) {
t.Helper()
select {
case input <- event:
case <-time.After(time.Second):
t.Fatal("debouncer did not receive the test event")
}
}
send(service(watch.Added, "initial"))
select {
case event := <-d.output:
if event.Type != watch.Added {
t.Fatalf("expected initial Added event, got %s", event.Type)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not emit the initial event")
}
eventNs, exists := d.getNs("default")
if !exists {
t.Fatal("debouncer did not create the namespace map")
}
oldObject, exists := eventNs.get("example")
if !exists {
t.Fatal("debouncer did not create the object")
}
send(service(watch.Deleted, "deleted"))
select {
case event := <-d.output:
if event.Type != watch.Deleted {
t.Fatalf("expected Deleted event, got %s", event.Type)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not emit the Deleted event")
}
select {
case <-oldObject.stopChan:
case <-time.After(time.Second):
t.Fatal("object did not self-terminate")
}
if _, exists := eventNs.get("example"); exists {
t.Fatal("self-terminated object remained in the namespace map")
}
send(service(watch.Modified, "fresh"))
select {
case event := <-d.output:
if event.Type != watch.Modified {
t.Fatalf("expected fresh Modified event, got %s", event.Type)
}
service, ok := event.Object.(*v1.Service)
if !ok {
t.Fatalf("expected a Service event, got %T", event.Object)
}
if service.ResourceVersion != "fresh" {
t.Fatalf("expected the fresh event, got resource version %q", service.ResourceVersion)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not process the fresh event after object deletion")
}
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("debouncer returned an error: %s", err)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not stop")
}
}

View File

@@ -2,14 +2,12 @@ package election
import (
"context"
"errors"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
@@ -17,7 +15,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/utils"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
@@ -62,7 +59,7 @@ func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Cl
func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
switch c.LeaderElectionType {
case "kubernetes", "":
runKubernetesLeaderElectionOrDie(ctx, run)
return runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
return err
@@ -74,20 +71,25 @@ func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
return nil
}
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
annotations, err := kubevip.WithLeaseVIPs(run.LeaseAnnotations, run.Config.InstanceName, run.Config.RoutingProtocol, run.VIPs)
if err != nil {
return err
}
leaseClient := run.Mgr.KubernetesClient.CoordinationV1().Leases(run.LeaseID.Namespace())
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
baseLock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.LeaseID.Name(),
Namespace: run.LeaseID.Namespace(),
Annotations: run.LeaseAnnotations,
Name: run.LeaseID.Name(),
Namespace: run.LeaseID.Namespace(),
},
Client: run.Mgr.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.Config.NodeName,
},
}
lock := newAnnotatedLeaseLock(baseLock, leaseClient, run.LeaseID.Name(), annotations)
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
@@ -108,6 +110,7 @@ func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
OnNewLeader: run.OnNewLeader,
},
})
return nil
}
func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
@@ -138,6 +141,7 @@ type RunConfig struct {
LeaseID lease.ID
Mgr *Manager
LeaseAnnotations map[string]string
VIPs []string
// onStartedLeading is called when this member starts leading.
OnStartedLeading func(context.Context)
@@ -199,7 +203,7 @@ func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBal
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("adding node to load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
if errors.Is(err, &utils.PanicError{}) {
if utils.IsPanicError(err) {
return fmt.Errorf("add IPVS backend: %w", err)
}
}
@@ -233,23 +237,20 @@ func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBal
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
watchErr = fmt.Errorf("node watcher error, status: %s", status.String())
watchErr = fmt.Errorf("node watcher error: %w", utils.WatchError(event.Object))
log.Error("watcher", "err", watchErr)
default:
}
}
log.Info("Exiting Node watcher")
return watchErr
if watchErr != nil {
return watchErr
}
if ctx.Err() != nil {
return nil
}
return utils.NewPanicError("node watcher channel closed unexpectedly")
}
func checkIfNodeIsReady(node *v1.Node) bool {

View File

@@ -0,0 +1,92 @@
package election
import (
"context"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/util/retry"
)
type annotatedLeaseLock struct {
resourcelock.Interface
leases coordinationv1client.LeaseInterface
name string
annotations map[string]string
}
func newAnnotatedLeaseLock(lock resourcelock.Interface, leases coordinationv1client.LeaseInterface,
name string, annotations map[string]string) resourcelock.Interface {
return &annotatedLeaseLock{Interface: lock, leases: leases, name: name, annotations: annotations}
}
func (lock *annotatedLeaseLock) Create(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
if err := lock.Interface.Create(ctx, record); err != nil {
return err
}
lock.ensure(ctx, record)
return nil
}
func (lock *annotatedLeaseLock) Update(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
if err := lock.Interface.Update(ctx, record); err != nil {
return err
}
lock.ensure(ctx, record)
return nil
}
// ensure applies the configured annotations once this process holds the lease. Failures are
// logged rather than returned: the lease write already succeeded, so reporting an error would
// make the elector stand down while it still holds the lease.
func (lock *annotatedLeaseLock) ensure(ctx context.Context, record resourcelock.LeaderElectionRecord) {
if record.HolderIdentity != lock.Identity() {
return
}
changed, err := lock.ensureAnnotations(ctx)
if err != nil {
log.Warn("failed to annotate lease", "lease", lock.name, "err", err)
return
}
if !changed {
return
}
// Annotating out of band bumps the resourceVersion, so refresh the wrapped lock's
// cached lease or its next optimistic Update conflicts.
if _, _, err := lock.Interface.Get(ctx); err != nil {
log.Warn("failed to refresh lease after annotating", "lease", lock.name, "err", err)
}
}
func (lock *annotatedLeaseLock) ensureAnnotations(ctx context.Context) (bool, error) {
changed := false
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
resource, err := lock.leases.Get(ctx, lock.name, metav1.GetOptions{})
if err != nil {
return err
}
if resource.Annotations == nil {
resource.Annotations = make(map[string]string, len(lock.annotations))
}
resourceChanged := false
for key, value := range lock.annotations {
if resource.Annotations[key] == value {
continue
}
resource.Annotations[key] = value
resourceChanged = true
}
if !resourceChanged {
return nil
}
_, err = lock.leases.Update(ctx, resource, metav1.UpdateOptions{})
if err == nil {
changed = true
}
return err
})
return changed, err
}

View File

@@ -0,0 +1,177 @@
package election
import (
"context"
"fmt"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
func TestAnnotatedLeaseLockPersistsAnnotationsOnCreateAndUpdate(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: "node-a",
},
}
annotations, err := kubevip.WithLeaseVIPs(map[string]string{"example.test/preserved": "true"},
"release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
lock := newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
if err := lock.Create(context.Background(), record); err != nil {
t.Fatalf("Create() error = %v", err)
}
if err := lock.Update(context.Background(), record); err != nil {
t.Fatalf("Update() error = %v", err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatalf("get Lease: %v", err)
}
value, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
if value.InstanceName != "release_a" || value.IFAProto != 248 || len(value.VIPs) != 1 ||
value.VIPs[0] != (kubevip.LeaseVIP{Index: 0, Value: "192.0.2.10", Kind: kubevip.LeaseVIPKindAddress}) {
t.Fatalf("Lease VIP metadata = %+v", value)
}
if resource.Annotations["example.test/preserved"] != "true" {
t.Fatal("Lease update dropped a configured annotation")
}
}
// A failed annotation write must not be reported to the leader elector: the lease itself
// was already written, and an error makes the elector stand down while it still holds it.
func TestAnnotatedLeaseLockAnnotationFailureDoesNotSurfaceToElector(t *testing.T) {
client := fake.NewSimpleClientset()
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: "node-a"},
}
failing := fake.NewSimpleClientset()
failing.PrependReactor("get", "leases", func(k8stesting.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("annotation backend unavailable")
})
annotations, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatal(err)
}
lock := newAnnotatedLeaseLock(base, failing.CoordinationV1().Leases("default"), "lease", annotations)
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
if err := lock.Create(context.Background(), record); err != nil {
t.Fatalf("Create() error = %v, want nil so the elector keeps the lease", err)
}
if err := lock.Update(context.Background(), record); err != nil {
t.Fatalf("Update() error = %v, want nil so the elector keeps the lease", err)
}
if _, err := client.CoordinationV1().Leases("default").Get(context.Background(), "lease",
metav1.GetOptions{}); err != nil {
t.Fatalf("wrapped lock did not write the lease: %v", err)
}
}
func TestAnnotatedLeaseLockFollowerDoesNotOverwriteAnnotations(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
newBase := func(identity string) *resourcelock.LeaseLock {
return &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
}
}
ownerBase := newBase("node-a")
followerBase := newBase("node-b")
active, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatal(err)
}
creator := newAnnotatedLeaseLock(ownerBase, leaseClient, "lease", active)
if err := creator.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
t.Fatalf("Create() error = %v", err)
}
follower, err := kubevip.WithLeaseVIPs(nil, "release_b", 249, []string{"192.0.2.20"})
if err != nil {
t.Fatal(err)
}
observer := newAnnotatedLeaseLock(followerBase, leaseClient, "lease", follower)
if _, _, err := observer.Get(context.Background()); err != nil {
t.Fatalf("Get() error = %v", err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatal(err)
}
if metadata.InstanceName != "release_a" || metadata.IFAProto != 248 {
t.Fatalf("follower overwrote active metadata: %+v", metadata)
}
}
func TestAnnotatedLeaseLockReleaseDoesNotOverwriteSuccessorMetadata(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
newLock := func(identity, instanceName string, protocol int, vip string) resourcelock.Interface {
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
}
annotations, err := kubevip.WithLeaseVIPs(nil, instanceName, protocol, []string{vip})
if err != nil {
t.Fatal(err)
}
return newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
}
first := newLock("node-a", "release_a", 248, "192.0.2.10")
if err := first.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
t.Fatal(err)
}
if err := first.Update(context.Background(), resourcelock.LeaderElectionRecord{}); err != nil {
t.Fatal(err)
}
second := newLock("node-b", "release_b", 249, "192.0.2.20")
if _, _, err := second.Get(context.Background()); err != nil {
t.Fatal(err)
}
if err := second.Update(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-b"}); err != nil {
t.Fatal(err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatal(err)
}
if metadata.InstanceName != "release_b" || metadata.IFAProto != 249 || metadata.VIPs[0].Value != "192.0.2.20" {
t.Fatalf("successor metadata = %+v", metadata)
}
}

View File

@@ -46,20 +46,24 @@ func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, b
}
}
func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Event,
// Reconcile applies a watch event to the provider and reconciles the service
// against the endpoints that remain afterwards. A deleted object is only one of
// potentially several backing the service, so deletions are recomputed rather
// than assumed to empty it. It reports whether the caller should skip this event
// and wait for the next one.
func (p *Processor) Reconcile(svcCtx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string,
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
if err := p.applyEvent(svcCtx, event); err != nil {
return false, err
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, err
return false, fmt.Errorf("[%s] error getting endpoints: %w", p.provider.GetLabel(), err)
}
if err := p.worker.setInstanceEndpointsStatus(svcCtx.Ctx, service, endpoints); err != nil {
@@ -89,11 +93,7 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
svcCtx.SignalReadiness()
// There are local endpoints available on the node
// Process immediately if:
// - No services/leader election is enabled, OR
// - WireGuard is enabled (it always needs immediate DNAT rule updates)
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
if p.shouldProcessInstance() {
if err := p.worker.processInstance(svcCtx, service); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
@@ -106,21 +106,13 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
}
svcCtx.SignalReadiness()
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
if p.shouldProcessInstance() {
if err := p.worker.processInstance(svcCtx, service); err != nil {
return false, fmt.Errorf("failed to process endpointless instance: %w", err)
}
}
} else if svcCtx.Signalled.Load() {
// There are no local endpoints
svcCtx.ResetReadiness()
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
if p.config.EnableARP && !p.config.EnableServicesElection {
i := instance.FindServiceInstance(service, *p.instances)
for _, c := range i.Clusters {
c.Stop()
}
}
p.handleNoEndpoints(svcCtx, service, lastKnownGoodEndpoint)
}
}
@@ -133,13 +125,42 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
return false, nil
}
func (p *Processor) Delete(ctx context.Context, service *v1.Service, id string) error {
if err := p.worker.delete(ctx, service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
// applyEvent updates the provider's view of the objects backing this service.
func (p *Processor) applyEvent(svcCtx *servicecontext.Context, event watch.Event) error {
if event.Type == watch.Deleted {
if err := p.provider.DeleteObject(event.Object); err != nil {
return fmt.Errorf("[%s] error deleting k8s object: %w", p.provider.GetLabel(), err)
}
return nil
}
if err := p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
return nil
}
// shouldProcessInstance reports whether this node has to program the datapath
// itself, rather than waiting to be told to by a leader election callback.
// WireGuard always reprograms, because its DNAT rules are per-endpoint.
func (p *Processor) shouldProcessInstance() bool {
return (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard
}
// handleNoEndpoints tears down everything backing a service that no longer has
// any usable endpoints.
func (p *Processor) handleNoEndpoints(svcCtx *servicecontext.Context, service *v1.Service, lastKnownGoodEndpoint *string) {
svcCtx.ResetReadiness()
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
if p.config.EnableARP && !p.config.EnableServicesElection && p.instances != nil {
if i := instance.FindServiceInstance(service, *p.instances); i != nil {
for _, c := range i.Clusters {
c.Stop()
}
}
}
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
// if we haven't populated one, then do so
family := utils.IPv4Family
@@ -179,28 +200,41 @@ func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint
egressUpdateFunc func(context.Context, *v1.Service) error) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
ip := net.ParseIP(*lastKnownGoodEndpoint)
if *lastKnownGoodEndpoint != "" {
ip := net.ParseIP(*lastKnownGoodEndpoint)
expectIPv6 := service.Annotations[kubevip.EgressIPv6] == "true"
if ip == nil || (ip.To4() == nil) != expectIPv6 {
log.Warn("ignoring active endpoint with unexpected address family",
"service", service.Name,
"namespace", service.Namespace,
"endpoint", *lastKnownGoodEndpoint,
"expected_ipv6", expectIPv6)
return
}
}
// Store old values from ServiceSnapshot to detect if annotation actually changed
// We use the ServiceSnapshot instead of the service parameter because the service parameter
// may have stale annotations if the last update failed
var oldEndpoint, oldEndpointIPv6 string
snapshotFound := false
if p.instances != nil {
serviceInstance := instance.FindServiceInstance(service, *p.instances)
if serviceInstance != nil {
if serviceInstance != nil && serviceInstance.ServiceSnapshot != nil {
snapshotFound = true
oldEndpoint = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
}
}
// Fall back to service annotations if we couldn't find the instance
if oldEndpoint == "" && oldEndpointIPv6 == "" {
// Empty annotations in an existing snapshot are meaningful after a zero-endpoint transition.
if !snapshotFound {
oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
}
// Determine which annotation to update based on IP version
var endpoint, endpointIPv6 string
if ip.To4() == nil && !p.config.EnableEndpoints {
if service.Annotations[kubevip.EgressIPv6] == "true" && !p.config.EnableEndpoints {
// IPv6
endpointIPv6 = *lastKnownGoodEndpoint
endpoint = oldEndpoint // Preserve existing IPv4 if any
@@ -290,7 +324,12 @@ func (p *Processor) startLeaderElection(svcCtx *servicecontext.Context, service
default:
leaseNamespace, serviceLease := lease.ServiceName(service)
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
// The lease is retired once its last service is gone, so an absent one means
// this loop has nothing left to elect for.
l := p.leaseMgr.Get(id)
if l == nil {
return
}
l.Lock()
if !l.Elected.Load() {

View File

@@ -2,7 +2,6 @@ package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
@@ -67,41 +66,13 @@ func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *strin
b.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
svcCtx.CallLeaderCancel()
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(ctx, service)
}
}
return nil
}
func (b *BGP) deleteAction(ctx context.Context, service *v1.Service) {
b.clearBGPHosts(ctx, service)
}
func (b *BGP) clearBGPHosts(ctx context.Context, service *v1.Service) {
ClearBGPHosts(ctx, service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
return nil
}

View File

@@ -22,7 +22,6 @@ type endpointWorker interface {
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(ctx context.Context, service *v1.Service, id string) error
setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error
}
@@ -65,9 +64,7 @@ func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service) erro
func (g *generic) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
g.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
svcCtx.CallLeaderCancel()
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
@@ -105,10 +102,6 @@ func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, err
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ context.Context, _ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
return nil
}

View File

@@ -66,9 +66,7 @@ func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpo
rt.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
svcCtx.CallLeaderCancel()
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
@@ -82,28 +80,6 @@ func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint
}
}
func (rt *RoutingTable) delete(_ context.Context, service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances, rt.routeMgr)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error {
inst := instance.FindServiceInstance(service, *rt.instances)
if inst == nil {

View File

@@ -8,6 +8,7 @@ import (
"time"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
@@ -17,6 +18,7 @@ import (
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
func TestShouldAllowReconcileWithoutEndpoints(t *testing.T) {
@@ -47,6 +49,191 @@ type fakeWorker struct {
processCalled bool
}
type annotationUpdate struct {
endpoint string
endpointIPv6 string
}
type recordingProvider struct {
providers.Provider
updates []annotationUpdate
}
func (p *recordingProvider) UpdateServiceAnnotation(_ context.Context, endpoint, endpointIPv6 string,
_ *v1.Service, _ *kubernetes.Clientset) error {
p.updates = append(p.updates, annotationUpdate{endpoint: endpoint, endpointIPv6: endpointIPv6})
return nil
}
func TestUpdateAnnotationsZeroEndpointsThenSameEndpoint(t *testing.T) {
for _, enableEndpoints := range []bool{true, false} {
providerName := "EndpointSlices"
provider := providers.NewEndpointslices()
if enableEndpoints {
providerName = "Endpoints"
provider = providers.NewEndpoints()
}
for _, family := range []struct {
name string
endpoint string
other string
egressIPv6 bool
}{
{name: "IPv4", endpoint: "10.0.0.1", other: "fd00::1"},
{name: "IPv6", endpoint: "fd00::1", other: "10.0.0.1", egressIPv6: true},
} {
t.Run(providerName+"/"+family.name, func(t *testing.T) {
annotations := map[string]string{kubevip.Egress: "true"}
if family.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
if !enableEndpoints {
if family.egressIPv6 {
annotations[kubevip.ActiveEndpoint] = family.other
annotations[kubevip.ActiveEndpointIPv6] = family.endpoint
} else {
annotations[kubevip.ActiveEndpoint] = family.endpoint
annotations[kubevip.ActiveEndpointIPv6] = family.other
}
} else {
annotations[kubevip.ActiveEndpoint] = family.endpoint
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
}}
serviceInstance := &instance.Instance{ServiceSnapshot: service.DeepCopy()}
instances := []*instance.Instance{serviceInstance}
recorder := &recordingProvider{Provider: provider}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: enableEndpoints},
provider: recorder,
instances: &instances,
}
updateSnapshot := func(_ context.Context, updated *v1.Service) error {
serviceInstance.ServiceSnapshot = updated
return nil
}
noEndpoint := ""
processor.updateAnnotations(service, &noEndpoint, nil, updateSnapshot)
repopulatedEndpoint := family.endpoint
processor.updateAnnotations(service, &repopulatedEndpoint, nil, updateSnapshot)
cleared := annotationUpdate{}
repopulated := annotationUpdate{endpoint: family.endpoint}
if !enableEndpoints {
if family.egressIPv6 {
cleared = annotationUpdate{endpoint: family.other}
repopulated = annotationUpdate{endpoint: family.other, endpointIPv6: family.endpoint}
} else {
cleared = annotationUpdate{endpointIPv6: family.other}
repopulated = annotationUpdate{endpoint: family.endpoint, endpointIPv6: family.other}
}
}
want := []annotationUpdate{cleared, repopulated}
if len(recorder.updates) != len(want) {
t.Fatalf("annotation updates = %+v, want %+v", recorder.updates, want)
}
for index := range want {
if recorder.updates[index] != want[index] {
t.Errorf("annotation update %d = %+v, want %+v", index, recorder.updates[index], want[index])
}
}
})
}
}
}
func TestUpdateAnnotationsEndpointSlicesClearsConfiguredFamily(t *testing.T) {
for _, test := range []struct {
name string
egressIPv6 bool
want annotationUpdate
}{
{name: "IPv4", want: annotationUpdate{endpointIPv6: "fd00::1"}},
{name: "IPv6", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1"}},
} {
t.Run(test.name, func(t *testing.T) {
annotations := map[string]string{
kubevip.Egress: "true",
kubevip.ActiveEndpoint: "10.0.0.1",
kubevip.ActiveEndpointIPv6: "fd00::1",
}
if test.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
}}
instances := []*instance.Instance{{ServiceSnapshot: service.DeepCopy()}}
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: false},
provider: recorder,
instances: &instances,
}
noEndpoint := ""
processor.updateAnnotations(service, &noEndpoint, nil, func(context.Context, *v1.Service) error { return nil })
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
}
})
}
}
func TestUpdateAnnotationsValidatesEndpointFamily(t *testing.T) {
tests := []struct {
name string
endpoint string
egressIPv6 bool
want annotationUpdate
wantUpdate bool
}{
{name: "invalid address", endpoint: "not-an-ip"},
{name: "IPv6 endpoint for IPv4 egress", endpoint: "fd00::1"},
{name: "IPv4 endpoint for IPv6 egress", endpoint: "10.0.0.1", egressIPv6: true},
{name: "IPv4 endpoint", endpoint: "10.0.0.2", want: annotationUpdate{endpoint: "10.0.0.2", endpointIPv6: "fd00::1"}, wantUpdate: true},
{name: "IPv6 endpoint", endpoint: "fd00::2", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1", endpointIPv6: "fd00::2"}, wantUpdate: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
annotations := map[string]string{
kubevip.Egress: "true",
kubevip.ActiveEndpoint: "10.0.0.1",
kubevip.ActiveEndpointIPv6: "fd00::1",
}
if test.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", Annotations: annotations,
}}
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: false},
provider: recorder,
}
processor.updateAnnotations(service, &test.endpoint, nil, nil)
if !test.wantUpdate {
if len(recorder.updates) != 0 {
t.Fatalf("annotation updates = %+v, want none", recorder.updates)
}
return
}
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
}
})
}
}
func (f *fakeWorker) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
f.processCalled = true
return nil
@@ -58,14 +245,101 @@ func (f *fakeWorker) clear(_ *servicecontext.Context, _ *string, _ *v1.Service)
func (f *fakeWorker) getEndpoints(_ *v1.Service, _ string) ([]string, error) { return f.endpoints, nil }
func (f *fakeWorker) removeEgress(_ *v1.Service, _ *string) {}
func (f *fakeWorker) delete(_ context.Context, _ *v1.Service, _ string) error {
return nil
}
func (f *fakeWorker) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
return nil
}
func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
// TestReconcile_RecomputesRemainingEndpoints asserts that deleting one EndpointSlice
// reconciles against the endpoints that remain, instead of assuming the service
// lost all of them.
func TestReconcile_RecomputesRemainingEndpoints(t *testing.T) {
t.Parallel()
tests := []struct {
name string
remaining []string
lastKnown string
expectReady bool
expectClear bool
expectProcess bool
expectedLastKnown string
}{
{
name: "remaining endpoints keep the service up",
remaining: []string{"10.0.0.2"},
lastKnown: "10.0.0.2",
expectReady: true,
expectProcess: true,
expectedLastKnown: "10.0.0.2",
},
{
name: "stale last known endpoint moves to a survivor",
remaining: []string{"10.0.0.2"},
lastKnown: "10.0.0.1",
expectReady: true,
expectProcess: true,
expectedLastKnown: "10.0.0.2",
},
{
name: "last endpoint removed tears the service down",
remaining: nil,
lastKnown: "10.0.0.1",
expectReady: false,
expectClear: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
worker := &fakeWorker{endpoints: test.remaining}
p := &Processor{
config: &kubevip.Config{},
provider: providers.NewEndpointslices(),
worker: worker,
}
svcCtx := servicecontext.New(context.Background())
svcCtx.SignalReadiness()
lastKnown := test.lastKnown
restart, err := p.Reconcile(
svcCtx,
watch.Event{
Type: watch.Deleted,
Object: &discoveryv1.EndpointSlice{ObjectMeta: metav1.ObjectMeta{Name: "slice-1"}},
},
&lastKnown,
&v1.Service{Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal}},
"node-1",
func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error { return nil },
&sync.WaitGroup{},
nil,
nil,
)
if err != nil {
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("Reconcile unexpectedly requested restart")
}
if ready := svcCtx.Signalled.Load(); ready != test.expectReady {
t.Fatalf("readiness mismatch: expected %v, got %v", test.expectReady, ready)
}
if worker.clearCalled != test.expectClear {
t.Fatalf("clearCalled mismatch: expected %v, got %v", test.expectClear, worker.clearCalled)
}
if worker.processCalled != test.expectProcess {
t.Fatalf("processCalled mismatch: expected %v, got %v", test.expectProcess, worker.processCalled)
}
if test.expectedLastKnown != "" && lastKnown != test.expectedLastKnown {
t.Fatalf("lastKnownGoodEndpoint mismatch: expected %q, got %q", test.expectedLastKnown, lastKnown)
}
})
}
}
func TestReconcile_ZeroEndpointsBehavior(t *testing.T) {
t.Parallel()
run := func(t *testing.T, service *v1.Service, presetSignalled bool, expectReady bool, expectClear bool, expectProcess bool) {
@@ -83,7 +357,7 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
svcCtx.SignalReadiness()
}
restart, err := p.AddOrModify(
restart, err := p.Reconcile(
svcCtx,
watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
new(string),
@@ -95,10 +369,10 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
nil,
)
if err != nil {
t.Fatalf("AddOrModify returned error: %v", err)
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("AddOrModify unexpectedly requested restart")
t.Fatal("Reconcile unexpectedly requested restart")
}
if ready := svcCtx.Signalled.Load(); ready != expectReady {
@@ -137,15 +411,15 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
})
}
// TestAddOrModify_ServicesElectionStartsOnce asserts that repeated endpoint events
// TestReconcile_ServicesElectionStartsOnce asserts that repeated endpoint events
// for the same service start the leader-election restart loop exactly once.
//
// AddOrModify runs on every EndpointSlice add/modify/resync event, and the loop it
// Reconcile runs on every EndpointSlice add/modify/resync event, and the loop it
// starts only returns once the service context is cancelled. Starting it per event
// therefore accumulates duplicate goroutines that all contend on the same lease.
//
// See https://github.com/kube-vip/kube-vip/issues/1665.
func TestAddOrModify_ServicesElectionStartsOnce(t *testing.T) {
func TestReconcile_ServicesElectionStartsOnce(t *testing.T) {
config := &kubevip.Config{
EnableServicesElection: true,
LeaderElectionType: "kubernetes",
@@ -189,13 +463,13 @@ func TestAddOrModify_ServicesElectionStartsOnce(t *testing.T) {
// Three endpoint events, as a flapping backend pod would produce.
for range 3 {
restart, err := p.AddOrModify(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
restart, err := p.Reconcile(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
new(string), service, "node-1", serviceFunc, wg, nil, nil)
if err != nil {
t.Fatalf("AddOrModify returned error: %v", err)
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("AddOrModify unexpectedly requested restart")
t.Fatal("Reconcile unexpectedly requested restart")
}
}

View File

@@ -212,8 +212,8 @@ func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEnd
}
}
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
if svcCtx != nil {
svcCtx.CallLeaderCancel()
}
}
@@ -242,14 +242,6 @@ func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoin
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
}
// delete removes all DNAT rules for a service
func (w *wireguardWorker) delete(ctx context.Context, service *v1.Service, id string) error {
log.Info("[wireguard] deleting DNAT rules for service", "service", service.Name, "namespace", service.Namespace)
w.clear(nil, nil, service)
return nil
}
// setInstanceEndpointsStatus updates the endpoint status on the service instance
func (w *wireguardWorker) setInstanceEndpointsStatus(_ context.Context, service *v1.Service, endpoints []string) error {
hasEndpoints := len(endpoints) > 0

View File

@@ -0,0 +1,14 @@
package endpoints
import (
"testing"
v1 "k8s.io/api/core/v1"
)
func TestWireguardClearDoesNotDereferenceNilServiceContext(t *testing.T) {
worker := &wireguardWorker{}
service := &v1.Service{}
worker.clear(nil, nil, service)
}

View File

@@ -61,6 +61,18 @@ func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelF
return nil
}
// DeleteObject drops the tracked object. A service is backed by exactly one
// v1.Endpoints object, so there is nothing to match on and the cache is reset.
func (ep *Endpoints) DeleteObject(endpoints runtime.Object) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
if _, ok := endpoints.(*v1.Endpoints); !ok {
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
}
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
ep.endpoints = &v1.Endpoints{}
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
@@ -87,7 +99,7 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
if address.NodeName == nil && id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue

View File

@@ -20,15 +20,14 @@ import (
)
type Endpointslices struct {
label string
endpointsv4 []discoveryv1.Endpoint
endpointsv6 []discoveryv1.Endpoint
ports []discoveryv1.EndpointPort
label string
slices map[string]*discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
label: "endpointslices",
slices: make(map[string]*discoveryv1.EndpointSlice),
}
}
@@ -59,56 +58,71 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
}
if eps.AddressType == discoveryv1.AddressTypeIPv6 {
ep.endpointsv6 = eps.Endpoints
} else {
ep.endpointsv4 = eps.Endpoints
if ep.slices == nil {
ep.slices = make(map[string]*discoveryv1.EndpointSlice)
}
// Store ports for resolving named ports
ep.ports = eps.Ports
ep.slices[eps.Name] = eps.DeepCopy()
return nil
}
func (ep *Endpointslices) DeleteObject(endpoints runtime.Object) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
}
delete(ep.slices, eps.Name)
return nil
}
// isServing reports whether an endpoint should receive traffic. Per the
// EndpointConditions godoc a nil Serving defers to Ready, and a nil Ready is an
// unknown state that consumers should interpret as ready.
func isServing(conditions discoveryv1.EndpointConditions) bool {
serving := conditions.Serving
if serving == nil {
serving = conditions.Ready
}
return serving == nil || *serving
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, e := range ep.endpointsv4 {
result = append(result, e.Addresses...)
}
for _, e := range ep.endpointsv6 {
result = append(result, e.Addresses...)
for _, eps := range ep.slices {
for _, e := range eps.Endpoints {
if !isServing(e.Conditions) {
continue
}
result = append(result, e.Addresses...)
}
}
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
tmpEps := []discoveryv1.Endpoint{}
tmpEps = append(tmpEps, ep.endpointsv4...)
tmpEps = append(tmpEps, ep.endpointsv6...)
for _, endpoint := range tmpEps {
if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
} else {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
}
localEndpoints = append(localEndpoints, address)
for _, eps := range ep.slices {
for _, endpoint := range eps.Endpoints {
if !isServing(endpoint.Conditions) {
continue
}
for _, address := range endpoint.Addresses {
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
} else {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
}
localEndpoints = append(localEndpoints, address)
continue
}
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
if endpoint.Hostname != nil && id == *endpoint.Hostname {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
localEndpoints = append(localEndpoints, address)
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
if endpoint.NodeName == nil && endpoint.Hostname != nil && id == *endpoint.Hostname {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
localEndpoints = append(localEndpoints, address)
}
}
}
}
@@ -153,9 +167,11 @@ func (ep *Endpointslices) GetLabel() string {
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, p := range ep.ports {
if p.Name != nil && *p.Name == name && p.Port != nil {
return *p.Port
for _, eps := range ep.slices {
for _, p := range eps.Ports {
if p.Name != nil && *p.Name == name && p.Port != nil {
return *p.Port
}
}
}
return 0

View File

@@ -0,0 +1,149 @@
package providers
import (
"context"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestEndpointslicesTracksAndDeletesSlices(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
serving := true
nodeName := "node-1"
slice1 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
NodeName: &nodeName,
}},
}
slice2 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-2"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.2"},
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
NodeName: &nodeName,
}},
}
for _, slice := range []*discoveryv1.EndpointSlice{slice1, slice2} {
if err := provider.LoadObject(slice, func() {}); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
}
assertEndpoints(t, provider, []string{"10.0.0.1", "10.0.0.2"})
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.1", "10.0.0.2"})
if err := provider.DeleteObject(slice1); err != nil {
t.Fatalf("DeleteObject returned error: %v", err)
}
assertEndpoints(t, provider, []string{"10.0.0.2"})
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.2"})
if err := provider.DeleteObject(slice2); err != nil {
t.Fatalf("DeleteObject returned error: %v", err)
}
assertEndpoints(t, provider, nil)
assertLocalEndpoints(t, provider, nodeName, nil)
}
func TestEndpointslicesReplacingSliceUpdatesState(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
first := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.1"}}},
}
replacement := first.DeepCopy()
replacement.Endpoints[0].Addresses = []string{"10.0.0.2"}
if err := provider.LoadObject(first, context.CancelFunc(func() {})); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
if err := provider.LoadObject(replacement, context.CancelFunc(func() {})); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
assertEndpoints(t, provider, []string{"10.0.0.2"})
}
func TestEndpointslicesEndpointConditions(t *testing.T) {
yes, no := true, false
nodeName := "node-1"
tests := []struct {
name string
conditions discoveryv1.EndpointConditions
want []string
}{
{"serving true", discoveryv1.EndpointConditions{Serving: &yes}, []string{"10.0.0.1"}},
{"serving false", discoveryv1.EndpointConditions{Serving: &no}, nil},
{"serving false overrides ready true", discoveryv1.EndpointConditions{Serving: &no, Ready: &yes}, nil},
{"nil serving defers to ready true", discoveryv1.EndpointConditions{Ready: &yes}, []string{"10.0.0.1"}},
{"nil serving defers to ready false", discoveryv1.EndpointConditions{Ready: &no}, nil},
{"both nil is treated as ready", discoveryv1.EndpointConditions{}, []string{"10.0.0.1"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
slice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Conditions: test.conditions,
NodeName: &nodeName,
}},
}
if err := provider.LoadObject(slice, func() {}); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
// Cluster and Local policy have to agree on which endpoints are usable.
assertEndpoints(t, provider, test.want)
assertLocalEndpoints(t, provider, nodeName, test.want)
})
}
}
func assertEndpoints(t *testing.T, provider *Endpointslices, want []string) {
t.Helper()
got, err := provider.GetAllEndpoints()
if err != nil {
t.Fatalf("GetAllEndpoints returned error: %v", err)
}
assertStringSet(t, got, want)
}
func assertLocalEndpoints(t *testing.T, provider *Endpointslices, nodeName string, want []string) {
t.Helper()
got, err := provider.GetLocalEndpoints(nodeName, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints returned error: %v", err)
}
assertStringSet(t, got, want)
}
func assertStringSet(t *testing.T, got, want []string) {
t.Helper()
counts := map[string]int{}
for _, value := range got {
counts[value]++
}
for _, value := range want {
counts[value]--
}
for value, count := range counts {
if count != 0 {
t.Fatalf("endpoint set mismatch for %q: got %v, want %v", value, got, want)
}
}
}

View File

@@ -18,6 +18,7 @@ type Provider interface {
GetLabel() string
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
DeleteObject(runtime.Object) error
// ResolvePort resolves a service port to the actual target port.
// For named ports, it looks up the port number from the endpoint.
// For numeric ports, it returns the port as-is.

View File

@@ -0,0 +1,319 @@
package providers
import (
"net"
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes/fake"
)
func TestEndpointProvidersParityForLocalAndAllEndpoints(t *testing.T) {
t.Parallel()
nodeA := "node-a"
nodeB := "node-b"
serving := true
v4Addresses := []discoveryv1.Endpoint{
{Addresses: []string{"10.0.0.1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"10.0.0.2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
}
v6Addresses := []discoveryv1.Endpoint{
{Addresses: []string{"2001:db8::1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"2001:db8::2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
}
legacy := NewEndpoints()
//nolint:staticcheck // this test covers the deprecated legacy Endpoints provider on purpose
if err := legacy.LoadObject(&v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{
{IP: "10.0.0.1", NodeName: &nodeA},
{IP: "10.0.0.2", NodeName: &nodeB},
{IP: "2001:db8::1", NodeName: &nodeA},
{IP: "2001:db8::2", NodeName: &nodeB},
},
}},
}, func() {}); err != nil {
t.Fatalf("loading legacy Endpoints: %v", err)
}
slices := NewEndpointslices()
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-v4", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: v4Addresses,
}, func() {}); err != nil {
t.Fatalf("loading IPv4 EndpointSlice: %v", err)
}
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-v6", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv6,
Endpoints: v6Addresses,
}, func() {}); err != nil {
t.Fatalf("loading IPv6 EndpointSlice: %v", err)
}
legacyAll, err := legacy.GetAllEndpoints()
if err != nil {
t.Fatalf("legacy GetAllEndpoints() error = %v", err)
}
sliceAll, err := slices.GetAllEndpoints()
if err != nil {
t.Fatalf("EndpointSlice GetAllEndpoints() error = %v", err)
}
wantAll := endpointSet([]string{"10.0.0.1", "10.0.0.2", "2001:db8::1", "2001:db8::2"})
if got := endpointSet(legacyAll); !reflect.DeepEqual(got, wantAll) {
t.Errorf("legacy all endpoints = %v, want %v", got, wantAll)
}
if got := endpointSet(sliceAll); !reflect.DeepEqual(got, wantAll) {
t.Errorf("EndpointSlice all endpoints = %v, want %v", got, wantAll)
}
legacyLocal, err := legacy.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("legacy GetLocalEndpoints() error = %v", err)
}
sliceLocal, err := slices.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("EndpointSlice GetLocalEndpoints() error = %v", err)
}
wantLocal := endpointSet([]string{"10.0.0.1", "2001:db8::1"})
if got := endpointSet(legacyLocal); !reflect.DeepEqual(got, wantLocal) {
t.Errorf("legacy local endpoints = %v, want %v", got, wantLocal)
}
if got := endpointSet(sliceLocal); !reflect.DeepEqual(got, wantLocal) {
t.Errorf("EndpointSlice local endpoints = %v, want %v", got, wantLocal)
}
assertEndpointFamilies(t, legacyAll, 2, 2)
assertEndpointFamilies(t, sliceAll, 2, 2)
}
func TestEndpointSlicesLocalFilteringRequiresServingEndpoint(t *testing.T) {
t.Parallel()
node := "node-a"
serving := true
notServing := false
provider := NewEndpointslices()
if err := provider.LoadObject(&discoveryv1.EndpointSlice{
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{
{Addresses: []string{"10.0.0.1"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"10.0.0.2"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &notServing}},
},
}, func() {}); err != nil {
t.Fatalf("loading EndpointSlice: %v", err)
}
local, err := provider.GetLocalEndpoints(node, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints() error = %v", err)
}
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
t.Errorf("local endpoints = %v, want serving endpoints %v", got, want)
}
}
func TestResolvePortFromFakeClientObjects(t *testing.T) {
t.Parallel()
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
legacyObject := &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
Subsets: []v1.EndpointSubset{{
Ports: []v1.EndpointPort{{Name: "web", Port: 8080}},
}},
}
legacyClient := fake.NewSimpleClientset(legacyObject)
legacyLoaded, err := legacyClient.CoreV1().Endpoints("default").Get(t.Context(), "service", metav1.GetOptions{})
if err != nil {
t.Fatalf("getting fake legacy Endpoints: %v", err)
}
legacy := NewEndpoints()
if err := legacy.LoadObject(legacyLoaded, func() {}); err != nil {
t.Fatalf("loading fake legacy Endpoints: %v", err)
}
portName := "web"
port := int32(8081)
sliceObject := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-slice", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv4,
Ports: []discoveryv1.EndpointPort{{Name: &portName, Port: &port}},
}
sliceClient := fake.NewSimpleClientset(sliceObject)
sliceLoaded, err := sliceClient.DiscoveryV1().EndpointSlices("default").Get(t.Context(), "service-slice", metav1.GetOptions{})
if err != nil {
t.Fatalf("getting fake EndpointSlice: %v", err)
}
slices := NewEndpointslices()
if err := slices.LoadObject(sliceLoaded, func() {}); err != nil {
t.Fatalf("loading fake EndpointSlice: %v", err)
}
namedPort := v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")}
if got := legacy.ResolvePort(namedPort); got != 8080 {
t.Errorf("legacy ResolvePort() = %d, want 8080", got)
}
if got := slices.ResolvePort(namedPort); got != 8081 {
t.Errorf("EndpointSlice ResolvePort() = %d, want 8081", got)
}
}
func TestResolvePortWithLookup(t *testing.T) {
t.Parallel()
tests := []struct {
name string
port v1.ServicePort
lookup func(string) int32
want int32
}{
{
name: "numeric target port wins",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromInt(8080)},
lookup: func(string) int32 { return 9090 },
want: 8080,
},
{
name: "named target port is looked up",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")},
lookup: func(name string) int32 {
if name == "web" {
return 8081
}
return 0
},
want: 8081,
},
{
name: "missing named target falls back to service port",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("missing")},
lookup: func(string) int32 { return 0 },
want: 80,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ResolvePortWithLookup(tt.port, tt.lookup); got != tt.want {
t.Errorf("ResolvePortWithLookup() = %d, want %d", got, tt.want)
}
})
}
}
func endpointSet(endpoints []string) map[string]struct{} {
result := make(map[string]struct{}, len(endpoints))
for _, endpoint := range endpoints {
result[endpoint] = struct{}{}
}
return result
}
func assertEndpointFamilies(t *testing.T, endpoints []string, wantIPv4, wantIPv6 int) {
t.Helper()
ipv4, ipv6 := 0, 0
for _, endpoint := range endpoints {
ip := net.ParseIP(endpoint)
if ip == nil {
t.Errorf("endpoint %q is not an IP address", endpoint)
continue
}
if ip.To4() != nil {
ipv4++
} else {
ipv6++
}
}
if ipv4 != wantIPv4 || ipv6 != wantIPv6 {
t.Errorf("endpoint families = IPv4 %d, IPv6 %d; want IPv4 %d, IPv6 %d", ipv4, ipv6, wantIPv4, wantIPv6)
}
}
func TestEndpointProvidersPreferNodeNameOverHostname(t *testing.T) {
t.Parallel()
nodeA := "node-a"
nodeB := "node-b"
serving := true
tests := []struct {
name string
load func(Provider) error
}{
{
name: "legacy Endpoints",
load: func(provider Provider) error {
//nolint:staticcheck // the legacy provider is deliberately under test
return provider.LoadObject(&v1.Endpoints{
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: "10.0.0.1",
NodeName: &nodeB,
Hostname: nodeA,
}},
}},
}, func() {})
},
},
{
name: "EndpointSlice",
load: func(provider Provider) error {
hostname := nodeA
return provider.LoadObject(&discoveryv1.EndpointSlice{
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
NodeName: &nodeB,
Hostname: &hostname,
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
}},
}, func() {})
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var provider Provider
if tt.name == "legacy Endpoints" {
provider = NewEndpoints()
} else {
provider = NewEndpointslices()
}
if err := tt.load(provider); err != nil {
t.Fatalf("LoadObject() error = %v", err)
}
local, err := provider.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeA, err)
}
if len(local) != 0 {
t.Fatalf("GetLocalEndpoints(%q) = %v, want no endpoints", nodeA, local)
}
local, err = provider.GetLocalEndpoints(nodeB, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeB, err)
}
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
t.Fatalf("GetLocalEndpoints(%q) = %v, want %v", nodeB, got, want)
}
})
}
}

View File

@@ -15,7 +15,6 @@ import (
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
@@ -28,7 +27,6 @@ const (
)
func TestMain(m *testing.M) {
logrus.SetLevel(logrus.DebugLevel)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
expectSuccess(startEtcd(ctx), "starting etcd")

View File

@@ -203,6 +203,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
@@ -269,6 +270,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
@@ -341,6 +343,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
return nil, err
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("context error while starting DHCPv4 for %s/%s: error: %w",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
case err := <-instance.DHCPv4Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
@@ -356,6 +361,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
return nil, err
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("context error while starting DHCPv6 for %s/%s: error: %w",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
case err := <-instance.DHCPv6Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
@@ -652,7 +660,8 @@ func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uin
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client error: %w")
log.Error("[instance] DHCP client", "error", err)
client.Stop()
}
})

View File

@@ -0,0 +1,62 @@
package instance_test
import (
"context"
"sync"
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/route"
)
func TestNewInstance_PropagatesBGPAttachIPToInterface(t *testing.T) {
tests := []struct {
name string
attach bool
}{
{name: "attach enabled is propagated", attach: true},
{name: "attach disabled is propagated", attach: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
globalConfig := &kubevip.Config{
Interface: "lo",
VIPSubnet: "32",
EnableBGP: true,
BGPAttachIPToInterface: tt.attach,
}
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-svc",
Namespace: "default",
Annotations: map[string]string{
kubevip.LoadbalancerIPAnnotation: "10.0.1.2",
},
},
}
inst, err := instance.NewInstance(context.Background(), svc, globalConfig,
networkinterface.NewManager(), arp.NewManager(globalConfig), route.NewManager(),
nil, &sync.WaitGroup{})
if err != nil {
t.Fatalf("NewInstance() error = %v", err)
}
if len(inst.VIPConfigs) != 1 {
t.Fatalf("VIPConfigs len = %d, want 1", len(inst.VIPConfigs))
}
if got := inst.VIPConfigs[0].BGPAttachIPToInterface; got != tt.attach {
t.Fatalf("BGPAttachIPToInterface = %t, want %t", got, tt.attach)
}
})
}
}

View File

@@ -18,7 +18,6 @@ import (
"bytes"
"fmt"
"io"
"net"
"os/exec"
"regexp"
"strconv"
@@ -87,21 +86,6 @@ type IPTables struct {
nftables bool
}
// Stat represents a structured statistic entry.
type Stat struct {
Packets uint64 `json:"pkts"`
Bytes uint64 `json:"bytes"`
Target string `json:"target"`
Protocol string `json:"prot"`
Opt string `json:"opt"`
Input string `json:"in"`
Output string `json:"out"`
Source *net.IPNet `json:"source"`
Destination *net.IPNet `json:"destination"`
Options string `json:"options"`
}
type Option func(*IPTables)
func IPFamily(proto Protocol) Option {
@@ -251,16 +235,6 @@ func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) err
return err
}
// List rules in specified table/chain
func (ipt *IPTables) ListByID(table, chain string, id int) (string, error) {
args := []string{"-t", table, "-S", chain, strconv.Itoa(id)}
rule, err := ipt.executeList(args)
if err != nil {
return "", err
}
return rule[0], nil
}
// List rules in specified table/chain
func (ipt *IPTables) List(table, chain string) ([]string, error) {
args := []string{"-t", table, "-S", chain}
@@ -313,129 +287,6 @@ func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
}
}
// Stats lists rules including the byte and packet counts
func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
lines, err := ipt.executeList(args)
if err != nil {
return nil, err
}
appendSubnet := func(addr string) string {
if strings.IndexByte(addr, byte('/')) < 0 {
if strings.IndexByte(addr, '.') < 0 {
return addr + "/128"
}
return addr + "/32"
}
return addr
}
ipv6 := ipt.proto == ProtocolIPv6
rows := [][]string{}
for i, line := range lines {
// Skip over chain name and field header
if i < 2 {
continue
}
// Fields:
// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
line = strings.TrimSpace(line)
fields := strings.Fields(line)
// The ip6tables verbose output cannot be naively split due to the default "opt"
// field containing 2 single spaces.
if ipv6 {
// Check if field 6 is "opt" or "source" address
dest := fields[6]
ip, _, _ := net.ParseCIDR(dest)
if ip == nil {
ip = net.ParseIP(dest)
}
// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
if ip != nil {
f := []string{}
f = append(f, fields[:4]...)
f = append(f, " ") // Empty "opt" field for ip6tables
f = append(f, fields[4:]...)
fields = f
}
}
// Adjust "source" and "destination" to include netmask, to match regular
// List output
fields[7] = appendSubnet(fields[7])
fields[8] = appendSubnet(fields[8])
// Combine "options" fields 9... into a single space-delimited field.
options := fields[9:]
fields = fields[:9]
fields = append(fields, strings.Join(options, " "))
rows = append(rows, fields)
}
return rows, nil
}
// ParseStat parses a single statistic row into a Stat struct. The input should
// be a string slice that is returned from calling the Stat method.
func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
// For forward-compatibility, expect at least 10 fields in the stat
if len(stat) < 10 {
return parsed, fmt.Errorf("stat contained fewer fields than expected")
}
// Convert the fields that are not plain strings
parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse packets")
}
parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
}
_, parsed.Source, err = net.ParseCIDR(stat[7])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse source")
}
_, parsed.Destination, err = net.ParseCIDR(stat[8])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse destination")
}
// Put the fields that are strings
parsed.Target = stat[2]
parsed.Protocol = stat[3]
parsed.Opt = stat[4]
parsed.Input = stat[5]
parsed.Output = stat[6]
parsed.Options = stat[9]
return parsed, nil
}
// StructuredStats returns statistics as structured data which may be further
// parsed and marshaled.
func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
rawStats, err := ipt.Stats(table, chain)
if err != nil {
return nil, err
}
structStats := []Stat{}
for _, rawStat := range rawStats {
stat, err := ipt.ParseStat(rawStat)
if err != nil {
return nil, err
}
structStats = append(structStats, stat)
}
return structStats, nil
}
func (ipt *IPTables) executeList(args []string) ([]string, error) {
var stdout bytes.Buffer
if err := ipt.runWithOutput(args, &stdout); err != nil {

View File

@@ -77,6 +77,9 @@ const (
// Name of the service lease object
ServiceLease = "kube-vip.io/leaseName"
// Versioned kube-vip ownership metadata stored on Kubernetes election Leases
LeaseVIPs = "kube-vip.io/lease-vips"
// Forces kube-vip to use per service election for this particular service
ForcePerServiceElection = "kube-vip.io/forcePerServiceElection"

View File

@@ -2,7 +2,6 @@ package kubevip
import (
"fmt"
"net"
"strconv"
"strings"
@@ -260,12 +259,12 @@ func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, s
}
if ipv4 != "" {
if net.ParseIP(ipv4) == nil {
if !utils.IsIPv4(ipv4) {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", ipv4)
}
}
if ipv6 != "" {
if net.ParseIP(ipv6) == nil {
if !utils.IsIPv6(ipv6) {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", ipv6)
}
}

View File

@@ -0,0 +1,38 @@
package kubevip
import (
"testing"
api "github.com/osrg/gobgp/v4/api"
)
func TestFindMpbgpAddressesRejectsFixedAddressFamilyMismatches(t *testing.T) {
tests := []struct {
name string
peer BGPPeer
}{
{
name: "IPv6 value in IPv4 field",
peer: BGPPeer{
MpbgpNexthop: "fixed",
MpbgpIPv4: "2001:db8::20",
},
},
{
name: "IPv4 value in IPv6 field",
peer: BGPPeer{
MpbgpNexthop: "fixed",
MpbgpIPv6: "192.0.2.20",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, _, err := tt.peer.FindMpbgpAddresses(&api.Peer{Transport: &api.Transport{}}, &BGPConfig{})
if err == nil {
t.Fatal("FindMpbgpAddresses() error = nil, want address-family error")
}
})
}
}

View File

@@ -125,6 +125,13 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
},
},
}
if !role {
newManifest.Rules = append(newManifest.Rules, applyRbacV1.PolicyRuleApplyConfiguration{
APIGroups: []string{"networking.k8s.io"},
Resources: []string{"servicecidrs"},
Verbs: []string{"list", "get", "watch"},
})
}
return newManifest
}

View File

@@ -2,10 +2,38 @@ package kubevip
import (
"os"
"slices"
"strings"
"testing"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
)
func TestGenerateRoleServiceCIDRAccess(t *testing.T) {
clusterRole := GenerateRole(&Config{}, false)
if !hasServiceCIDRRule(clusterRole) {
t.Fatal("generated ClusterRole is missing ServiceCIDR access")
}
role := GenerateRole(&Config{ServiceNamespace: "kube-vip"}, true)
if hasServiceCIDRRule(role) {
t.Fatal("generated namespaced Role contains ineffective ServiceCIDR access")
}
}
func hasServiceCIDRRule(role *applyRbacV1.RoleApplyConfiguration) bool {
for _, rule := range role.Rules {
if slices.Contains(rule.APIGroups, "networking.k8s.io") &&
slices.Contains(rule.Resources, "servicecidrs") &&
slices.Contains(rule.Verbs, "get") &&
slices.Contains(rule.Verbs, "list") &&
slices.Contains(rule.Verbs, "watch") {
return true
}
}
return false
}
func TestParseEnvironment(t *testing.T) {
tests := []struct {

View File

@@ -0,0 +1,134 @@
package kubevip
import (
"encoding/json"
"fmt"
"net/netip"
"slices"
"strings"
)
const LeaseVIPsVersion = "v1"
// LeaseVIPKind distinguishes literal addresses from names that resolve to one.
type LeaseVIPKind string
const (
LeaseVIPKindAddress LeaseVIPKind = "address"
LeaseVIPKindName LeaseVIPKind = "name"
)
type LeaseVIPsValue struct {
Version string `json:"version"`
InstanceName string `json:"instance_name"`
IFAProto int `json:"ifa_proto"`
VIPs []LeaseVIP `json:"vips"`
}
type LeaseVIP struct {
Index int `json:"index"`
Value string `json:"value"`
Kind LeaseVIPKind `json:"kind"`
}
func WithLeaseVIPs(annotations map[string]string, instanceName string, ifaProto int, vips []string) (map[string]string, error) {
result := make(map[string]string, len(annotations)+1)
for key, value := range annotations {
result[key] = value
}
encoded, err := json.Marshal(LeaseVIPsValue{
Version: LeaseVIPsVersion,
InstanceName: instanceName,
IFAProto: ifaProto,
VIPs: normalizeLeaseVIPs(vips),
})
if err != nil {
return nil, fmt.Errorf("encode %s annotation: %w", LeaseVIPs, err)
}
result[LeaseVIPs] = string(encoded)
return result, nil
}
func ParseLeaseVIPs(value string) (LeaseVIPsValue, error) {
var parsed LeaseVIPsValue
if err := json.Unmarshal([]byte(value), &parsed); err != nil {
return LeaseVIPsValue{}, fmt.Errorf("decode %s annotation: %w", LeaseVIPs, err)
}
if parsed.Version != LeaseVIPsVersion {
return LeaseVIPsValue{}, fmt.Errorf("unsupported %s annotation version %q", LeaseVIPs, parsed.Version)
}
for index, vip := range parsed.VIPs {
if vip.Index != index {
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP index %d at position %d", LeaseVIPs, vip.Index, index)
}
switch vip.Kind {
case LeaseVIPKindAddress, LeaseVIPKindName:
default:
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP kind %q at index %d", LeaseVIPs, vip.Kind, vip.Index)
}
}
return parsed, nil
}
func normalizeLeaseVIPs(values []string) []LeaseVIP {
unique := make(map[string]struct{}, len(values))
addresses := make([]string, 0, len(values))
for _, value := range values {
for candidate := range strings.SplitSeq(value, ",") {
candidate = strings.TrimSpace(candidate)
if candidate == "" {
continue
}
if _, exists := unique[candidate]; exists {
continue
}
unique[candidate] = struct{}{}
addresses = append(addresses, candidate)
}
}
// Sorting keeps the annotation byte-identical however callers happen to order VIPs.
slices.SortFunc(addresses, compareLeaseVIPs)
result := make([]LeaseVIP, 0, len(addresses))
for _, address := range addresses {
kind := LeaseVIPKindName
if _, isAddress := leaseVIPAddress(address); isAddress {
kind = LeaseVIPKindAddress
}
result = append(result, LeaseVIP{Index: len(result), Value: address, Kind: kind})
}
return result
}
// compareLeaseVIPs orders addresses numerically and ahead of names, which keeps VIPs like
// 10.0.0.2 and 10.0.0.10 in the order an operator expects. Values that are not addresses,
// such as DNS records, are kept and ordered lexically.
func compareLeaseVIPs(a, b string) int {
addressA, isAddressA := leaseVIPAddress(a)
addressB, isAddressB := leaseVIPAddress(b)
switch {
case isAddressA && isAddressB:
if order := addressA.Compare(addressB); order != 0 {
return order
}
// Distinct spellings of one address still need a stable order.
return strings.Compare(a, b)
case isAddressA:
return -1
case isAddressB:
return 1
default:
return strings.Compare(a, b)
}
}
func leaseVIPAddress(value string) (netip.Addr, bool) {
if address, err := netip.ParseAddr(value); err == nil {
return address.Unmap(), true
}
if prefix, err := netip.ParsePrefix(value); err == nil {
return prefix.Addr().Unmap(), true
}
return netip.Addr{}, false
}

View File

@@ -0,0 +1,94 @@
package kubevip
import (
"slices"
"testing"
)
func TestWithLeaseVIPsEncodesVersionedInstanceOwnership(t *testing.T) {
base := map[string]string{"example.test/preserved": "true", LeaseVIPs: "stale"}
annotations, err := WithLeaseVIPs(base, "release_a", 248, []string{
"2001:db8::10/128", "192.0.2.10", "192.0.2.10/32", "api.example.test",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
if annotations["example.test/preserved"] != "true" {
t.Fatal("WithLeaseVIPs() dropped an existing annotation")
}
if base[LeaseVIPs] != "stale" {
t.Fatal("WithLeaseVIPs() mutated the input annotations")
}
value, err := ParseLeaseVIPs(annotations[LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
if value.Version != LeaseVIPsVersion || value.InstanceName != "release_a" || value.IFAProto != 248 {
t.Fatalf("Lease VIP metadata = %+v", value)
}
// Values are stored verbatim so DNS records survive alongside addresses.
want := []LeaseVIP{
{Index: 0, Value: "192.0.2.10", Kind: LeaseVIPKindAddress},
{Index: 1, Value: "192.0.2.10/32", Kind: LeaseVIPKindAddress},
{Index: 2, Value: "2001:db8::10/128", Kind: LeaseVIPKindAddress},
{Index: 3, Value: "api.example.test", Kind: LeaseVIPKindName},
}
if !slices.Equal(value.VIPs, want) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
}
// The annotation is rewritten whenever a node starts campaigning, so the encoding
// has to be stable even when callers collect the same VIPs in a different order.
func TestWithLeaseVIPsIsIndependentOfInputOrder(t *testing.T) {
first, err := WithLeaseVIPs(nil, "release_a", 248, []string{
"2001:db8::10", "192.0.2.10", "10.0.0.2", "10.0.0.10",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
second, err := WithLeaseVIPs(nil, "release_a", 248, []string{
"10.0.0.10", "192.0.2.10", "2001:db8::10", "10.0.0.2",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
if first[LeaseVIPs] != second[LeaseVIPs] {
t.Fatalf("annotation changed with input order:\n%s\n%s", first[LeaseVIPs], second[LeaseVIPs])
}
value, err := ParseLeaseVIPs(first[LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
want := []string{"10.0.0.2", "10.0.0.10", "192.0.2.10", "2001:db8::10"}
if len(value.VIPs) != len(want) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
for index, address := range want {
if value.VIPs[index] != (LeaseVIP{Index: index, Value: address, Kind: LeaseVIPKindAddress}) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
}
}
func TestParseLeaseVIPsRejectsUnknownVersion(t *testing.T) {
if _, err := ParseLeaseVIPs(`{"version":"v2","instance_name":"release_a","ifa_proto":248,"vips":[]}`); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an unknown version")
}
}
func TestParseLeaseVIPsRejectsUnknownKind(t *testing.T) {
if _, err := ParseLeaseVIPs(
`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":0,"value":"192.0.2.10","kind":"cidr"}]}`,
); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an unknown VIP kind")
}
}
func TestParseLeaseVIPsRejectsOutOfOrderIndexes(t *testing.T) {
if _, err := ParseLeaseVIPs(`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":1,"value":"192.0.2.10"}]}`); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an out-of-order VIP index")
}
}

View File

@@ -240,7 +240,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
// Fatal error at this point as IPVS is probably not working
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Error("IPVS service", "err", err)
return utils.NewPanicError(fmt.Sprintf("unable to create an IPVS service - %s", err))
return utils.WrapPanicError(err, "unable to create an IPVS service")
}
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)

View File

@@ -2,7 +2,6 @@ package manager
import (
"context"
"errors"
"fmt"
"net/http"
"os"
@@ -33,7 +32,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -58,10 +56,6 @@ type Manager struct {
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
// This mutex is to protect calls from various goroutines
mutex sync.Mutex
@@ -260,16 +254,10 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
intfMgr, arpMgr, nodeLabelManager, electionMgr, leaseMgr, routeMgr)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
configMap: configMap,
config: config,
countServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
clientSet: clientset,
rwClientSet: rwClientSet,
configMap: configMap,
config: config,
signalChan: signalChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
@@ -414,7 +402,7 @@ func (sm *Manager) startMode(ctx context.Context) error {
return nil
default:
if err = w.StartServices(modeCtx); err != nil {
if errors.Is(err, &utils.PanicError{}) {
if utils.IsPanicError(err) {
sm.Kill()
return fmt.Errorf("failed to reconcile services, non-recoverable error: %w", err)
} else {

View File

@@ -11,10 +11,9 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
@@ -42,6 +41,9 @@ func annotationsWatcher(ctx context.Context, clientSet,
if err != nil {
return err
}
if len(nodeList.Items) == 0 {
return fmt.Errorf("no node found with hostname %q", config.NodeName)
}
// We'll assume there's only one node with the hostname annotation. If that's not true,
// there's probably bigger problems
@@ -108,22 +110,15 @@ func annotationsWatcher(ctx context.Context, clientSet,
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error(status.String())
log.Error("annotations watcher failed", "err", utils.WatchError(event.Object))
default:
}
}
log.Info("[annotations] exiting annotations watcher")
return nil
if ctx.Err() != nil {
return nil
}
return utils.NewPanicError("annotations watcher channel closed unexpectedly")
}
// parseNodeAnnotations parses the annotations on the node and updates the configuration

View File

@@ -0,0 +1,21 @@
package manager
import (
"context"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"k8s.io/client-go/kubernetes/fake"
)
func TestAnnotationsWatcherHandlesEmptyNodeList(t *testing.T) {
client := fake.NewSimpleClientset()
config := &kubevip.Config{
NodeName: "node-a",
Annotations: "kube-vip.io",
}
if err := annotationsWatcher(context.Background(), client, client, config); err == nil {
t.Fatal("annotationsWatcher() error = nil, want empty node-list error")
}
}

View File

@@ -113,6 +113,10 @@ func (b *BGP) StartServices(ctx context.Context) error {
if err := b.PerServiceLeader(ctx, false); err != nil {
return err
}
} else if b.config.EnableLeaderElection {
log.Warn("leader election is enabled, only the elected leader will advertise service VIPs; unset enable_leader_election to keep advertising from every node (ECMP)",
"lease", b.config.ServicesLeaseName)
b.GlobalLeader(ctx, b.config.ServicesLeaseName)
} else {
if err := b.ServicesNoLeader(ctx); err != nil {
return err
@@ -121,10 +125,6 @@ func (b *BGP) StartServices(ctx context.Context) error {
return nil
}
func (b *BGP) ServicesGlobalLeader(ctx context.Context, id string) {
// NOT IMPLEMENTED
}
func (b *BGP) Name() string {
return "BGP"
}

View File

@@ -43,7 +43,11 @@ func (t *Table) Configure(ctx context.Context, wg *sync.WaitGroup) error {
if t.config.CleanRoutingTable {
wg.Go(func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
select {
case <-ctx.Done():
return
case <-time.After(10 * time.Second):
}
if err := t.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
}

View File

@@ -0,0 +1,47 @@
package worker
import (
"context"
"sync"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestConfigureCleanRoutingTableStopsWithContext(t *testing.T) {
t.Parallel()
var wg sync.WaitGroup
table := &Table{Common: Common{
config: &kubevip.Config{
CleanRoutingTable: true,
EnableControlPlane: true,
Address: "10.254.254.254",
RoutingTableID: 0x7fffffff,
RoutingProtocol: 255,
},
mutex: &sync.Mutex{},
}}
ctx, cancel := context.WithCancel(context.Background())
if err := table.Configure(ctx, &wg); err != nil {
t.Fatalf("Configure returned an error: %v", err)
}
cancel()
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(500 * time.Millisecond):
// DEFECT: pkg/manager/worker/table.go:43-48 uses an unconditional
// 10-second sleep for cleanRoutingTable and ignores the canceled RT
// worker context, delaying shutdown/recovery.
t.Fatal("cleanRoutingTable worker did not stop after context cancellation")
}
}

View File

@@ -102,12 +102,15 @@ func (w *WireGuard) ConfigureServices() {
}
func (w *WireGuard) StartServices(ctx context.Context) error {
// WireGuard has no multipath mechanism, so every service must be advertised by
// exactly one node: leader election (per-service or global) is required.
if w.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx, false)
if err != nil {
if err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx, false); err != nil {
return err
}
} else {
w.GlobalLeader(ctx, w.config.ServicesLeaseName)
}
return nil
}
@@ -147,12 +150,6 @@ func (w *WireGuard) OnStartedLeading(ctx context.Context) {
w.endpointWatcherWg.Go(func() {
w.watchKubernetesEndpoints(w.endpointWatcherCtx, tunnelConfig)
})
if w.config.EnableServices && !w.config.EnableServicesElection {
if err := w.svcProcessor.ServicesWatcher(ctx, services.NewCallback(w.svcProcessor.SyncServices, false), false); err != nil {
log.Error("failed to start services watcher", "err", err)
}
}
}
// watchKubernetesEndpoints watches the kubernetes service EndpointSlices for changes
@@ -185,8 +182,15 @@ func (w *WireGuard) watchKubernetesEndpoints(ctx context.Context, tunnelConfig *
switch event.Type {
case watch.Added, watch.Modified, watch.Deleted:
if err := provider.LoadObject(event.Object, func() {}); err != nil {
log.Error("failed to load endpoint object", "err", err)
// A deleted slice has to be dropped so it stops counting toward the endpoint set.
var err error
if event.Type == watch.Deleted {
err = provider.DeleteObject(event.Object)
} else {
err = provider.LoadObject(event.Object, func() {})
}
if err != nil {
log.Error("failed to update endpoint object", "eventType", event.Type, "err", err)
continue
}
endpoints, _ := provider.GetAllEndpoints()

View File

@@ -1,6 +1,10 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
)
var (
// Service / VIP Lifecycle
@@ -68,19 +72,25 @@ var (
)
)
var registerOnce sync.Once
// RegisterPrometheusMetrics registers all kube-vip metrics with the default
// Prometheus registry.
func RegisterPrometheusMetrics() {
// Register all metrics with Prometheus
prometheus.MustRegister(
ActiveServices,
ServiceReconcileErrorsTotal,
ServiceReconcileDuration,
LeaderTransitionsTotal,
IsLeader,
ServiceElectionLoops,
ServiceElectionAttemptsTotal,
ServiceElectionErrorsTotal,
BGPSessionInfoGauge,
BuildInfo,
CountServiceWatchEvent,
)
registerOnce.Do(func() {
// Register all metrics with Prometheus
prometheus.MustRegister(
ActiveServices,
ServiceReconcileErrorsTotal,
ServiceReconcileDuration,
LeaderTransitionsTotal,
IsLeader,
ServiceElectionLoops,
ServiceElectionAttemptsTotal,
ServiceElectionErrorsTotal,
BGPSessionInfoGauge,
BuildInfo,
CountServiceWatchEvent,
)
})
}

94
pkg/metrics/server.go Normal file
View File

@@ -0,0 +1,94 @@
package metrics
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"sync"
"time"
log "log/slog"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// shutdownTimeout bounds how long the server waits for in-flight requests to
// finish once the context is cancelled.
const shutdownTimeout = 5 * time.Second
// ServerConfig defines the Prometheus server configuration.
type ServerConfig struct {
// Addr sets the http server address used to expose the metric endpoint
Addr string
}
// Serve exposes the Prometheus metrics endpoint on the configured address.
func Serve(ctx context.Context, config ServerConfig) error {
ln, err := net.Listen("tcp", config.Addr)
if err != nil {
return fmt.Errorf("listening on %q: %w", config.Addr, err)
}
return serve(ctx, ln)
}
// serve starts the metrics endpoint on the provided listener
func serve(ctx context.Context, ln net.Listener) error {
srv := &http.Server{
Handler: newServeMux(),
ReadHeaderTimeout: 2 * time.Second,
}
wg := sync.WaitGroup{}
defer wg.Wait()
serveErr := make(chan error, 1)
wg.Go(func() {
err := srv.Serve(ln)
if errors.Is(err, http.ErrServerClosed) {
err = nil
}
serveErr <- err
})
log.Info("prometheus HTTP server started", "addr", ln.Addr().String())
select {
case err := <-serveErr:
if err != nil {
return fmt.Errorf("serving prometheus metrics: %w", err)
}
return nil
case <-ctx.Done():
}
// create prometheus shutdown context (independent of other contexts)
ctxShutDown, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
if err := srv.Shutdown(ctxShutDown); err != nil {
return fmt.Errorf("shutting down prometheus HTTP server: %w", err)
}
log.Info("prometheus HTTP server stopped")
return nil
}
func newServeMux() *http.ServeMux {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`<html>
<head><title>kube-vip</title></head>
<body>
<h1>kube-vip Metrics</h1>
<p><a href="/metrics">Metrics</a></p>
</body>
</html>`))
})
return mux
}

213
pkg/metrics/server_test.go Normal file
View File

@@ -0,0 +1,213 @@
package metrics
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
)
func TestServeExposesKubeVipMetrics(t *testing.T) {
RegisterPrometheusMetrics()
version, build, node := "v1.2.3", "test-build", "node-1"
BuildInfo.WithLabelValues(version, build, node)
base, stop := startServer(t, newTestListener(t))
body, code := get(t, base+"/metrics")
if code != http.StatusOK {
t.Fatalf("GET /metrics status = %d, want %d", code, http.StatusOK)
}
// Label names are exposed in alphabetical order.
want := fmt.Sprintf("kube_vip_build_info{build=\"%s\",node=\"%s\",version=\"%s\"}", build, node, version)
if !strings.Contains(body, want) {
t.Errorf("GET /metrics body does not contain %s, got:\n%s", want, body)
}
if err := stop(); err != nil {
t.Errorf("serve returned an error on shutdown: %v", err)
}
}
func TestServeRootPageLinksToMetrics(t *testing.T) {
base, stop := startServer(t, newTestListener(t))
body, code := get(t, base+"/")
if code != http.StatusOK {
t.Fatalf("GET / status = %d, want %d", code, http.StatusOK)
}
if !strings.Contains(body, `href="/metrics"`) {
t.Errorf("GET / body does not link to /metrics, got:\n%s", body)
}
if err := stop(); err != nil {
t.Errorf("serve returned an error on shutdown: %v", err)
}
}
func TestServeStopsOnContextCancellation(t *testing.T) {
ln := newTestListener(t)
addr := ln.Addr().String()
_, stop := startServer(t, ln)
// stop blocks until serve returns, and serve waits on its serving
// goroutine, so a clean return means nothing was left running.
if err := stop(); err != nil {
t.Fatalf("serve returned an error on shutdown: %v", err)
}
// Shutdown must have closed the listener, freeing the port.
reopened, err := net.Listen("tcp", addr)
if err != nil {
t.Fatalf("listener still bound to %s after shutdown: %v", addr, err)
}
_ = reopened.Close()
}
func TestServeWithAlreadyCancelledContext(t *testing.T) {
ln := newTestListener(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()
// Shutdown can win the race against the serving goroutine here. That is
// safe: a server already told to shut down makes Serve return
// ErrServerClosed straight away, so nothing blocks.
done := make(chan error, 1)
go func() {
done <- serve(ctx, ln)
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("serve on an already cancelled context returned: %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("serve hung on an already cancelled context")
}
}
func TestServeReturnsErrorWhenAddressUnavailable(t *testing.T) {
ln := newTestListener(t)
defer ln.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// The port is already used in newTestListener, so Serve should return an error
if err := Serve(ctx, ServerConfig{Addr: ln.Addr().String()}); err == nil {
t.Fatal("Serve on an address already in use returned no error")
}
}
func TestRegisterPrometheusMetricsIsIdempotent(t *testing.T) {
RegisterPrometheusMetrics()
// Registering a collector that is already registered is an error,
// RegisterPrometheusMetrics should be guarded with sync.Once.
mustNotPanic(t, "repeated RegisterPrometheusMetrics call", RegisterPrometheusMetrics)
// Confirm the collectors were really registered.
err := prometheus.DefaultRegisterer.Register(ActiveServices)
var alreadyRegistered prometheus.AlreadyRegisteredError
if !errors.As(err, &alreadyRegistered) {
t.Fatalf("Register(ActiveServices) error = %v, want AlreadyRegisteredError", err)
}
}
// newTestListener binds a loopback listener on an arbitrary free port.
func newTestListener(t *testing.T) net.Listener {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listening on a free loopback port: %v", err)
}
return ln
}
// startServer runs serve on ln and returns the base URL along with a stop
// function that cancels the context and reports what serve returned.
func startServer(t *testing.T, ln net.Listener) (string, func() error) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
serveErr := make(chan error, 1)
go func() {
serveErr <- serve(ctx, ln)
}()
var (
once sync.Once
err error
)
stop := func() error {
once.Do(func() {
cancel()
select {
case err = <-serveErr:
case <-time.After(10 * time.Second):
err = errors.New("serve did not return after the context was cancelled")
}
})
return err
}
t.Cleanup(func() {
_ = stop()
})
return "http://" + ln.Addr().String(), stop
}
func get(t *testing.T, url string) (string, int) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
t.Fatalf("building request for %s: %v", url, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", url, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("reading body of %s: %v", url, err)
}
return string(body), resp.StatusCode
}
// mustNotPanic reports a panic in fn as a test failure describing what
// panicked, rather than letting it take down the test binary.
func mustNotPanic(t *testing.T, what string, fn func()) {
t.Helper()
defer func() {
if r := recover(); r != nil {
t.Fatalf("%s panicked: %v", what, r)
}
}()
fn()
}

View File

@@ -1,19 +1,19 @@
package networkinterface
import (
log "log/slog"
"sync"
"github.com/vishvananda/netlink"
)
type Manager struct {
lock sync.Mutex
interfaces map[string]*Link
}
type Link struct {
Lock sync.Mutex
Intf netlink.Link
mu sync.Mutex
intf netlink.Link
}
func NewManager() *Manager {
@@ -23,19 +23,37 @@ func NewManager() *Manager {
}
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
updated, err := netlink.LinkByName(l.Intf.Attrs().Name)
if err != nil {
log.Error("failed to get interface %q: %w", l.Intf.Attrs().Name, err)
return nil
}
l.Intf = updated
return l
if intf == nil || intf.Attrs() == nil {
return nil
}
result := &Link{
Intf: intf,
attrs := intf.Attrs()
m.lock.Lock()
defer m.lock.Unlock()
if link, ok := m.interfaces[attrs.Name]; ok {
link.replace(intf)
return link
}
m.interfaces[intf.Attrs().Name] = result
return result
link := &Link{intf: intf}
m.interfaces[attrs.Name] = link
return link
}
func (l *Link) WithInterface(run func(netlink.Link) error) error {
l.mu.Lock()
defer l.mu.Unlock()
return run(l.intf)
}
func (l *Link) replace(intf netlink.Link) {
l.mu.Lock()
defer l.mu.Unlock()
l.intf = intf
}
func (m *Manager) Len() int {
m.lock.Lock()
defer m.lock.Unlock()
return len(m.interfaces)
}

View File

@@ -0,0 +1,65 @@
package networkinterface_test
import (
"context"
"sync"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node/noop"
"github.com/kube-vip/kube-vip/pkg/route"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestManagerReconstructsProductionInstanceConcurrently(t *testing.T) {
config := &kubevip.Config{Interface: "lo", ServicesInterface: "lo", VIPSubnet: "32", DisableServiceUpdates: true}
manager := networkinterface.NewManager()
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default", UID: "service"},
Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10"},
}
start := make(chan struct{})
var ready sync.WaitGroup
ready.Add(2)
results := make(chan struct {
instance *instance.Instance
err error
}, 2)
for range 2 {
go func() {
ready.Done()
<-start
instanceConfig := *config
created, err := instance.NewInstance(context.Background(), service.DeepCopy(), &instanceConfig, manager,
arp.NewManager(&instanceConfig), route.NewManager(), noop.NewManager(), &sync.WaitGroup{})
results <- struct {
instance *instance.Instance
err error
}{created, err}
}()
}
ready.Wait()
close(start)
for range 2 {
select {
case result := <-results:
if result.err != nil {
t.Fatalf("NewInstance() error = %v", result.err)
}
if len(result.instance.Clusters) != 1 {
t.Fatalf("cluster count = %d, want 1", len(result.instance.Clusters))
}
case <-time.After(3 * time.Second):
t.Fatal("timed out waiting for concurrent NewInstance calls")
}
}
if got := manager.Len(); got != 1 {
t.Fatalf("cached link count = %d, want 1", got)
}
}

View File

@@ -0,0 +1,74 @@
package networkinterface
import (
"sync"
"testing"
"github.com/vishvananda/netlink"
)
func TestManagerGetReplacesChangedInterfaceIndex(t *testing.T) {
manager := NewManager()
firstInterface := dummyLink("eth0", 1)
first := manager.Get(firstInterface)
if got := manager.Get(dummyLink("eth0", 1)); got != first {
t.Fatal("Get returned a new link for the same interface generation")
}
secondInterface := dummyLink("eth0", 2)
second := manager.Get(secondInterface)
if second != first {
t.Fatal("Get replaced the shared link after the interface index changed")
}
var current netlink.Link
if err := first.WithInterface(func(intf netlink.Link) error {
current = intf
return nil
}); err != nil {
t.Fatalf("WithInterface() error = %v", err)
}
if current != secondInterface {
t.Fatal("Get did not retain the new link generation")
}
}
func TestManagerGetConcurrent(t *testing.T) {
manager := NewManager()
interfaces := []netlink.Link{dummyLink("eth0", 1), dummyLink("eth1", 2)}
var wg sync.WaitGroup
results := make(chan struct {
index int
link *Link
}, 64)
for index := range cap(results) {
interfaceIndex := index % len(interfaces)
wg.Go(func() {
results <- struct {
index int
link *Link
}{index: interfaceIndex, link: manager.Get(interfaces[interfaceIndex])}
})
}
wg.Wait()
close(results)
var cached [2]*Link
for result := range results {
if result.link == nil {
t.Fatal("concurrent interface lookup returned nil")
}
if cached[result.index] == nil {
cached[result.index] = result.link
} else if result.link != cached[result.index] {
t.Fatalf("interface %d produced multiple cached Link objects", result.index)
}
}
if cached[0] == cached[1] {
t.Fatal("different interfaces shared one cached Link object")
}
}
func dummyLink(name string, index int) netlink.Link {
return &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name, Index: index}}
}

View File

@@ -49,9 +49,6 @@ func (m *Manager) Add(object string, r route, precheck, update bool) error {
itm, exists := m.tracker[key]
if !exists {
m.tracker[key] = newItem(r)
itm = m.tracker[key]
added, err := r.AddRoute(precheck)
if err != nil {
if update && errors.Is(err, syscall.EEXIST) && update {
@@ -70,9 +67,13 @@ func (m *Manager) Add(object string, r route, precheck, update bool) error {
}
}
itm = newItem(r)
m.tracker[key] = itm
if added {
log.Debug("[RT] added route", "path", key, "object", object)
}
m.tracker[key] = itm
}
itm.objects[object] = true
@@ -112,6 +113,8 @@ func (m *Manager) Delete(object string, r route) error {
}
func (m *Manager) Clear() {
m.mtx.Lock()
defer m.mtx.Unlock()
for _, itm := range m.tracker {
if err := itm.route.DeleteRoute(); err != nil {
log.Warn("[RT] failed to delete route", "err", err.Error())
@@ -121,6 +124,8 @@ func (m *Manager) Clear() {
}
func (m *Manager) Check(key string) bool {
m.mtx.Lock()
defer m.mtx.Unlock()
_, exists := m.tracker[key]
return exists
}

View File

@@ -0,0 +1,29 @@
package route
import (
"errors"
"testing"
)
var errTransientRouteAdd = errors.New("transient route add failure")
func TestManagerRetriesRouteAfterInitialAddFailure(t *testing.T) {
m := NewManager()
r := &mockRoute{hash: "retry", addErr: errTransientRouteAdd}
if err := m.Add("service", r, false, true); !errors.Is(err, errTransientRouteAdd) {
t.Fatalf("first add error = %v, want %v", err, errTransientRouteAdd)
}
// A transient netlink failure (for example while the link is being recreated)
// must not poison the in-memory tracker. The next reconciliation has to retry
// the kernel operation.
r.addErr = nil
r.added = true
if err := m.Add("service", r, false, true); err != nil {
t.Fatalf("retry add failed: %v", err)
}
if r.addCalls != 2 {
t.Fatalf("AddRoute called %d times, want 2", r.addCalls)
}
}

View File

@@ -1,7 +1,9 @@
package route
import (
"errors"
"fmt"
"sync"
"testing"
)
@@ -184,8 +186,50 @@ func Test_MultipleRoutesAddDel(t *testing.T) {
})
}
func TestAddFailureDoesNotTrackRoute(t *testing.T) {
m := NewManager()
r := &mockRoute{hash: "failed-route", addErr: errors.New("add failed")}
if err := m.Add("service", r, false, false); err == nil {
t.Fatal("Add error = nil, want route failure")
}
if m.Check(r.RouteHash()) {
t.Fatal("failed route was tracked")
}
r.addErr = nil
if err := m.Add("service", r, false, false); err != nil {
t.Fatalf("retry Add error = %v", err)
}
if !m.Check(r.RouteHash()) {
t.Fatal("successful retry was not tracked")
}
}
func TestClearAndCheckAreSafeWithRouteUpdates(t *testing.T) {
manager := NewManager()
route := &mockRoute{hash: "concurrent-route", added: true}
if err := manager.Add("service", route, false, false); err != nil {
t.Fatalf("Add() error = %v", err)
}
var wg sync.WaitGroup
for range 20 {
wg.Go(func() {
manager.Check(route.RouteHash())
})
}
wg.Go(manager.Clear)
wg.Wait()
if manager.Check(route.RouteHash()) {
t.Fatal("route remained tracked after concurrent Clear")
}
}
type mockRoute struct {
added bool
addCalls int
addErr error
updated bool
updateErr error
@@ -195,6 +239,7 @@ type mockRoute struct {
}
func (mr *mockRoute) AddRoute(_ bool) (bool, error) {
mr.addCalls++
return mr.added, mr.addErr
}

View File

@@ -12,6 +12,7 @@ type Context struct {
IsWatched bool
ConfiguredNetworks sync.Map
EndpointsReady chan any
mu sync.Mutex
epReady sync.Once
leaderElection sync.Once
Signalled atomic.Bool
@@ -51,6 +52,9 @@ func (ctx *Context) StartLeaderElectionOnce(f func()) {
}
func (ctx *Context) SignalReadiness() {
ctx.mu.Lock()
defer ctx.mu.Unlock()
ctx.epReady.Do(func() {
close(ctx.EndpointsReady)
ctx.Signalled.Store(true)
@@ -58,9 +62,50 @@ func (ctx *Context) SignalReadiness() {
}
func (ctx *Context) ResetReadiness() {
ctx.mu.Lock()
defer ctx.mu.Unlock()
if ctx.Signalled.Load() {
ctx.EndpointsReady = make(chan any)
ctx.epReady = sync.Once{}
ctx.Signalled.Store(false)
}
}
func (ctx *Context) GetEndpointsReady() chan any {
ctx.mu.Lock()
defer ctx.mu.Unlock()
return ctx.EndpointsReady
}
func (ctx *Context) SetLeaderCancel(cancel context.CancelFunc) {
ctx.mu.Lock()
defer ctx.mu.Unlock()
ctx.LeaderCancel = cancel
}
func (ctx *Context) CallLeaderCancel() {
ctx.mu.Lock()
cancel := ctx.LeaderCancel
ctx.mu.Unlock()
if cancel != nil {
cancel()
}
}
func (ctx *Context) SetWatched(watched bool) {
ctx.mu.Lock()
defer ctx.mu.Unlock()
ctx.IsWatched = watched
}
func (ctx *Context) IsWatchedLocked() bool {
ctx.mu.Lock()
defer ctx.mu.Unlock()
return ctx.IsWatched
}

View File

@@ -0,0 +1,56 @@
package servicecontext
import (
"context"
"sync"
"testing"
)
func TestReadinessResetConcurrentWithSignal(t *testing.T) {
ctx := New(context.Background())
start := make(chan struct{})
var wg sync.WaitGroup
wg.Go(func() {
<-start
for range 1000 {
ctx.SignalReadiness()
ctx.ResetReadiness()
}
})
wg.Go(func() {
<-start
for range 1000 {
ready := ctx.GetEndpointsReady()
select {
case <-ready:
default:
}
}
})
close(start)
wg.Wait()
}
func TestLeaderCancelConcurrentWithEndpointCleanup(t *testing.T) {
ctx := New(context.Background())
start := make(chan struct{})
var wg sync.WaitGroup
wg.Go(func() {
<-start
for range 1000 {
ctx.SetLeaderCancel(func() {})
}
})
wg.Go(func() {
<-start
for range 1000 {
ctx.CallLeaderCancel()
}
})
close(start)
wg.Wait()
}

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"net"
"os"
"slices"
"strings"
log "log/slog"
@@ -17,6 +18,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
)
@@ -130,6 +132,22 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
return cidrs[0]
}
// getSameFamilyCidrs returns every CIDR in the comma-separated list that has the
// same IP family as ip.
func getSameFamilyCidrs(sourceCidrs, ip string) []string {
if sourceCidrs == "" {
return nil
}
isV6 := utils.IsIPv6(ip)
var matching []string
for _, cidr := range strings.Split(sourceCidrs, ",") {
if (isV6 && utils.IsIPv6CIDR(cidr)) || (!isV6 && utils.IsIPv4CIDR(cidr)) {
matching = append(matching, cidr)
}
}
return matching
}
func checkCIDR(ip, cidr string) (string, error) {
_, ipnetA, err := net.ParseCIDR(cidr)
if err != nil {
@@ -187,12 +205,12 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
log.Warn("autodiscover CIDR", "err", discoverErr)
}
if p.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
} else {
if discoverErr == nil {
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
}
podCidrSource := p.config.EgressPodCidr
if podCidrSource == "" && discoverErr == nil {
podCidrSource = autoPodCIDR
}
if podCidrSource != "" {
podCidr = getSameFamilyCidr(podCidrSource, podIP)
}
if podCidr == "" {
@@ -204,6 +222,14 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
podCidr = defaultPodCIDR
}
// Pod-to-pod traffic to any pod CIDR must not be SNAT'd. Node auto-discovery
// yields one CIDR per node, so collect every same-family CIDR instead of only
// the one containing the local pod IP.
podCidrs := getSameFamilyCidrs(podCidrSource, podIP)
if len(podCidrs) == 0 {
podCidrs = []string{podCidr}
}
if p.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
} else {
@@ -243,10 +269,8 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
var ignoreCIDRs []string
// Create an array of CIDRs that we wont SNAT to.
if noInternalTraffic == "" || strings.Clone(noInternalTraffic) == "false" || p.config.EnableInternalSNAT {
ignoreCIDRs = append(ignoreCIDRs, []string{
podCidr,
serviceCidr,
}...)
ignoreCIDRs = append(ignoreCIDRs, podCidrs...)
ignoreCIDRs = append(ignoreCIDRs, serviceCidr)
}
// Add any specifically denied networks
@@ -300,9 +324,11 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
return fmt.Errorf("error creating mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, podCidr)
if err != nil {
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
for _, cidr := range podCidrs {
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, cidr)
if err != nil {
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, serviceCidr)
@@ -392,6 +418,49 @@ func (p *Processor) prepareEgressNftablesTable(serviceUID string, ipv6 bool) err
func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
serviceCIDR, podCIDR = p.discoverCIDRsFromAPI(ctx)
if serviceCIDR != "" && podCIDR != "" {
return serviceCIDR, podCIDR, nil
}
// Fall back to the kube-controller-manager flags; e.g. CNIs doing their own
// IPAM don't allocate Node PodCIDRs and clusters may lack the ServiceCIDR API.
legacyServiceCIDR, legacyPodCIDR, legacyErr := p.discoverCIDRsFromControllerManager(ctx)
if serviceCIDR == "" {
serviceCIDR = legacyServiceCIDR
}
if podCIDR == "" {
podCIDR = legacyPodCIDR
}
if serviceCIDR == "" || podCIDR == "" {
if legacyErr != nil {
return serviceCIDR, podCIDR, legacyErr
}
return serviceCIDR, podCIDR, fmt.Errorf("unable to fully determine cluster CIDR configurations")
}
return serviceCIDR, podCIDR, nil
}
func (p *Processor) discoverCIDRsFromAPI(ctx context.Context) (serviceCIDR, podCIDR string) {
serviceCIDRs, err := p.clientSet.NetworkingV1().ServiceCIDRs().List(ctx, metav1.ListOptions{})
if err == nil {
serviceCIDR = serviceCIDRsFromItems(serviceCIDRs.Items)
} else {
log.Debug("Unable to discover Service CIDRs from ServiceCIDR API", "err", err)
}
nodes, err := p.clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err == nil {
podCIDR = podCIDRsFromNodes(nodes.Items)
} else {
log.Debug("Unable to discover CNI Pod CIDRs from Node API", "err", err)
}
return serviceCIDR, podCIDR
}
func (p *Processor) discoverCIDRsFromControllerManager(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
options := metav1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
@@ -404,19 +473,45 @@ func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR
}
pod := podList.Items[0]
for flags := range pod.Spec.Containers[0].Command {
if strings.Contains(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=") {
podCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=", "")
for _, flag := range pod.Spec.Containers[0].Command {
if strings.Contains(flag, "--cluster-cidr=") {
podCIDR = strings.ReplaceAll(flag, "--cluster-cidr=", "")
}
if strings.Contains(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=") {
serviceCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=", "")
if strings.Contains(flag, "--service-cluster-ip-range=") {
serviceCIDR = strings.ReplaceAll(flag, "--service-cluster-ip-range=", "")
}
}
if podCIDR == "" || serviceCIDR == "" {
err = fmt.Errorf("unable to fully determine cluster CIDR configurations")
}
return serviceCIDR, podCIDR, nil
}
return
func serviceCIDRsFromItems(items []networkingv1.ServiceCIDR) string {
var cidrs []string
for _, item := range items {
cidrs = appendUnique(cidrs, item.Spec.CIDRs...)
}
return strings.Join(cidrs, ",")
}
func podCIDRsFromNodes(nodes []corev1.Node) string {
var cidrs []string
for _, node := range nodes {
if len(node.Spec.PodCIDRs) > 0 {
cidrs = appendUnique(cidrs, node.Spec.PodCIDRs...)
} else if node.Spec.PodCIDR != "" {
cidrs = appendUnique(cidrs, node.Spec.PodCIDR)
}
}
return strings.Join(cidrs, ",")
}
func appendUnique(values []string, additions ...string) []string {
for _, addition := range additions {
if addition == "" || slices.Contains(values, addition) {
continue
}
values = append(values, addition)
}
return values
}
func (p *Processor) updateEgressNftablesTableAnnotation(ctx context.Context, service *corev1.Service) error {

View File

@@ -0,0 +1,79 @@
package services
import (
"testing"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestServiceCIDRsFromItems(t *testing.T) {
serviceCIDR := serviceCIDRsFromItems([]networkingv1.ServiceCIDR{
{
ObjectMeta: metav1.ObjectMeta{Name: "kubernetes"},
Spec: networkingv1.ServiceCIDRSpec{
CIDRs: []string{"10.96.0.0/16", "fd00:10:96::/112"},
},
},
})
if serviceCIDR != "10.96.0.0/16,fd00:10:96::/112" {
t.Fatalf("serviceCIDR = %q", serviceCIDR)
}
}
func TestPodCIDRsFromNodes(t *testing.T) {
tests := []struct {
name string
nodes []corev1.Node
want string
}{
{
name: "dual stack across nodes",
nodes: []corev1.Node{
{
ObjectMeta: metav1.ObjectMeta{Name: "node-a"},
Spec: corev1.NodeSpec{
PodCIDRs: []string{"10.244.0.0/24", "fd00:10:244::/64"},
},
},
{
ObjectMeta: metav1.ObjectMeta{Name: "node-b"},
Spec: corev1.NodeSpec{
PodCIDRs: []string{"10.244.1.0/24", "fd00:10:244:1::/64"},
},
},
},
want: "10.244.0.0/24,fd00:10:244::/64,10.244.1.0/24,fd00:10:244:1::/64",
},
{
name: "legacy singular CIDR",
nodes: []corev1.Node{
{Spec: corev1.NodeSpec{PodCIDR: "10.244.0.0/24"}},
},
want: "10.244.0.0/24",
},
{
name: "duplicate CIDRs",
nodes: []corev1.Node{
{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
},
want: "10.244.0.0/16",
},
{
name: "missing CNI CIDRs",
nodes: []corev1.Node{{}},
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := podCIDRsFromNodes(test.nodes); got != test.want {
t.Fatalf("podCIDRsFromNodes() = %q, want %q", got, test.want)
}
})
}
}

View File

@@ -46,6 +46,14 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
return fmt.Errorf("no existing lease found for service %q with UID %q", service.Name, service.UID)
}
// A cancelled service context means this call belongs to a torn-down incarnation of
// the service. Its replacement is built as Cancel -> Delete -> Add, so the lease
// fetched above may already be the replacement's. Registering on it here would let
// the cleanup goroutine below retire a lease that is still in use.
if err := svcCtx.Ctx.Err(); err != nil {
return fmt.Errorf("service context cancelled before election start: %w", err)
}
isNew := svcLease.Add(objectName)
svcLease.Lock()
@@ -91,7 +99,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
return fmt.Errorf("service context cancelled before election start: %w", svcCtx.Ctx.Err())
case <-svcLease.Ctx.Done():
return fmt.Errorf("lease context cancelled before election start: %w", svcLease.Ctx.Err())
case <-svcCtx.EndpointsReady:
case <-svcCtx.GetEndpointsReady():
}
// this service is sharing lease with another service
@@ -114,7 +122,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
// Block until service context is cancelled
<-svcCtx.Ctx.Done()
if err := p.onStoppedLeading(svcLease, service); err != nil {
if err := p.onStoppedLeading(svcCtx, svcLease, service); err != nil {
log.Error("error on stopped leading", "error", err)
}
@@ -127,7 +135,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
leaderCtx, leaderCancel := context.WithCancel(svcLease.Ctx)
svcCtx.LeaderCancel = leaderCancel
svcCtx.SetLeaderCancel(leaderCancel)
run := election.RunConfig{
Config: p.config,
@@ -151,7 +159,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
// we can do cleanup here
svcLease.Elected.Store(false)
log.Info("leadership lost", "service", service.Name, "uid", service.UID, "leader", p.config.NodeName)
if err := p.onStoppedLeading(svcLease, service); err != nil {
if err := p.onStoppedLeading(svcCtx, svcLease, service); err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(service.Namespace, service.Name, "delete_service").Inc()
leaderCancel()
}
@@ -177,8 +185,6 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
}
func (p *Processor) onStartedLeading(svcCtx *servicecontext.Context, service *v1.Service, wg *sync.WaitGroup) error {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
err := p.SyncServices(svcCtx, service, wg, true)
if err != nil {
log.Error("service sync", "uid", service.UID, "err", err)
@@ -187,9 +193,18 @@ func (p *Processor) onStartedLeading(svcCtx *servicecontext.Context, service *v1
return nil
}
func (p *Processor) onStoppedLeading(svcLease *lease.Lease, service *v1.Service) error {
func (p *Processor) onStoppedLeading(svcCtx *servicecontext.Context, svcLease *lease.Lease, service *v1.Service) error {
currentSvcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return err
}
if currentSvcCtx != nil && currentSvcCtx != svcCtx {
log.Debug("skipping cleanup from superseded service context", "service", service.Name, "uid", service.UID)
return nil
}
log.Debug("deleting service due to lost leadership", "uid", service.UID)
err := p.deleteService(svcLease.Ctx, service.UID)
err = p.deleteService(svcLease.Ctx, service.UID)
if err != nil {
log.Error("service deletion", "err", err)
return err

View File

@@ -2,7 +2,6 @@ package services
import (
"context"
"errors"
"fmt"
log "log/slog"
"reflect"
@@ -91,7 +90,8 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc *Callback, forcedOnly bool, wg *sync.WaitGroup) error {
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc *Callback, forcedOnly bool,
wg *sync.WaitGroup, cancelWatcher context.CancelCauseFunc) error {
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
@@ -105,9 +105,9 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
return nil
}
// We only care about LoadBalancer services
// A tracked LoadBalancer must be torn down when its type changes.
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
return p.deleteTrackedService(svc)
}
// Check if we ignore this service
@@ -142,16 +142,6 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
}
svcInstance := instance.FindServiceInstance(svc, p.ServiceInstances)
var err error
if svcInstance == nil {
svcInstance, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
if err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)
}
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
p.updateActiveServicesMetric()
}
_, usesCommonLease := svc.Annotations[kubevip.ServiceLease]
if usesCommonLease && svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
@@ -194,17 +184,16 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "delete_service").Inc()
log.Error("(svc) unable to remove", "service", svc.UID)
}
// in theory this should never fail
p.svcMap.Delete(svc.UID)
// Drop this service from its lease now, so the replacement context
// below is not parented to a lease the pending cleanup is about to
// cancel. A lease shared with other services stays alive for them.
// Retire the lease before the replacement context is built, so Add below
// cannot hand back an instance the pending cleanup is about to cancel.
// A lease shared with other services keeps their references and survives.
ns, name := lease.ServiceName(svc)
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
p.leaseMgr.Delete(leaseID, lease.ServiceNamespacedName(svc), nil)
// Reset the the svcCtx when it was garbage collected
// As the next function will create a new context when nil
svcCtx = nil
svcInstance = nil
p.updateActiveServicesMetric()
}
}
@@ -216,19 +205,31 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
if svcCtx == nil {
ns, name := lease.ServiceName(svc)
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
lease := p.leaseMgr.Add(ctx, leaseID)
svcCtx = servicecontext.New(lease.Ctx)
p.leaseMgr.Add(ctx, leaseID)
// The service context is parented to the watcher, not to the lease: losing a
// lease must not tear the service down, it has to let the election restart.
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if svcInstance == nil {
svcInstance, err = instance.NewInstance(svcCtx.Ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
if err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)
}
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
p.updateActiveServicesMetric()
}
// this goroutine starts service handling function (with or without leaderelection)
if !svcCtx.IsWatched {
if !svcCtx.IsWatchedLocked() {
wg.Go(func() {
watchWg := sync.WaitGroup{}
defer func() {
// wait for the sub-goroutines and tag service as not watched
watchWg.Wait()
svcCtx.IsWatched = false
svcCtx.SetWatched(false)
}()
watchWg.Go(func() {
@@ -238,7 +239,7 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
err = serviceFunc.Run(svcCtx, svc, wg)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
if utils.IsPanicError(err) {
// cancel service context on panic error
// TODO: should we quit kube-vip altogether here?
svcCtx.Cancel()
@@ -256,15 +257,18 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
if err := p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider, cancelWatcher); err != nil {
log.Error("endpoint watcher failed", "service", svc.Name, "namespace", svc.Namespace, "err", err)
if utils.IsPanicError(err) {
cancelWatcher(err)
}
}
})
})
// tag service as watched
svcCtx.IsWatched = true
svcCtx.SetWatched(true)
}
if !p.config.EnableServicesElection {
@@ -307,23 +311,16 @@ func (p *Processor) Delete(event watch.Event, forcedOnly bool) error {
return nil
}
return p.deleteTrackedService(svc)
}
func (p *Processor) deleteTrackedService(svc *v1.Service) error {
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
}
// We can ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
log.Info("(svcs) ignore annotation for kube-vip", "service name", svc.Name)
return nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
@@ -347,14 +344,17 @@ func (p *Processor) Delete(event watch.Event, forcedOnly bool) error {
// Drop the per-service election series so a recreated service starts clean.
metrics.ServiceElectionLoops.DeleteLabelValues(svc.Namespace, svc.Name)
p.updateActiveServicesMetric()
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
}
return nil
}
func (p *Processor) Stop() {
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()

View File

@@ -4,14 +4,70 @@ import (
"context"
"testing"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
)
func TestAddOrModifyStopsTrackedServiceWhenTypeChanges(t *testing.T) {
for _, ignored := range []bool{false, true} {
name := "normal"
if ignored {
name = "ignored"
}
t.Run(name, func(t *testing.T) {
annotations := map[string]string{}
if ignored {
annotations[kubevip.LoadbalancerIgnore] = "true"
}
uid := types.UID("service-uid")
tracked := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "example",
Namespace: "default",
UID: uid,
Annotations: map[string]string{},
},
Spec: v1.ServiceSpec{
Type: v1.ServiceTypeLoadBalancer,
LoadBalancerIP: "192.0.2.10",
},
}
modified := tracked.DeepCopy()
modified.Spec.Type = v1.ServiceTypeClusterIP
modified.Annotations = annotations
p := &Processor{
config: &kubevip.Config{},
leaseMgr: lease.NewManager(),
ServiceInstances: []*instance.Instance{{ServiceSnapshot: tracked}},
}
svcCtx := servicecontext.New(context.Background())
p.svcMap.Store(uid, svcCtx)
if err := p.AddOrModify(context.Background(), watch.Event{Type: watch.Modified, Object: modified}, nil, false, nil, nil); err != nil {
t.Fatalf("AddOrModify returned error: %v", err)
}
if svcCtx.Ctx.Err() == nil {
t.Fatal("tracked service context was not cancelled")
}
if _, ok := p.svcMap.Load(uid); ok {
t.Fatal("tracked service context was not removed from svcMap")
}
if len(p.ServiceInstances) != 0 {
t.Fatalf("tracked service instance count = %d, want 0", len(p.ServiceInstances))
}
})
}
}
// TestDropCancelledServiceContext is a regression test for the lease/svcMap desync that
// permanently stops a LoadBalancer VIP from being advertised.
//
@@ -117,3 +173,37 @@ func TestDropCancelledServiceContextAllowsLeaseRecreation(t *testing.T) {
t.Fatal("expected a new lease to be created once the cancelled service context was dropped")
}
}
func TestOnStoppedLeadingDoesNotDeleteReplacementContext(t *testing.T) {
p := &Processor{
config: &kubevip.Config{},
leaseMgr: lease.NewManager(),
}
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "example",
Namespace: "default",
UID: types.UID("service-uid"),
},
}
oldCtx := servicecontext.New(context.Background())
replacementCtx := servicecontext.New(context.Background())
p.svcMap.Store(service.UID, replacementCtx)
replacementInstance := &instance.Instance{ServiceSnapshot: service.DeepCopy()}
p.ServiceInstances = []*instance.Instance{replacementInstance}
leaseNamespace, serviceLease := lease.ServiceName(service)
svcLease := p.leaseMgr.Add(context.Background(), lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease))
if err := p.onStoppedLeading(oldCtx, svcLease, service); err != nil {
t.Fatalf("onStoppedLeading returned an error: %v", err)
}
if got, err := p.getServiceContext(service.UID); err != nil || got != replacementCtx {
t.Fatalf("replacement context was changed: got %v, err %v", got, err)
}
if len(p.ServiceInstances) != 1 || p.ServiceInstances[0] != replacementInstance {
t.Fatal("replacement service instance was removed by superseded cleanup")
}
}

View File

@@ -0,0 +1,33 @@
package services
import (
"context"
"sync"
"testing"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
)
func TestWatchedFlagConcurrentWithWatcherTeardown(t *testing.T) {
svcCtx := servicecontext.New(context.Background())
start := make(chan struct{})
var wg sync.WaitGroup
wg.Go(func() {
<-start
for range 1000 {
svcCtx.SetWatched(false)
}
})
wg.Go(func() {
<-start
for range 1000 {
if !svcCtx.IsWatchedLocked() {
svcCtx.SetWatched(true)
}
}
})
close(start)
wg.Wait()
}

View File

@@ -23,7 +23,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
@@ -66,7 +65,7 @@ func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, w
select {
case <-ctx.Ctx.Done():
return nil
case <-ctx.EndpointsReady:
case <-ctx.GetEndpointsReady():
}
}
@@ -76,6 +75,15 @@ func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, w
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
// Egress: when the service reaches ActionNone it means AddCalled is already true.
// If ActiveEndpoint is now set (by the endpoint watcher) and the service has a
// LB IP, the initial addService call may have missed the SNAT configuration because
// ActiveEndpoint was not yet present. Re-run it here.
if svc.Annotations[kubevip.Egress] == "true" && svc.Annotations[kubevip.ActiveEndpoint] != "" {
if err := p.updateEgressConfiguration(ctx.Ctx, svc); err != nil {
log.Warn("[service] egress reconfigure on ActionNone", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
return nil
@@ -219,19 +227,23 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv4")
} else {
for ip := range inst.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv4 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
for {
select {
case <-ctx.Done():
log.Debug("IPv4 update watcher stopping")
return
case ip := <-inst.DHCPv4Client.IPChannel():
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv4 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
}
log.Debug("IPv4 update channel closed, stopping")
}
})
}
@@ -248,17 +260,22 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv6")
} else {
for ip := range inst.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv6 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
for {
select {
case <-ctx.Done():
log.Debug("IPv6 update watcher stopping")
return
case ip := <-inst.DHCPv6Client.IPChannel():
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv6 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
}
log.Debug("IPv6 update channel closed, stopping")
}
})
}
@@ -367,17 +384,6 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
return err
}
}
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
err := provider.UpdateServiceAnnotation(ctx, svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
@@ -454,7 +460,9 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
endpoints.ClearBGPHostsByInstance(ctx, serviceInstance, p.bgpServer)
}
if p.config.EnableRoutingTable && (p.config.EnableLeaderElection || p.config.EnableServicesElection) {
// ClearRoutesByInstance is reference-counted per route, so calling it here is safe
// even when the no-election path in Processor.Delete already cleared it.
if p.config.EnableRoutingTable {
if errs := endpoints.ClearRoutesByInstance(serviceInstance.ServiceSnapshot, serviceInstance, &p.ServiceInstances, p.routeMgr); len(errs) > 0 {
for _, err := range errs {
log.Error("unable to clear routes", "err", err)
@@ -546,11 +554,24 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
oldIPv6 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
newIPv6 := svc.Annotations[kubevip.ActiveEndpointIPv6]
// Skip update if endpoints haven't changed
// Skip update if endpoints haven't changed, without touching the API.
if oldIPv4 == newIPv4 && oldIPv6 == newIPv6 {
return nil
}
// The svc snapshot may have been captured before the LB IP was assigned.
// Refresh from the API so FetchServiceAddresses sees the current ingress.
if current, err := p.clientSet.CoreV1().Services(svc.Namespace).Get(ctx, svc.Name, metav1.GetOptions{}); err == nil {
// Preserve the caller-supplied annotations (ActiveEndpoint etc.) that triggered this call.
for k, v := range svc.Annotations {
if current.Annotations == nil {
current.Annotations = make(map[string]string)
}
current.Annotations[k] = v
}
svc = current
}
log.Info("[service] updating egress configuration",
"service", svc.Name,
"namespace", svc.Namespace,

View File

@@ -1,14 +1,90 @@
package services
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync"
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestConfigureServiceDoesNotOverwriteActiveEndpoint(t *testing.T) {
const selectedEndpoint = "172.30.2.40"
staleService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", UID: "test-uid",
Annotations: map[string]string{kubevip.Egress: "true", kubevip.ActiveEndpoint: ""},
},
Spec: v1.ServiceSpec{LoadBalancerIP: "10.114.44.149"},
}
currentService := staleService.DeepCopy()
currentService.Annotations[kubevip.ActiveEndpoint] = selectedEndpoint
currentService.ResourceVersion = "2"
var mutex sync.Mutex
updateRequests := 0
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
mutex.Lock()
defer mutex.Unlock()
writer.Header().Set("Content-Type", "application/json")
switch request.Method {
case http.MethodGet:
if err := json.NewEncoder(writer).Encode(currentService); err != nil {
t.Errorf("encode Service response: %v", err)
}
case http.MethodPut:
updatedService := &v1.Service{}
if err := json.NewDecoder(request.Body).Decode(updatedService); err != nil {
http.Error(writer, err.Error(), http.StatusBadRequest)
return
}
updateRequests++
currentService = updatedService
if err := json.NewEncoder(writer).Encode(currentService); err != nil {
t.Errorf("encode updated Service response: %v", err)
}
default:
http.Error(writer, "unexpected request", http.StatusMethodNotAllowed)
}
}))
defer server.Close()
clientSet, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL})
if err != nil {
t.Fatalf("create Kubernetes client: %v", err)
}
processor := &Processor{
config: &kubevip.Config{
DisableServiceUpdates: true,
EnableServicesElection: true,
},
clientSet: clientSet,
}
serviceInstance := &instance.Instance{ServiceSnapshot: staleService}
if err := processor.configureService(context.Background(), serviceInstance, staleService, &sync.WaitGroup{}); err != nil {
t.Fatalf("configureService returned error: %v", err)
}
mutex.Lock()
defer mutex.Unlock()
if updateRequests != 0 {
t.Fatalf("configureService sent %d stale Service updates, want none", updateRequests)
}
if got := currentService.Annotations[kubevip.ActiveEndpoint]; got != selectedEndpoint {
t.Fatalf("active endpoint = %q, want %q", got, selectedEndpoint)
}
}
// Test_upnpLeaseDurationForService tests whether the default lease duration is used, and whether the annotation
// overrides it correctly.
//

View File

@@ -1,6 +1,7 @@
package services
import (
"context"
"fmt"
"sync"
@@ -10,28 +11,31 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service,
provider providers.Provider, cancelWatcher context.CancelCauseFunc) error {
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := provider.CreateRetryWatcher(svcCtx.Ctx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
if svcCtx.Ctx.Err() != nil {
return nil
}
return utils.WrapPanicError(err, "[%s] error watching endpoints", provider.GetLabel())
}
d, err := debouncer.New(rw.ResultChan(), p.config.DebounceTime)
if err != nil {
rw.Stop()
return fmt.Errorf("failed to create debouncer for endpoints event: %w", err)
return utils.WrapPanicError(err, "failed to create debouncer for endpoints event")
}
wg := sync.WaitGroup{}
var wg sync.WaitGroup
stopChan := make(chan any)
defer func() {
@@ -46,6 +50,9 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
if d != nil {
if err := d.Start(svcCtx.Ctx); err != nil {
log.Error("[endpoint watcher] debouncer, cancelling context", "error", err.Error())
if svcCtx.Ctx.Err() == nil {
cancelWatcher(utils.WrapPanicError(err, "[%s] endpoint debouncer failed", provider.GetLabel()))
}
svcCtx.Cancel()
}
}
@@ -71,28 +78,31 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id,
case watch.Added, watch.Modified, watch.Deleted:
if event.Type == watch.Deleted {
log.Info("[endpoint watcher] endpoint object deleted", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
}
restart, err := epProcessor.Reconcile(svcCtx, event, &lastKnownGoodEndpoint, service, id,
p.StartServicesLeaderElection, &wg, p.clientSet, p.updateEgressConfiguration)
if restart {
continue
} else if err != nil {
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
return fmt.Errorf("[%s] error while processing %s event: %w", provider.GetLabel(), event.Type, err)
}
case watch.Deleted:
if err := epProcessor.Delete(svcCtx.Ctx, service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
log.Info("[endpoint watcher] stopping watching - endpoint object deleted", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
if svcCtx.Ctx.Err() != nil {
return nil
}
watchErr := utils.WatchError(event.Object)
log.Error("watch error", "provider", provider.GetLabel(), "err", watchErr)
return utils.WrapPanicError(watchErr, "[%s] endpoint watch failed", provider.GetLabel())
}
}
if svcCtx.Ctx.Err() != nil {
return nil
}
log.Info("[endpoint watcher] stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
return utils.NewPanicError("[%s] endpoint watch channel closed unexpectedly for service %s/%s", provider.GetLabel(), service.Namespace, service.Name)
}

View File

@@ -2,13 +2,11 @@ package services
import (
"context"
"errors"
"fmt"
"sync"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/debouncer"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/metrics"
@@ -16,7 +14,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
@@ -62,7 +59,7 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
return fmt.Errorf("failed to create debouncer for endpoints event: %w", err)
}
wg := sync.WaitGroup{}
var wg sync.WaitGroup
defer func() {
if d != nil {
d.Stop()
@@ -71,14 +68,14 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
wg.Wait()
}()
watcherCtx, watcherCancel := context.WithCancel(ctx)
defer watcherCancel()
watcherCtx, cancelWatcher := context.WithCancelCause(ctx)
defer cancelWatcher(nil)
wg.Go(func() {
if d != nil {
if err := d.Start(watcherCtx); err != nil {
log.Error("(svcs) debouncer, cancelling context", "error", err.Error())
watcherCancel()
cancelWatcher(utils.WrapPanicError(err, "service debouncer failed"))
}
}
<-watcherCtx.Done()
@@ -96,47 +93,52 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
}
// Used for tracking an active endpoint / pod
EventLoop:
for event := range ch {
metrics.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg); err != nil {
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("add/modify service error: %w", err)
} else {
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
}
case watch.Deleted:
if err := p.Delete(event, forcedOnly); err != nil {
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("delete service error: %w", err)
} else {
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
select {
case <-ctx.Done():
log.Info("global context done")
case <-watcherCtx.Done():
log.Info("WatcheConotext done")
break EventLoop
default:
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg, cancelWatcher); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("add/modify service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Deleted:
if err := p.Delete(event, forcedOnly); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("delete service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
watchErr := utils.WatchError(event.Object)
log.Error("services", "err", watchErr)
return utils.WrapPanicError(watchErr, "service watch failed")
default:
}
}
}
if ctx.Err() != nil {
return nil
}
if watcherErr := context.Cause(watcherCtx); watcherErr != nil {
return watcherErr
}
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
return utils.NewPanicError("service watch channel closed unexpectedly")
}
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {

View File

@@ -146,7 +146,7 @@ func getQdiscFromInterfaceByType(nicID int, nicName string, qType string) (uint3
// get id through tc qdisc show dev fromNICName
qs, err := netlink.QdiscList(&netlink.Ifb{LinkAttrs: netlink.LinkAttrs{Index: nicID}})
if err != nil {
fmt.Printf("Failed to list qdisc for interface %s: %v", nicName, err)
log.Error("failed to list qdisc", "interface", nicName, "err", err)
return 0, err
}
for _, q := range qs {

View File

@@ -1,15 +1,31 @@
package utils
import "fmt"
import (
"errors"
"fmt"
)
type PanicError struct {
cause string
cause error
}
func (e *PanicError) Error() string {
return fmt.Sprintf("%s - unrecoverable error", e.cause)
}
func NewPanicError(cause string) error {
return &PanicError{cause: cause}
func (e *PanicError) Unwrap() error {
return e.cause
}
func NewPanicError(format string, args ...any) error {
return &PanicError{cause: fmt.Errorf(format, args...)}
}
func WrapPanicError(err error, format string, args ...any) error {
return &PanicError{cause: fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), err)}
}
func IsPanicError(err error) bool {
var panicErr *PanicError
return errors.As(err, &panicErr)
}

41
pkg/utils/panic_test.go Normal file
View File

@@ -0,0 +1,41 @@
package utils
import (
"errors"
"fmt"
"testing"
)
func TestIsPanicError(t *testing.T) {
panicErr := NewPanicError("endpoint watch stopped")
tests := []struct {
name string
err error
want bool
}{
{name: "direct", err: panicErr, want: true},
{name: "wrapped", err: fmt.Errorf("watch failed: %w", panicErr), want: true},
{name: "ordinary", err: errors.New("watch failed"), want: false},
{name: "nil", err: nil, want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := IsPanicError(test.err); got != test.want {
t.Fatalf("IsPanicError() = %t, want %t", got, test.want)
}
})
}
}
func TestWrapPanicErrorPreservesCause(t *testing.T) {
cause := errors.New("endpointslices is forbidden")
err := WrapPanicError(cause, "endpoint watch failed")
if !IsPanicError(err) {
t.Fatal("expected wrapped error to be classified as PanicError")
}
if !errors.Is(err, cause) {
t.Fatal("expected wrapped PanicError to preserve its cause")
}
}

View File

@@ -7,10 +7,20 @@ import (
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
)
// WatchError converts a Kubernetes watch error object into a safe Go error.
func WatchError(object runtime.Object) error {
errObject := apierrors.FromObject(object)
if statusErr, ok := errObject.(*apierrors.StatusError); ok {
return statusErr
}
return fmt.Errorf("unknown watch error object of type %T: %v", object, object)
}
// watchWithAuthRetry retries watchFn with exponential backoff on transient 403 Forbidden
// and 401 Unauthorized errors. On joining control plane nodes with K8s 1.34+, the local
// etcd may still be a learner when kube-vip starts, causing RBAC data to be unavailable.
@@ -39,7 +49,14 @@ func WatchWithAuthRetry(ctx context.Context, watchFn func(context.Context) (watc
return false, nil
})
if err != nil {
return nil, NewPanicError(fmt.Sprintf("watch failed after retries: %q (last: %v)", err.Error(), lastErr))
if ctx.Err() != nil {
return nil, ctx.Err()
}
if lastErr != nil {
log.Error("watch auth retries exhausted", "err", lastErr)
return nil, lastErr
}
return nil, WrapPanicError(err, "watch failed after retries (last: %v)", lastErr)
}
return w, nil
}

37
pkg/utils/watcher_test.go Normal file
View File

@@ -0,0 +1,37 @@
package utils
import (
"errors"
"strings"
"testing"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestWatchErrorPreservesStatusError(t *testing.T) {
status := &metav1.Status{
Status: metav1.StatusFailure,
Reason: metav1.StatusReasonForbidden,
Message: "endpointslices is forbidden",
}
err := WatchError(status)
var statusErr *apierrors.StatusError
if !errors.As(err, &statusErr) {
t.Fatalf("expected a status error, got %T: %v", err, err)
}
if !strings.Contains(err.Error(), status.Message) {
t.Fatalf("expected error to contain %q, got %q", status.Message, err)
}
}
func TestWatchErrorHandlesUnexpectedObject(t *testing.T) {
err := WatchError(&metav1.APIGroup{})
if err == nil {
t.Fatal("expected an error for an unexpected watch object")
}
if !strings.Contains(err.Error(), "unknown watch error object") {
t.Fatalf("expected unexpected-object context, got %q", err)
}
}

View File

@@ -35,6 +35,9 @@ import (
const (
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
// Linux reserves IFA_PROTO values 0-3 for unspecified and kernel-generated
// addresses; user-space protocols start at 4. This is only a safety floor,
firstUserAddressProtocol = 4
DefaultMaskIPv4 = 32
DefaultMaskIPv6 = 128
@@ -77,8 +80,7 @@ type Network interface {
// network - This allows network configuration
type network struct {
mu sync.Mutex
mu sync.RWMutex
address *netlink.Addr
link *networkinterface.Link
ports []v1.ServicePort
@@ -168,6 +170,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
markKubeVIPAddress(result.address, result.routingProtocol)
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
@@ -265,6 +268,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
return networks, err
}
markKubeVIPAddress(result.address, result.routingProtocol)
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
@@ -310,7 +314,17 @@ func ListRoutesByDst(table int, dst *net.IPNet) ([]netlink.Route, error) {
return routes, nil
}
func (configurator *network) PrepareRoute() *netlink.Route {
func (configurator *network) PrepareRoute() (route *netlink.Route) {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
route = configurator.prepareRoute(intf)
return nil
})
return route
}
func (configurator *network) prepareRoute(intf netlink.Link) *netlink.Route {
routeScope := netlink.SCOPE_UNIVERSE
if configurator.routingTableType == unix.RTN_LOCAL {
routeScope = netlink.SCOPE_LINK
@@ -318,7 +332,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
route := &netlink.Route{
Scope: routeScope,
Dst: configurator.address.IPNet,
LinkIndex: configurator.link.Intf.Attrs().Index,
LinkIndex: intf.Attrs().Index,
Table: configurator.routeTable,
Type: configurator.routingTableType,
Protocol: netlink.RouteProtocol(configurator.routingProtocol),
@@ -339,32 +353,33 @@ func NetlinkHash(r *netlink.Route) string {
}
// AddRoute - Add an IP address to a route table
func (configurator *network) AddRoute(precheck bool) (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
route := configurator.PrepareRoute()
exists := false
var err error
if precheck {
exists, err = configurator.routeExists(route)
if err != nil {
return false, errors.Wrap(err, "failed to check route")
func (configurator *network) AddRoute(precheck bool) (added bool, err error) {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
err = configurator.link.WithInterface(func(intf netlink.Link) error {
route := configurator.prepareRoute(intf)
exists := false
if precheck {
var existsErr error
exists, existsErr = configurator.routeExists(intf, route)
if existsErr != nil {
return errors.Wrap(existsErr, "failed to check route")
}
}
}
if !exists {
if err := netlink.RouteAdd(route); err != nil {
return false, errors.Wrap(err, "failed to add route")
if exists {
return nil
}
return true, nil
}
return false, nil
if routeErr := netlink.RouteAdd(route); routeErr != nil {
return errors.Wrap(routeErr, "failed to add route")
}
added = true
return nil
})
return added, err
}
func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
func (configurator *network) routeExists(intf netlink.Link, route *netlink.Route) (bool, error) {
routes, err := netlink.RouteList(intf, netlink.FAMILY_ALL)
if err != nil {
return false, errors.Wrap(err, "failed to list routes")
}
@@ -385,16 +400,18 @@ func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
// the same-prefix interface address is processed in the same netlink batch,
// leaving the route in the kernel but never redistributed.
func (configurator *network) ReplaceRoute() error {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
route := configurator.PrepareRoute()
configurator.reassertToggle = !configurator.reassertToggle
if configurator.reassertToggle {
route.Realm = 1
} else {
route.Realm = 2
}
return netlink.RouteReplace(route)
configurator.mu.Lock()
defer configurator.mu.Unlock()
return configurator.link.WithInterface(func(intf netlink.Link) error {
route := configurator.prepareRoute(intf)
configurator.reassertToggle = !configurator.reassertToggle
if configurator.reassertToggle {
route.Realm = 1
} else {
route.Realm = 2
}
return netlink.RouteReplace(route)
})
}
// DeleteRoute - Delete an IP address from a route table
@@ -405,6 +422,8 @@ func (configurator *network) DeleteRoute() error {
// GetRoutes - Get an IP addresses from a route table
func (configurator *network) getRoutes() (*[]netlink.Route, error) {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
routes, err := ListRoutesByDst(configurator.routeTable, configurator.address.IPNet)
if err != nil {
return nil, fmt.Errorf("error getting routes: %w", err)
@@ -443,12 +462,22 @@ func (configurator *network) shouldSkipDAD(override bool) bool {
// precheck: if true, check if the IP already exists before adding
// skipDAD: if true, set IFA_F_NODAD flag for IPv6 addresses to skip Duplicate Address Detection
func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
configurator.mu.Lock()
defer configurator.mu.Unlock()
var added bool
err := configurator.link.WithInterface(func(intf netlink.Link) error {
var addErr error
added, addErr = configurator.addIP(intf, precheck, skipDAD, minLifetime...)
return addErr
})
return added, err
}
func (configurator *network) addIP(intf netlink.Link, precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
var existing *netlink.Addr
var err error
if precheck {
if existing, err = configurator.IsSet(); err != nil {
if existing, err = configurator.isSet(intf); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
}
}
@@ -469,14 +498,18 @@ func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...i
// an address that we know should be ours (e.g., after DADFAILED recovery).
// We also allow to globally configure NODAD in case user knows they are running in an
// environment where multiple nodes may advertise the same VIP (e.g., ECMP routing).
if configurator.shouldSkipDAD(skipDAD) && utils.IsIPv6(configurator.address.IP.String()) {
configurator.address.Flags |= unix.IFA_F_NODAD
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
if utils.IsIPv6(configurator.address.IP.String()) {
if configurator.shouldSkipDAD(skipDAD) {
configurator.address.Flags |= unix.IFA_F_NODAD
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
} else {
configurator.address.Flags &^= unix.IFA_F_NODAD
}
}
log.Debug("replacing IP", "address", configurator.address)
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", configurator.link.Intf.Attrs().Name))
if err := netlink.AddrReplace(intf, configurator.address); err != nil {
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", intf.Attrs().Name))
}
if configurator.nftables {
@@ -514,7 +547,7 @@ func (configurator *network) configureNFTables() error {
"ports", configurator.ports, "service-name", configurator.serviceName)
opt := nftables.TableFamilyIPv4
if utils.IsIPv6(configurator.IP()) {
if utils.IsIPv6(configurator.address.IP.String()) {
opt = nftables.TableFamilyIPv6
}
@@ -591,12 +624,13 @@ func (configurator *network) addNftablesRulesToLimitTrafficPorts(c *nfinternal.C
}
}
firstRule, err := insertCommonNFTablesRules(c, configurator.IP(), comment)
vip := configurator.address.IP.String()
firstRule, err := insertCommonNFTablesRules(c, vip, comment)
if err != nil {
return fmt.Errorf("could not add common nftables rules: %w", err)
}
if err := configurator.insertNFTablesRulesForServicePorts(c, configurator.IP(), comment, firstRule.Handle); err != nil {
if err := configurator.insertNFTablesRulesForServicePorts(c, vip, comment, firstRule.Handle); err != nil {
return fmt.Errorf("could not add nftables rules for service ports: %v", err)
}
@@ -1053,10 +1087,19 @@ func (configurator *network) removeNftablesRuleToLimitTrafficPorts(c *nfinternal
// DeleteIP - Remove an IP address from the interface
func (configurator *network) DeleteIP() (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
configurator.mu.Lock()
defer configurator.mu.Unlock()
var deleted bool
err := configurator.link.WithInterface(func(intf netlink.Link) error {
var deleteErr error
deleted, deleteErr = configurator.deleteIP(intf)
return deleteErr
})
return deleted, err
}
result, err := configurator.IsSet()
func (configurator *network) deleteIP(intf netlink.Link) (bool, error) {
result, err := configurator.isSet(intf)
if err != nil {
return false, errors.Wrap(err, "ip check in DeleteIP failed")
}
@@ -1066,7 +1109,7 @@ func (configurator *network) DeleteIP() (bool, error) {
return false, nil
}
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
if err = netlink.AddrDel(intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not delete ip")
}
@@ -1136,7 +1179,8 @@ func (configurator *network) addIptablesRulesForMasquerade() error {
// TO DO: It seems it is not be possible to use google/nftables with IPVS due to lack of IPVS matcher in nft
func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client, comment string) error {
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.IP(), configurator.IPVSMark())
vip := configurator.address.IP.String()
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, vip, configurator.ipvsMark)
markChain := &nftables.Chain{
Name: "ipvs_prerouting",
@@ -1150,7 +1194,7 @@ func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client,
markChain = c.AddChain(markChain)
ip := net.ParseIP(configurator.IP())
ip := net.ParseIP(vip)
if ip.To4() != nil {
ip = ip.To4()
@@ -1160,7 +1204,7 @@ func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client,
port := binaryutil.BigEndian.PutUint16(configurator.ipvsPort)
mark := binaryutil.NativeEndian.PutUint32(configurator.IPVSMark())
mark := binaryutil.NativeEndian.PutUint32(configurator.ipvsMark)
markRule := &nftables.Rule{
Table: markChain.Table,
@@ -1282,7 +1326,7 @@ func (configurator *network) removeNftablesRulesForMasquerade(c *nfinternal.Clie
if configurator.serviceName != "" {
comment = fmt.Sprintf(iptablesComment, configurator.serviceName)
}
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.IP(), configurator.IPVSMark())
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.address.IP.String(), configurator.ipvsMark)
r, err := c.FindRuleByComment(chain.Table, chain, cmt)
if err != nil {
@@ -1345,27 +1389,26 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
func (configurator *network) IsDADFAIL() bool {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
return false
}
// Get all the address
addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
if err != nil {
return false
}
// Find the VIP and check if it is DADFAILED
for _, address := range addresses {
if address.IP.Equal(configurator.address.IP) && addressHasDADFAILEDFlag(address) {
return true
configurator.mu.RLock()
defer configurator.mu.RUnlock()
var dadFailed bool
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
return nil
}
}
return false
addresses, err := netlink.AddrList(intf, netlink.FAMILY_V6)
if err != nil {
return nil
}
for _, address := range addresses {
if address.IP.Equal(configurator.address.IP) && addressHasDADFAILEDFlag(address) {
dadFailed = true
break
}
}
return nil
})
return dadFailed
}
func addressHasDADFAILEDFlag(address netlink.Addr) bool {
@@ -1374,6 +1417,16 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
// isSet - Check to see if VIP is set
func (configurator *network) IsSet() (result *netlink.Addr, err error) {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
err = configurator.link.WithInterface(func(intf netlink.Link) error {
result, err = configurator.isSet(intf)
return err
})
return result, err
}
func (configurator *network) isSet(intf netlink.Link) (result *netlink.Addr, err error) {
var addresses []netlink.Addr
if configurator.address == nil {
@@ -1384,7 +1437,7 @@ func (configurator *network) IsSet() (result *netlink.Addr, err error) {
return nil, nil
}
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
addresses, err = netlink.AddrList(intf, 0)
if err != nil {
err = errors.Wrap(err, "could not list addresses")
@@ -1404,10 +1457,10 @@ func (configurator *network) IsSet() (result *netlink.Addr, err error) {
func (configurator *network) SetIP(ip string) error {
configurator.mu.Lock()
defer configurator.mu.Unlock()
return configurator.setIP(ip)
}
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
func (configurator *network) setIP(ip string) error {
if strings.Contains("/", ip) {
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
}
@@ -1439,7 +1492,7 @@ func (configurator *network) SetIP(ip string) error {
if err != nil {
return err
}
if configurator.address != nil && configurator.IsDNS() {
if configurator.address != nil && configurator.dnsName != "" {
addr.ValidLft = defaultValidLft
} else {
addr.ValidLft = math.MaxInt
@@ -1449,9 +1502,94 @@ func (configurator *network) SetIP(ip string) error {
addr.PreferedLft = 0
configurator.address = addr
if configurator.routingProtocol != 0 {
markKubeVIPAddress(configurator.address, configurator.routingProtocol)
}
return nil
}
// IsKubeVIPAddress reports whether an address carries kube-vip's configured
// protocol origin. IFA_PROTO is supported for both IPv4 and IPv6 on Linux 5.18+.
func IsKubeVIPAddress(address netlink.Addr, protocol int) bool {
return protocol >= firstUserAddressProtocol && address.Protocol == protocol
}
// RetainedKubeVIPAddressKeys returns the tagged kernel addresses matching the
// supplied VIPs. Callers use the keys with CleanupKubeVIPAddresses after they
// determine which VIPs remain locally referenced.
func RetainedKubeVIPAddressKeys(protocol int, vips map[string]struct{}) (map[string]struct{}, error) {
retained := make(map[string]struct{})
err := forEachKubeVIPAddress(protocol, func(_ netlink.Link, address netlink.Addr) error {
if _, retain := vips[address.IP.String()]; retain {
retained[addressKey(address)] = struct{}{}
}
return nil
})
if err != nil {
return nil, err
}
return retained, nil
}
// CleanupKubeVIPAddresses removes kube-vip addresses not reasserted by this
// process. The retained keys must come from successful AddrReplace operations.
func CleanupKubeVIPAddresses(protocol int, retained map[string]struct{}) (int, error) {
removed := 0
err := forEachKubeVIPAddress(protocol, func(link netlink.Link, address netlink.Addr) error {
key := addressKey(address)
if key == "" {
return nil
}
if _, keep := retained[key]; keep {
return nil
}
if err := netlink.AddrDel(link, &address); err != nil {
return errors.Wrapf(err, "deleting kube-vip address %q from interface %q", address.IP, link.Attrs().Name)
}
removed++
return nil
})
if err != nil {
return removed, err
}
return removed, nil
}
func forEachKubeVIPAddress(protocol int, visit func(netlink.Link, netlink.Addr) error) error {
links, err := netlink.LinkList()
if err != nil {
return errors.Wrap(err, "listing network links")
}
for _, link := range links {
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
return errors.Wrapf(err, "listing addresses on interface %q", link.Attrs().Name)
}
for _, address := range addresses {
if IsKubeVIPAddress(address, protocol) {
if err := visit(link, address); err != nil {
return err
}
}
}
}
return nil
}
func addressKey(address netlink.Addr) string {
if address.LinkIndex <= 0 || address.IP == nil {
return ""
}
prefixLength, _ := address.Mask.Size()
return fmt.Sprintf("%d/%s/%d", address.LinkIndex, address.IP, prefixLength)
}
func markKubeVIPAddress(address *netlink.Addr, protocol int) {
if address != nil {
address.Protocol = protocol
}
}
// SetServicePorts updates the service ports from the service
// If you want to limit traffic to the VIP to only the service ports, add service ports to the network firstly.
func (configurator *network) SetServicePorts(service *v1.Service) {
@@ -1465,8 +1603,8 @@ func (configurator *network) SetServicePorts(service *v1.Service) {
// IP - return the IP Address
func (configurator *network) IP() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.mu.RLock()
defer configurator.mu.RUnlock()
if configurator.address == nil || configurator.address.IP == nil {
return ""
@@ -1476,8 +1614,8 @@ func (configurator *network) IP() string {
}
func (configurator *network) CIDR() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.mu.RLock()
defer configurator.mu.RUnlock()
if configurator.address == nil || configurator.address.IPNet == nil {
return ""
@@ -1488,24 +1626,30 @@ func (configurator *network) CIDR() string {
// IP - return the IP Address
func (configurator *network) IPisLinkLocal() bool {
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.address.IP.IsLinkLocalUnicast()
}
// DNSName return the configured dnsName when use DNS
func (configurator *network) DNSName() string {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.dnsName
}
// IsDNS - when dnsName is configured
func (configurator *network) IsDNS() bool {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.dnsName != ""
}
// IsDDNS - return true if use dynamic dns
func (configurator *network) IsDDNS() bool {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.isDDNS
}
@@ -1514,12 +1658,19 @@ func (configurator *network) IsDDNS() bool {
// it's expected that dynamic DNS should be configured so
// the fqdn for apiserver endpoint is dDNSHostName.{LocalDomain}
func (configurator *network) DDNSHostName() string {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return getHostName(configurator.dnsName)
}
// Interface - return the Interface name
func (configurator *network) Interface() string {
return configurator.link.Intf.Attrs().Name
var name string
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
name = intf.Attrs().Name
return nil
})
return name
}
func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
@@ -1530,32 +1681,28 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
}
l := intfMgr.Get(link)
l.Lock.Lock()
defer l.Lock.Unlock()
// Get addresses on adapter
addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
if err != nil {
return false, err
}
// Compare all addresses to new service address, and remove if needed
for _, existing := range addrs {
if existing.IP.String() == address {
// We've found the existing address
found = true
// linting issue
existing := existing
if err = netlink.AddrDel(l.Intf, &existing); err != nil {
return true, errors.Wrap(err, "could not delete ip")
err = l.WithInterface(func(intf netlink.Link) error {
addrs, listErr := netlink.AddrList(intf, netlink.FAMILY_ALL)
if listErr != nil {
return listErr
}
for _, existing := range addrs {
if existing.IP.String() == address {
found = true
existing := existing
if deleteErr := netlink.AddrDel(intf, &existing); deleteErr != nil {
return errors.Wrap(deleteErr, "could not delete ip")
}
}
}
}
return // Didn't find the address on the adapter
return nil
})
return found, err
}
func (configurator *network) SetMask(mask string) error {
configurator.mu.Lock()
defer configurator.mu.Unlock()
selectedMask := mask
var err error
@@ -1563,8 +1710,12 @@ func (configurator *network) SetMask(mask string) error {
return fmt.Errorf("no mask provided")
}
if configurator.IP() != "" {
selectedMask, err = SelectSubnet(configurator.IP(), mask)
ip := ""
if configurator.address != nil && configurator.address.IP != nil {
ip = configurator.address.IP.String()
}
if ip != "" {
selectedMask, err = SelectSubnet(ip, mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
@@ -1580,8 +1731,8 @@ func (configurator *network) SetMask(mask string) error {
size := DefaultMaskIPv4
family := utils.IPv4Family
if configurator.IP() != "" {
if utils.IsIPv6(configurator.IP()) {
if ip != "" {
if utils.IsIPv6(ip) {
size = DefaultMaskIPv6
family = utils.IPv6Family
}
@@ -1596,20 +1747,29 @@ func (configurator *network) SetMask(mask string) error {
return fmt.Errorf("failed to create mask /%d", m)
}
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.address.Mask = toSet
return nil
}
func (configurator *network) SetHasEndpoints(value bool) {
log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
configurator.mu.Lock()
defer configurator.mu.Unlock()
ip := ""
if configurator.address != nil && configurator.address.IP != nil {
ip = configurator.address.IP.String()
}
log.Debug("setting HasEndpoints", "ip", ip, "value", value)
configurator.hasEndpoints = value
}
func (configurator *network) HasEndpoints() bool {
log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
configurator.mu.RLock()
defer configurator.mu.RUnlock()
ip := ""
if configurator.address != nil && configurator.address.IP != nil {
ip = configurator.address.IP.String()
}
log.Debug("getting HasEndpoints", "ip", ip, "value", configurator.hasEndpoints)
return configurator.hasEndpoints
}
@@ -1618,14 +1778,20 @@ func (configurator *network) ARPName() string {
}
func (configurator *network) GetPossibleSubnets() string {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.possibleSubnets
}
func (configurator *network) DHCPFamily() string {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.dhcpFamily
}
func (configurator *network) IPVSMark() uint32 {
configurator.mu.RLock()
defer configurator.mu.RUnlock()
return configurator.ipvsMark
}

View File

@@ -0,0 +1,36 @@
//go:build linux
package vip
import (
"testing"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)
func TestAddIPPerCallDADSkipDoesNotPersist(t *testing.T) {
address, err := netlink.ParseAddr("2001:db8::10/128")
if err != nil {
t.Fatal(err)
}
interfaceManager := networkinterface.NewManager()
configurator := &network{
address: address,
link: interfaceManager.Get(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kube-vip-dad-test"}}),
}
// The netlink operation may fail without CAP_NET_ADMIN, but the address
// flags are set before that operation and are what this test exercises.
_, _ = configurator.AddIP(false, true)
if configurator.address.Flags&unix.IFA_F_NODAD == 0 {
t.Fatal("skipDAD=true did not set IFA_F_NODAD")
}
_, _ = configurator.AddIP(false, false)
if configurator.address.Flags&unix.IFA_F_NODAD != 0 {
t.Fatal("IFA_F_NODAD persisted into a normal AddIP call")
}
}

View File

@@ -0,0 +1,220 @@
//go:build linux
package vip
import (
"net"
"os"
"runtime"
"testing"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
const kubeVIPProtocol = 248
// requireNetworkNamespaces makes the privileged CI job fail instead of silently
// skipping when it cannot enter a network namespace.
var requireNetworkNamespaces = os.Getenv("KUBE_VIP_REQUIRE_NETNS") != ""
func TestAddressProtocolRoundTripsThroughNetlink(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
originalNamespace, err := netns.Get()
if err != nil {
t.Fatalf("getting current network namespace: %v", err)
}
defer originalNamespace.Close()
testNamespace, err := netns.New()
if err != nil {
if requireNetworkNamespaces {
t.Fatalf("creating isolated network namespace: %v", err)
}
t.Skipf("creating isolated network namespace: %v", err)
}
defer testNamespace.Close()
defer func() {
if err := netns.Set(originalNamespace); err != nil {
t.Errorf("restoring network namespace: %v", err)
}
}()
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto0"}}
if err := netlink.LinkAdd(link); err != nil {
t.Fatalf("creating test interface: %v", err)
}
if err := netlink.LinkSetUp(link); err != nil {
t.Fatalf("bringing test interface up: %v", err)
}
parsed, err := netlink.ParseAddr("192.0.2.10/32")
if err != nil {
t.Fatalf("parsing IPv4 address: %v", err)
}
markKubeVIPAddress(parsed, kubeVIPProtocol)
if err := netlink.AddrReplace(link, parsed); err != nil {
t.Fatalf("adding IPv4 address with protocol: %v", err)
}
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
t.Fatalf("listing addresses: %v", err)
}
for _, configured := range addresses {
if configured.IP.Equal(net.ParseIP("192.0.2.10")) {
if !IsKubeVIPAddress(configured, kubeVIPProtocol) {
t.Fatalf("configured address = %+v, want kube-vip protocol", configured)
}
return
}
}
t.Fatal("IPv4 address with kube-vip protocol was not configured")
}
func TestKubeVIPAddressProtocolRoundTripsThroughIPv6Netlink(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
originalNamespace, err := netns.Get()
if err != nil {
t.Fatalf("getting current network namespace: %v", err)
}
defer originalNamespace.Close()
testNamespace, err := netns.New()
if err != nil {
if requireNetworkNamespaces {
t.Fatalf("creating isolated network namespace: %v", err)
}
t.Skipf("creating isolated network namespace: %v", err)
}
defer testNamespace.Close()
defer func() {
if err := netns.Set(originalNamespace); err != nil {
t.Errorf("restoring network namespace: %v", err)
}
}()
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto1"}}
if err := netlink.LinkAdd(link); err != nil {
t.Fatalf("creating test interface: %v", err)
}
if err := netlink.LinkSetUp(link); err != nil {
t.Fatalf("bringing test interface up: %v", err)
}
address, err := netlink.ParseAddr("2001:db8::10/128")
if err != nil {
t.Fatalf("parsing IPv6 address: %v", err)
}
markKubeVIPAddress(address, kubeVIPProtocol)
if err := netlink.AddrReplace(link, address); err != nil {
t.Fatalf("adding IPv6 address with protocol: %v", err)
}
addresses, err := netlink.AddrList(link, netlink.FAMILY_V6)
if err != nil {
t.Fatalf("listing IPv6 addresses: %v", err)
}
for _, configured := range addresses {
if configured.IP.Equal(net.ParseIP("2001:db8::10")) {
if !IsKubeVIPAddress(configured, kubeVIPProtocol) {
t.Fatalf("configured IPv6 address = %+v, want kube-vip protocol", configured)
}
return
}
}
t.Fatalf("configured IPv6 addresses = %+v, want 2001:db8::10", addresses)
}
func TestCleanupKubeVIPAddressesRemovesOnlyUnretainedProtocolAddresses(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
originalNamespace, err := netns.Get()
if err != nil {
t.Fatalf("getting current network namespace: %v", err)
}
defer originalNamespace.Close()
testNamespace, err := netns.New()
if err != nil {
if requireNetworkNamespaces {
t.Fatalf("creating isolated network namespace: %v", err)
}
t.Skipf("creating isolated network namespace: %v", err)
}
defer testNamespace.Close()
defer func() {
if err := netns.Set(originalNamespace); err != nil {
t.Errorf("restoring network namespace: %v", err)
}
}()
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto2"}}
if err := netlink.LinkAdd(link); err != nil {
t.Fatalf("creating test interface: %v", err)
}
if err := netlink.LinkSetUp(link); err != nil {
t.Fatalf("bringing test interface up: %v", err)
}
for _, input := range []struct {
cidr string
protocol int
}{
{cidr: "192.0.2.10/32", protocol: kubeVIPProtocol},
{cidr: "192.0.2.11/32", protocol: kubeVIPProtocol},
{cidr: "192.0.2.12/32", protocol: 0},
{cidr: "192.0.2.13/32", protocol: kubeVIPProtocol + 1},
{cidr: "2001:db8::10/128", protocol: kubeVIPProtocol},
{cidr: "2001:db8::11/128", protocol: kubeVIPProtocol},
} {
address, err := netlink.ParseAddr(input.cidr)
if err != nil {
t.Fatalf("parsing address: %v", err)
}
markKubeVIPAddress(address, input.protocol)
if err := netlink.AddrReplace(link, address); err != nil {
t.Fatalf("adding address %s: %v", input.cidr, err)
}
}
retained, err := netlink.ParseAddr("192.0.2.10/32")
if err != nil {
t.Fatalf("parsing retained address: %v", err)
}
retained.LinkIndex = link.Attrs().Index
retainedIPv6, err := netlink.ParseAddr("2001:db8::10/128")
if err != nil {
t.Fatalf("parsing retained IPv6 address: %v", err)
}
retainedIPv6.LinkIndex = link.Attrs().Index
removed, err := CleanupKubeVIPAddresses(kubeVIPProtocol, map[string]struct{}{
addressKey(*retained): {},
addressKey(*retainedIPv6): {},
})
if err != nil {
t.Fatalf("cleaning kube-vip addresses: %v", err)
}
if removed != 2 {
t.Fatalf("removed = %d, want 2", removed)
}
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
t.Fatalf("listing remaining addresses: %v", err)
}
wantAddresses := map[string]bool{
"192.0.2.10": false,
"192.0.2.12": false,
"192.0.2.13": false,
"2001:db8::10": false,
}
for _, address := range addresses {
if _, wanted := wantAddresses[address.IP.String()]; wanted {
wantAddresses[address.IP.String()] = true
}
}
for address, found := range wantAddresses {
if !found {
t.Fatalf("remaining addresses = %+v, missing %s", addresses, address)
}
}
}

View File

@@ -2,8 +2,32 @@ package vip
import (
"testing"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
)
func TestNewConfigTagsStaticAndUpdatedAddresses(t *testing.T) {
const protocol = 248
networks, err := NewConfig("192.0.2.10", "lo", false, "32", false, "", false, false, 0, 0, protocol,
"", "", "", false, 0, false, networkinterface.NewManager(), false, false)
if err != nil {
t.Fatalf("NewConfig() error = %v", err)
}
configured, ok := networks[0].(*network)
if !ok {
t.Fatalf("network type = %T, want *network", networks[0])
}
if configured.address.Protocol != protocol {
t.Fatalf("static address protocol = %d, want %d", configured.address.Protocol, protocol)
}
if err := configured.SetIP("192.0.2.11"); err != nil {
t.Fatalf("SetIP() error = %v", err)
}
if configured.address.Protocol != protocol {
t.Fatalf("updated address protocol = %d, want %d", configured.address.Protocol, protocol)
}
}
func TestShouldSkipDAD(t *testing.T) {
cases := []struct {
name string

View File

@@ -27,12 +27,24 @@ type DHCPv4Client struct {
initRebootFlag bool
requestedIP net.IP
broadcastFlag bool
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
stopChan chan struct{} // is used by external clients to stop DHCP
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
backoffAttempts uint
stop sync.Once
stopOnce sync.Once
mtx sync.RWMutex
}
func (c *DHCPv4Client) storeLease(lease *nclient4.Lease) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.lease = lease
}
func (c *DHCPv4Client) loadLease() *nclient4.Lease {
c.mtx.RLock()
defer c.mtx.RUnlock()
return c.lease
}
// NewDHCPv4Client returns a new DHCP Client.
@@ -40,7 +52,6 @@ func NewDHCPv4Client(iface *net.Interface, initRebootFlag bool, requestedIP stri
return &DHCPv4Client{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
@@ -57,11 +68,13 @@ func (c *DHCPv4Client) WithHostName(hostname string) DHCPClient {
// Stop state-transition process and close dhcp client
func (c *DHCPv4Client) Stop() {
c.stop.Do(func() {
close(c.ipChan)
c.close()
}
func (c *DHCPv4Client) close() {
c.stopOnce.Do(func() {
close(c.stopChan)
})
<-c.releasedChan
}
// Gets the IPChannel for consumption
@@ -129,72 +142,93 @@ func (c *DHCPv4Client) ErrorChannel() chan error {
// ----------
// Figure: State-transition diagram for DHCP clients
func (c *DHCPv4Client) Start(ctx context.Context) error {
dhcpCtx, cancel := context.WithCancel(ctx)
defer cancel()
lease := c.requestWithBackoff(dhcpCtx)
lease, err := c.requestWithBackoff(ctx)
if err != nil {
return fmt.Errorf("DHCPv4 client failed: %w", err)
}
c.initRebootFlag = false
c.lease = lease
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
c.storeLease(lease)
// Set up two timers to renew/rebind regularly
t1Timeout, t2Timeout := getLeaseTimeouts(lease)
log.Debug("[DHCPv4] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
for {
select {
case <-c.stopChan:
return c.killProcessing(t1, t2)
case <-ctx.Done():
c.close()
return c.killProcessing(t1, t2)
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
lease, err := c.renew(dhcpCtx)
lease, err := c.renew(ctx)
if err == nil {
c.lease = lease
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
log.Info("[DHCPv4] renew", "lease", lease)
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv4] renew failed", "err", err)
}
t1.Reset(t1Timeout)
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(dhcpCtx, true)
lease, err := c.request(ctx, true)
if err == nil {
c.lease = lease
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
log.Info("[DHCPv4] rebind", "lease", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("[DHCPv4] rebind failed", "err", err)
}
log.Warn("[DHCPv4] ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
lease = c.loadLease()
log.Warn("[DHCPv4] ip may have changed", "ip", lease.ACK.YourIPAddr, "err", err)
c.initRebootFlag = false
c.lease = c.requestWithBackoff(dhcpCtx)
lease, backoffErr := c.requestWithBackoff(ctx)
if backoffErr != nil {
log.Error("[DHCPv4] failed to reacquire lease", "err", backoffErr)
continue
}
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// release is a unicast request of the IP release.
var err error
if err = c.release(); err != nil {
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
} else {
log.Info("[DHCPv4] release", "lease", lease)
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
func getLeaseTimeouts(lease *nclient4.Lease) (time.Duration, time.Duration) {
t1Timeout, t2Timeout := lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/2, (lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/8)*7
log.Debug("[DHCPv4] timeouts", "address", lease.ACK.YourIPAddr.String(), "T1", t1Timeout, "T2", t2Timeout)
return t1Timeout, t2Timeout
}
func (c *DHCPv4Client) killProcessing(t1, t2 *time.Timer) error {
// release is a unicast request of the IP release.
var err error
lease := c.loadLease()
if lease != nil {
if err = c.release(); err != nil {
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
} else {
log.Info("[DHCPv4] release", "lease", lease)
}
}
t1.Stop()
t2.Stop()
return err
}
// --------------------------------------------------------
// | |INIT-REBOOT | RENEWING |REBINDING |
// --------------------------------------------------------
@@ -204,7 +238,7 @@ func (c *DHCPv4Client) Start(ctx context.Context) error {
// |ciaddr |zero | IP address |IP address|
// --------------------------------------------------------
func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) *nclient4.Lease {
func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) (*nclient4.Lease, error) {
backoff := backoff.Backoff{
Factor: 2,
Jitter: true,
@@ -217,32 +251,47 @@ func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) *nclient4.Lease {
log.Debug("[DHCPv4]", "attempts", c.backoffAttempts)
RequestLoop:
for {
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
lease, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
c.errorChan <- errMsg
c.Stop()
return nil
select {
case <-ctx.Done():
return nil, fmt.Errorf("[DHCPv4] context error: %w", ctx.Err())
default:
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
lease, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, errMsg
}
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
t := time.NewTimer(dur)
select {
case <-t.C:
t.Stop()
case <-ctx.Done():
}
continue RequestLoop
}
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
backoff.Reset()
break RequestLoop
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv4] using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
// Nothing closes ipChan, so never block on a consumer that already stopped.
select {
case c.ipChan <- lease.ACK.YourIPAddr.String():
case <-c.stopChan:
case <-ctx.Done():
}
}
return lease
return lease, nil
}
func (c *DHCPv4Client) request(ctx context.Context, rebind bool) (*nclient4.Lease, error) {
@@ -289,7 +338,12 @@ func (c *DHCPv4Client) release() error {
defer dhclient.Close()
// TODO modify lease
return dhclient.Release(c.lease)
err = dhclient.Release(c.lease)
if err != nil {
return fmt.Errorf("DHCPv4 release failed: %w", err)
}
return nil
}
func (c *DHCPv4Client) renew(ctx context.Context) (*nclient4.Lease, error) {

View File

@@ -0,0 +1,40 @@
package vip
import (
"context"
"net"
"testing"
"time"
)
func TestDHCPv4BackoffExhaustionDoesNotDeadlock(t *testing.T) {
// DEFECT: requestWithBackoff calls Stop from the Start/request goroutine before Start can close releasedChan, so exhausted DHCPv4 retries deadlock (pkg/vip/dhcpv4.go:229).
client := NewDHCPv4Client(
&net.Interface{Name: "definitely-not-a-kube-vip-interface"},
false,
"",
1,
false,
)
done := make(chan struct{})
go func() {
_, _ = client.requestWithBackoff(context.Background())
close(done)
}()
select {
case err := <-client.ErrorChannel():
if err == nil {
t.Fatal("expected DHCPv4 request error")
}
case <-time.After(time.Second):
t.Fatal("DHCPv4 did not report exhausted backoff")
}
select {
case <-done:
case <-time.After(100 * time.Millisecond):
t.Fatal("DHCPv4 backoff exhaustion deadlocked")
}
}

View File

@@ -23,6 +23,9 @@ func init() {
}
type DHCPv6ClientManager struct {
// mu guards clients and the reference counts of its entries together, so a
// concurrent Add cannot join a client that Delete is already retiring.
mu sync.Mutex
clients map[string]*DHCPv6InternalClient
}
@@ -33,17 +36,17 @@ func NewDHCPv6ClientManager() *DHCPv6ClientManager {
}
func (m *DHCPv6ClientManager) Get(iface string) *DHCPv6InternalClient {
c, exists := m.clients[iface]
if !exists {
return nil
}
return c
m.mu.Lock()
defer m.mu.Unlock()
return m.clients[iface]
}
func (m *DHCPv6ClientManager) Add(iface string) (*DHCPv6InternalClient, error) {
c := m.Get(iface)
m.mu.Lock()
defer m.mu.Unlock()
if c != nil {
if c := m.clients[iface]; c != nil {
c.references.Add(1)
return c, nil
}
@@ -57,15 +60,16 @@ func (m *DHCPv6ClientManager) Add(iface string) (*DHCPv6InternalClient, error) {
}
func (m *DHCPv6ClientManager) Delete(iface string) {
c := m.Get(iface)
m.mu.Lock()
defer m.mu.Unlock()
if c != nil {
c.references.Add(-1)
ref := c.references.Load()
if ref < 1 {
c.client.Close()
delete(m.clients, iface)
}
c := m.clients[iface]
if c == nil {
return
}
if c.references.Add(-1) < 1 {
c.client.Close()
delete(m.clients, iface)
}
}
@@ -90,17 +94,30 @@ func NewDHCPv6InternalClient(iface string) (*DHCPv6InternalClient, error) {
type DHCPv6Client struct {
iface *net.Interface
managerKey string
ddnsHostName string
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
ic *DHCPv6InternalClient
addr *dhcpv6.OptIAAddress
backoffAttempts uint
stop sync.Once
mtx sync.RWMutex
}
func (c *DHCPv6Client) storeAddr(addr *dhcpv6.OptIAAddress) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.addr = addr
}
func (c *DHCPv6Client) loadAddr() *dhcpv6.OptIAAddress {
c.mtx.RLock()
defer c.mtx.RUnlock()
return c.addr
}
// NewDHCPv6Client returns a new DHCP6 Client.
@@ -117,8 +134,8 @@ func NewDHCPv6Client(iface *net.Interface, parent netlink.Link, initRebootFlag b
return &DHCPv6Client{
iface: iface,
managerKey: name,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
@@ -135,12 +152,15 @@ func (c *DHCPv6Client) WithHostName(hostname string) DHCPClient {
// Stop state-transition process and close dhcp client
func (c *DHCPv6Client) Stop() {
c.close()
}
// Close dhcp client channels
func (c *DHCPv6Client) close() {
c.stop.Do(func() {
close(c.ipChan)
close(c.stopChan)
})
<-c.releasedChan
dhcpv6ClientManager.Delete(c.iface.Name)
dhcpv6ClientManager.Delete(c.managerKey)
}
// Gets the IPChannel for consumption
@@ -154,27 +174,27 @@ func (c *DHCPv6Client) ErrorChannel() chan error {
}
func (c *DHCPv6Client) Start(ctx context.Context) error {
dhcpCtx, cancel := context.WithCancel(ctx)
defer cancel()
addr, err := c.requestWithBackoff(dhcpCtx)
addr, err := c.requestWithBackoff(ctx)
if err != nil {
return fmt.Errorf("DHCPv6 client failed: %w", err)
}
c.addr = addr
c.initRebootFlag = false
c.storeAddr(addr)
// Set up two ticker to renew/rebind regularly
t1Timeout := c.addr.PreferredLifetime / 2
t2Timeout := (c.addr.ValidLifetime / 8) * 7
log.Debug("[DHCPv6] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
t1Timeout, t2Timeout := getAddrTimeouts(addr)
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
for {
select {
case <-c.stopChan:
return c.killProcessing(t1, t2)
case <-ctx.Done():
c.close()
return c.killProcessing(t1, t2)
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
@@ -182,49 +202,65 @@ func (c *DHCPv6Client) Start(ctx context.Context) error {
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
addr, err := c.renew(dhcpCtx)
addr, err := c.renew(ctx)
if err == nil {
c.addr = addr
c.storeAddr(addr)
log.Info("[DHCPv6] renew", "addr", addr.IPv6Addr.String())
t1Timeout, t2Timeout = getAddrTimeouts(addr)
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv6] renew failed", "err", err)
}
t1.Reset(t1Timeout)
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
addr, err := c.request(dhcpCtx, true)
addr, err := c.request(ctx, true)
if err == nil {
c.addr = addr
c.storeAddr(addr)
log.Info("[DHCPv6] rebind", "lease", addr)
t1Timeout, t2Timeout = getAddrTimeouts(addr)
} else {
addr = c.loadAddr()
log.Warn("[DHCPv6] ip may have changed", "ip", addr.IPv6Addr.String(), "err", err)
c.initRebootFlag = false
c.addr, err = c.requestWithBackoff(dhcpCtx)
log.Error("[DHCPv6] rebind failed", "err", err)
addr, backoffErr := c.requestWithBackoff(ctx)
if backoffErr != nil {
log.Error("[DHCPv6] failed to reacquire lease", "err", backoffErr)
continue
}
c.storeAddr(addr)
t1Timeout, t2Timeout = getAddrTimeouts(addr)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// create new context for DHCP cleanup (independent)
dhcpStopCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// IP address release.
var err error
if err = c.release(dhcpStopCtx); err != nil {
log.Error("[DHCPv6] release failed", "err", err)
} else {
log.Info("[DHCPv6] released", "address", c.addr.String())
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
func getAddrTimeouts(addr *dhcpv6.OptIAAddress) (time.Duration, time.Duration) {
t1Timeout, t2Timeout := addr.PreferredLifetime/2, (addr.ValidLifetime/8)*7
log.Debug("[DHCPv6] timeouts", "address", addr.IPv6Addr.String(), "T1", t1Timeout, "T2", t2Timeout)
return t1Timeout, t2Timeout
}
func (c *DHCPv6Client) killProcessing(t1, t2 *time.Timer) error {
// create new context for DHCP cleanup (independent)
dhcpStopCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// IP address release.
var err error
if c.loadAddr() != nil {
if err = c.release(dhcpStopCtx); err != nil {
log.Error("[DHCPv6] release failed", "err", err)
} else {
log.Info("[DHCPv6] released", "address", c.addr.String())
}
}
t1.Stop()
t2.Stop()
return err
}
func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAddress, error) {
backoff := backoff.Backoff{
Factor: 2,
@@ -233,34 +269,50 @@ func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAdd
Max: 1 * time.Minute,
}
var err error
var addr *dhcpv6.OptIAAddress
var err error
log.Debug("[DHCPv6]", "attempts", c.backoffAttempts)
RequestLoop:
for {
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
select {
case <-ctx.Done():
return nil, fmt.Errorf("[DHCPv6] context error: %w", ctx.Err())
default:
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
addr, err = c.request(ctx, false)
addr, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
c.errorChan <- errMsg
c.Stop()
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv6 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
}
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
t := time.NewTimer(dur)
select {
case <-t.C:
t.Stop()
case <-ctx.Done():
}
continue RequestLoop
}
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
backoff.Reset()
break RequestLoop
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv6] using channel")
c.ipChan <- addr.IPv6Addr.String()
// Nothing closes ipChan, so never block on a consumer that already stopped.
select {
case c.ipChan <- addr.IPv6Addr.String():
case <-c.stopChan:
case <-ctx.Done():
}
}
return addr, nil
@@ -385,9 +437,10 @@ func getAddress(iana []*dhcpv6.OptIANA) (*dhcpv6.OptIAAddress, error) {
return nil, fmt.Errorf("failed to get IANA")
}
if len(iana) < 1 {
return nil, fmt.Errorf("failed to get addresses data")
addrs := iana[0].Options.Addresses()
if len(addrs) == 0 {
return nil, fmt.Errorf("IANA contained no addresses")
}
return iana[0].Options.Addresses()[0], nil
return addrs[0], nil
}

View File

@@ -0,0 +1,83 @@
package vip
import (
"net"
"sync"
"sync/atomic"
"testing"
"github.com/insomniacslk/dhcp/dhcpv6"
)
func TestDHCPv6StopReleasesManagerReferenceForParentInterface(t *testing.T) {
// DEFECT: Stop deletes the manager entry using the VLAN child name even though NewDHCPv6Client keyed the shared client by its parent name (pkg/vip/dhcpv6.go:143).
previousManager := dhcpv6ClientManager
t.Cleanup(func() { dhcpv6ClientManager = previousManager })
references := &atomic.Int32{}
references.Store(2)
shared := &DHCPv6InternalClient{references: references}
dhcpv6ClientManager = &DHCPv6ClientManager{
clients: map[string]*DHCPv6InternalClient{"parent0": shared},
}
client := &DHCPv6Client{
iface: &net.Interface{Name: "vlan-child"},
managerKey: "parent0",
ipChan: make(chan string),
stopChan: make(chan struct{}),
ic: shared,
addr: &dhcpv6.OptIAAddress{},
}
client.Stop()
if got := references.Load(); got != 1 {
t.Fatalf("manager reference count = %d, want 1 after stopping one VLAN client", got)
}
}
func TestDHCPv6ClientManagerSharesOneClientPerParentInterface(t *testing.T) {
references := &atomic.Int32{}
references.Store(1)
shared := &DHCPv6InternalClient{references: references}
manager := &DHCPv6ClientManager{
clients: map[string]*DHCPv6InternalClient{"parent0": shared},
}
var wg sync.WaitGroup
for range 64 {
wg.Go(func() {
client, err := manager.Add("parent0")
if err != nil {
t.Errorf("Add() error = %v", err)
return
}
if client != shared {
t.Errorf("Add() client = %p, want the shared client %p", client, shared)
}
manager.Delete("parent0")
})
}
wg.Wait()
if got := manager.Get("parent0"); got != shared {
t.Fatalf("shared client = %v, want it retained while still referenced", got)
}
if got := references.Load(); got != 1 {
t.Fatalf("manager reference count = %d, want 1", got)
}
}
func TestGetAddressRejectsIANAWithoutAddresses(t *testing.T) {
// DEFECT: getAddress indexes the first IAADDR without checking whether the IANA contains one, so a malformed/expired reply panics (pkg/vip/dhcpv6.go:392).
defer func() {
if recovered := recover(); recovered != nil {
t.Fatalf("getAddress panicked on an IANA without IAADDR: %v", recovered)
}
}()
if _, err := getAddress([]*dhcpv6.OptIANA{{}}); err == nil {
t.Fatal("getAddress accepted an IANA without an IAADDR")
}
}

View File

@@ -7,6 +7,7 @@ import (
"net"
"strings"
"syscall"
"time"
log "log/slog"
@@ -14,6 +15,8 @@ import (
"github.com/vishvananda/netlink"
)
var errDefaultInterfaceSubscriptionClosed = errors.New("default interface subscription closed")
// getHostName return the hostname from the fqdn
func getHostName(dnsName string) string {
if dnsName == "" {
@@ -75,26 +78,123 @@ func getDefaultRoute(family int) (*net.Interface, error) {
return nil, errors.New("default route not found")
}
// MonitorDefaultInterface monitor the default interface and catch the event of the default route
// MonitorDefaultInterface monitors the default interface for route removal or link loss.
func MonitorDefaultInterface(ctx context.Context, defaultIF *net.Interface) error {
routeCh := make(chan netlink.RouteUpdate)
return monitorDefaultInterfaceWithRetry(ctx, defaultIF, subscribeDefaultInterface, GetDefaultGatewayInterface, 100*time.Millisecond)
}
func monitorDefaultInterfaceWithRetry(ctx context.Context, defaultIF *net.Interface,
subscribe func(context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error),
lookup func() (*net.Interface, error), retryDelay time.Duration) error {
for {
monitorCtx, cancel := context.WithCancel(ctx)
routeCh, linkCh, err := subscribe(monitorCtx)
if err == nil {
err = monitorDefaultInterface(monitorCtx, defaultIF, routeCh, linkCh)
}
cancel()
drainDefaultInterfaceSubscriptions(routeCh, linkCh)
if ctx.Err() != nil {
return nil
}
if err != nil && !errors.Is(err, errDefaultInterfaceSubscriptionClosed) {
log.Warn("default interface subscription failed, retrying", "err", err)
} else if err == nil {
return nil
}
if refreshed, lookupErr := lookup(); lookupErr == nil {
defaultIF = refreshed
} else {
log.Warn("failed to refresh default interface while resubscribing", "err", lookupErr)
}
timer := time.NewTimer(retryDelay)
select {
case <-ctx.Done():
timer.Stop()
return nil
case <-timer.C:
}
}
}
func subscribeDefaultInterface(ctx context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error) {
const subscriptionBuffer = 64
routeCh := make(chan netlink.RouteUpdate, subscriptionBuffer)
if err := netlink.RouteSubscribe(routeCh, ctx.Done()); err != nil {
return fmt.Errorf("subscribe route failed, error: %w", err)
return nil, nil, fmt.Errorf("subscribe route failed, error: %w", err)
}
linkCh := make(chan netlink.LinkUpdate, subscriptionBuffer)
if err := netlink.LinkSubscribe(linkCh, ctx.Done()); err != nil {
return routeCh, nil, fmt.Errorf("subscribe link failed, error: %w", err)
}
return routeCh, linkCh, nil
}
func monitorDefaultInterface(ctx context.Context, defaultIF *net.Interface, routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) error {
for {
select {
case r := <-routeCh:
case r, ok := <-routeCh:
if !ok {
return subscriptionClosed(ctx, "route")
}
log.Debug(fmt.Sprintf("type: %d, route: %+v", r.Type, r.Route))
if r.Type == syscall.RTM_DELROUTE && (r.Dst == nil || r.Dst.String() == "0.0.0.0/0") && r.LinkIndex == defaultIF.Index {
if r.Type == syscall.RTM_DELROUTE && isDefaultRoute(r.Dst) && r.LinkIndex == defaultIF.Index {
return fmt.Errorf("default route deleted and the default interface may be invalid")
}
case update, ok := <-linkCh:
if !ok {
return subscriptionClosed(ctx, "link")
}
if update.Link == nil {
continue
}
attrs := update.Attrs()
if attrs != nil && attrs.Index == defaultIF.Index && attrs.Flags&net.FlagUp == 0 {
return fmt.Errorf("default interface %q is down", defaultIF.Name)
}
case <-ctx.Done():
return nil
}
}
}
func subscriptionClosed(ctx context.Context, subscription string) error {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("%w: %s subscription closed", errDefaultInterfaceSubscriptionClosed, subscription)
}
// isDefaultRoute accepts both families, matching the selection made by
// GetDefaultGatewayInterface.
func isDefaultRoute(dst *net.IPNet) bool {
if dst == nil {
return true
}
return dst.String() == "0.0.0.0/0" || dst.String() == "::/0"
}
func drainDefaultInterfaceSubscriptions(routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) {
timer := time.NewTimer(100 * time.Millisecond)
defer timer.Stop()
for routeCh != nil || linkCh != nil {
select {
case _, ok := <-routeCh:
if !ok {
routeCh = nil
}
case _, ok := <-linkCh:
if !ok {
linkCh = nil
}
case <-timer.C:
return
}
}
}
func GenerateMac() (mac string) {
buf := make([]byte, 3)
_, err := rand.Read(buf)

293
pkg/vip/util_linux_test.go Normal file
View File

@@ -0,0 +1,293 @@
//go:build linux
package vip
import (
"context"
"net"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
func TestMonitorDefaultInterfaceDetectsDefaultRouteDeletionPerFamily(t *testing.T) {
for _, test := range []struct {
name string
cidr string
}{
{name: "IPv4", cidr: "0.0.0.0/0"},
{name: "IPv6", cidr: "::/0"},
} {
t.Run(test.name, func(t *testing.T) {
defaultIF := &net.Interface{Index: 7, Name: "test0"}
_, defaultRoute, err := net.ParseCIDR(test.cidr)
if err != nil {
t.Fatalf("ParseCIDR() error = %v", err)
}
routeCh := make(chan netlink.RouteUpdate, 1)
routeCh <- netlink.RouteUpdate{
Type: syscall.RTM_DELROUTE,
Route: netlink.Route{Dst: defaultRoute, LinkIndex: defaultIF.Index},
}
linkCh := make(chan netlink.LinkUpdate)
err = monitorDefaultInterfaceForTest(t, context.Background(), defaultIF, routeCh, linkCh)
if err == nil || !strings.Contains(err.Error(), "default route deleted") {
t.Fatalf("monitor error = %v, want a default route deletion error", err)
}
})
}
}
func TestMonitorDefaultInterfaceReturnsErrorWhenLinkGoesDown(t *testing.T) {
defaultIF := &net.Interface{Index: 7, Name: "test0"}
routeCh := make(chan netlink.RouteUpdate)
linkCh := make(chan netlink.LinkUpdate, 1)
linkCh <- netlink.LinkUpdate{
Link: &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Index: defaultIF.Index}},
}
err := monitorDefaultInterfaceForTest(t, context.Background(), defaultIF, routeCh, linkCh)
if err == nil {
t.Fatal("expected an error when the default interface goes down")
}
if !strings.Contains(err.Error(), "default interface \"test0\" is down") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestMonitorDefaultInterfaceHandlesClosedSubscriptions(t *testing.T) {
for _, test := range []struct {
name string
closeRoute bool
cancel bool
wantErr string
}{
{name: "closed route subscription", closeRoute: true, wantErr: "route subscription closed"},
{name: "closed link subscription", wantErr: "link subscription closed"},
{name: "context cancellation with closed route subscription", closeRoute: true, cancel: true},
{name: "context cancellation with closed link subscription", cancel: true},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
routeCh := make(chan netlink.RouteUpdate)
linkCh := make(chan netlink.LinkUpdate)
if test.closeRoute {
close(routeCh)
} else {
close(linkCh)
}
if test.cancel {
cancel()
}
err := monitorDefaultInterfaceForTest(t, ctx, &net.Interface{}, routeCh, linkCh)
if test.wantErr == "" {
if err != nil {
t.Fatalf("monitor error = %v, want nil", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("monitor error = %v, want %q", err, test.wantErr)
}
})
}
}
func monitorDefaultInterfaceForTest(t *testing.T, ctx context.Context, defaultIF *net.Interface,
routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) error {
t.Helper()
errCh := make(chan error, 1)
go func() {
errCh <- monitorDefaultInterface(ctx, defaultIF, routeCh, linkCh)
}()
select {
case err := <-errCh:
return err
case <-time.After(time.Second):
t.Fatal("default interface monitor did not return")
return nil
}
}
func TestMonitorDefaultInterfaceHandlesClosedSubscriptionsAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
routeCh := make(chan netlink.RouteUpdate)
linkCh := make(chan netlink.LinkUpdate)
errCh := make(chan error, 1)
go func() {
errCh <- monitorDefaultInterface(ctx, &net.Interface{}, routeCh, linkCh)
}()
cancel()
close(routeCh)
close(linkCh)
select {
case err := <-errCh:
if err != nil {
t.Fatalf("monitor error = %v, want nil", err)
}
case <-time.After(time.Second):
t.Fatal("monitor did not stop after cancellation")
}
}
func TestMonitorDefaultInterfaceRetriesClosedSubscription(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
secondSubscribed := make(chan struct{})
attempts := 0
subscribe := func(ctx context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
attempts++
routeCh := make(chan netlink.RouteUpdate)
linkCh := make(chan netlink.LinkUpdate)
if attempts == 1 {
close(routeCh)
close(linkCh)
return routeCh, linkCh, nil
}
close(secondSubscribed)
go func() {
<-ctx.Done()
close(routeCh)
close(linkCh)
}()
return routeCh, linkCh, nil
}
lookup := func() (*net.Interface, error) {
return &net.Interface{Index: 2, Name: "refreshed"}, nil
}
done := make(chan error, 1)
go func() {
done <- monitorDefaultInterfaceWithRetry(ctx, &net.Interface{Index: 1, Name: "original"}, subscribe, lookup, time.Millisecond)
}()
select {
case <-secondSubscribed:
case <-time.After(time.Second):
t.Fatal("monitor did not resubscribe after channel closure")
}
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("monitor returned an error after cancellation: %v", err)
}
case <-time.After(time.Second):
t.Fatal("resubscribed monitor did not stop after cancellation")
}
if attempts != 2 {
t.Fatalf("subscription attempts = %d, want 2", attempts)
}
}
func TestMonitorDefaultInterfaceIgnoresNilLink(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
routeCh := make(chan netlink.RouteUpdate)
linkCh := make(chan netlink.LinkUpdate)
errCh := make(chan error, 1)
go func() {
errCh <- monitorDefaultInterface(ctx, &net.Interface{}, routeCh, linkCh)
}()
linkCh <- netlink.LinkUpdate{}
cancel()
select {
case err := <-errCh:
if err != nil {
t.Fatalf("monitor error = %v, want nil", err)
}
case <-time.After(time.Second):
t.Fatal("monitor did not stop after nil link update")
}
}
func TestDrainDefaultInterfaceSubscriptionsIsBounded(t *testing.T) {
done := make(chan struct{})
go func() {
drainDefaultInterfaceSubscriptions(make(chan netlink.RouteUpdate), make(chan netlink.LinkUpdate))
close(done)
}()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("subscription drain did not stop")
}
}
func TestMonitorDefaultInterfaceReturnsErrorWhenTestLinkIsSetDown(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
originalNS, err := netns.Get()
if err != nil {
t.Fatalf("getting current network namespace: %v", err)
}
defer originalNS.Close()
testNS, err := netns.New()
if err != nil {
if requireNetworkNamespaces {
t.Fatalf("creating isolated network namespace: %v", err)
}
t.Skipf("creating isolated network namespace: %v", err)
}
defer testNS.Close()
defer func() {
if err := netns.Set(originalNS); err != nil {
t.Errorf("restoring network namespace: %v", err)
}
}()
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kv-monitor0"}}
if err := netlink.LinkAdd(link); err != nil {
t.Fatalf("creating test interface: %v", err)
}
defer func() {
if err := netlink.LinkDel(link); err != nil {
t.Errorf("deleting test interface: %v", err)
}
}()
if err := netlink.LinkSetUp(link); err != nil {
t.Fatalf("bringing test interface up: %v", err)
}
defaultIF, err := net.InterfaceByName(link.Attrs().Name)
if err != nil {
t.Fatalf("getting test interface: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
routeCh, linkCh, err := subscribeDefaultInterface(ctx)
if err != nil {
t.Fatalf("subscribing to link updates: %v", err)
}
errCh := make(chan error, 1)
go func() {
errCh <- monitorDefaultInterface(ctx, defaultIF, routeCh, linkCh)
}()
if err := netlink.LinkSetDown(link); err != nil {
t.Fatalf("bringing test interface down: %v", err)
}
select {
case err := <-errCh:
if err == nil || !strings.Contains(err.Error(), "default interface \"kv-monitor0\" is down") {
t.Fatalf("monitor error = %v, want default-interface-down error", err)
}
case <-time.After(time.Second):
t.Fatal("monitor did not report the interface going down")
}
}

View File

@@ -214,6 +214,10 @@ func ResolveVIP(ctx context.Context, c api.GoBgpServiceClient, vip string) []str
// This counts Destination objects (used by the existing service BGP tests).
func CheckPaths(ctx context.Context, c api.GoBgpServiceClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedDests int) []*api.Destination {
var paths []*api.Destination
// 120s: with ginkgo --procs=4, sibling processes create Kind clusters and load
// images concurrently, which can delay kube-vip's endpoint-watch/advertise loop
// well past 30s (observed >13s even in passing specs). Matches the timeout the
// BGP health-check suite uses for route re-announcement.
Eventually(func() error {
var err error
paths, err = ListPaths(ctx, c, family, prefixes)
@@ -224,7 +228,7 @@ func CheckPaths(ctx context.Context, c api.GoBgpServiceClient, family *api.Famil
return fmt.Errorf("expected %d destinations, found %d", expectedDests, len(paths))
}
return nil
}, "30s", "1s").ShouldNot(HaveOccurred(), "should have %d destinations, found %d", expectedDests, len(paths))
}, "120s", "1s").ShouldNot(HaveOccurred(), "should have %d destinations, found %d", expectedDests, len(paths))
return paths
}

View File

@@ -20,9 +20,11 @@ import (
"github.com/kube-vip/kube-vip/testing/e2e"
)
var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func() {
var _ = Describe("kube-vip BGP when deployed as a regular pod", func() {
if Mode == ModeBGP {
var (
ctx context.Context
cancel context.CancelFunc
logger log.Logger
imagePath string
k8sImagePath string
@@ -30,9 +32,8 @@ var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func()
tempDirPathRoot string
)
ctx, cancel := context.WithCancel(context.TODO())
BeforeEach(func() {
BeforeEach(OncePerOrdered, func() {
ctx, cancel = context.WithCancel(context.TODO())
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
@@ -44,11 +45,11 @@ var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func()
}
})
BeforeAll(func() {
BeforeEach(OncePerOrdered, func() {
tempDirPathRoot = MustMkdirTemp("", fmt.Sprintf("%s-bgp-ds", testDirPrefix))
})
AfterAll(func() {
AfterEach(OncePerOrdered, func() {
if os.Getenv("E2E_KEEP_LOGS") != "true" {
Expect(os.RemoveAll(tempDirPathRoot)).To(Succeed())
}

Some files were not shown because too many files have changed in this diff Show More