mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/kube-vip/kube-vip.git
synced 2026-09-20 08:03:47 +08:00
Compare commits
99 Commits
fix/logrus
...
c925fb7cd9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c925fb7cd9 | ||
|
|
d6129c8299 | ||
|
|
3d73039cce | ||
|
|
a60f80b1b2 | ||
|
|
d99186480c | ||
|
|
d557211b55 | ||
|
|
e4eb5e8e7f | ||
|
|
a7b068f9d2 | ||
|
|
a6bf5280d4 | ||
|
|
61a52e8f2a | ||
|
|
7e2dd0d262 | ||
|
|
7a085cb3ce | ||
|
|
1f5c135fd1 | ||
|
|
b4771c5319 | ||
|
|
3771ccee29 | ||
|
|
ee64dceb36 | ||
|
|
38fabeba9e | ||
|
|
7ef1899567 | ||
|
|
637c3da47e | ||
|
|
e666a0cdd1 | ||
|
|
9a2142c028 | ||
|
|
0606e9477c | ||
|
|
8a968618bf | ||
|
|
5cd138a85b | ||
|
|
246a786fe2 | ||
|
|
6ee3024bc6 | ||
|
|
2bc2df53fc | ||
|
|
618904dea3 | ||
|
|
b116d5a469 | ||
|
|
6cbf5aaeda | ||
|
|
dc453f07fc | ||
|
|
14b2f51aba | ||
|
|
01e8fbc3e3 | ||
|
|
4504649c91 | ||
|
|
47f4e75183 | ||
|
|
f589a18bd9 | ||
|
|
d79f3ddb52 | ||
|
|
1b25ce7d0e | ||
|
|
96c4406d63 | ||
|
|
a51944b89d | ||
|
|
a36dc36947 | ||
|
|
47b546073a | ||
|
|
2b126dceed | ||
|
|
110d34b844 | ||
|
|
5388fed92b | ||
|
|
487859e76f | ||
|
|
0e57deef3d | ||
|
|
b25badd185 | ||
|
|
77fa726c99 | ||
|
|
920a0182cb | ||
|
|
8d043de910 | ||
|
|
b0b12cfc13 | ||
|
|
b034c81bef | ||
|
|
ca9640227a | ||
|
|
5276f0123f | ||
|
|
c2e5be6d6a | ||
|
|
a8293f66e4 | ||
|
|
9d72f43f62 | ||
|
|
3c3096d89d | ||
|
|
5d5c893501 | ||
|
|
d417a0c8e8 | ||
|
|
825ffdb20c | ||
|
|
fe3a379f2d | ||
|
|
c42a207c26 | ||
|
|
d6e5753464 | ||
|
|
5131b92810 | ||
|
|
e1fd9ac3e3 | ||
|
|
1831a05525 | ||
|
|
19198d47fa | ||
|
|
1af388ff9f | ||
|
|
d1ff5f2952 | ||
|
|
54881a117c | ||
|
|
426a409a5d | ||
|
|
5871bec56c | ||
|
|
b7f3379514 | ||
|
|
a15745c442 | ||
|
|
b684eed5a4 | ||
|
|
0bdd6a9015 | ||
|
|
b864abf27d | ||
|
|
6fa38027e2 | ||
|
|
b478234d29 | ||
|
|
03241ee27f | ||
|
|
6e8b391685 | ||
|
|
e5ff483a23 | ||
|
|
bee5cfe4a2 | ||
|
|
81d54050c7 | ||
|
|
5e9fcf642c | ||
|
|
83797e4da1 | ||
|
|
ed28c49f48 | ||
|
|
42ba1fefb2 | ||
|
|
988eb0994a | ||
|
|
90a3892271 | ||
|
|
c553663654 | ||
|
|
b6151a4454 | ||
|
|
6f69d4511f | ||
|
|
5e2220fd4d | ||
|
|
4e13a81af0 | ||
|
|
a19500b116 | ||
|
|
ea916c5a31 |
13
.github/suggestion-comment.md
vendored
Normal file
13
.github/suggestion-comment.md
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
I'll help you add a suggestion. Unfortunately, I can't directly add a suggestion to an existing comment through the API. However, here's what I recommend:
|
||||
|
||||
**Option 1: Reply with a suggestion**
|
||||
Create a new comment with a suggested fix:
|
||||
|
||||
```suggestion
|
||||
failed to get an IPv6 address after %d attempt(s), giving up, error: %s
|
||||
```
|
||||
|
||||
**Option 2: Edit your existing comment**
|
||||
Update your comment to include the suggestion details pointing out that line 284 in the error message says "IPv4" but should say "IPv6" since this is the DHCPv6Client.
|
||||
|
||||
Would you like me to create a new reply comment with the suggestion instead?
|
||||
2
.github/workflows/anchore-syft.yml
vendored
2
.github/workflows/anchore-syft.yml
vendored
@@ -26,6 +26,6 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
- name: Anchore SBOM Action
|
||||
uses: anchore/sbom-action@v0.24.0
|
||||
uses: anchore/sbom-action@v0.24.2
|
||||
with:
|
||||
format: cyclonedx-json
|
||||
|
||||
41
.github/workflows/ci-pull-request.yaml
vendored
41
.github/workflows/ci-pull-request.yaml
vendored
@@ -1,18 +1,33 @@
|
||||
name: For each PR
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E tests
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
GINKGO_PROCS: ${{ matrix.ginkgo-procs }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 3
|
||||
matrix:
|
||||
mode: ["arp", "rt", "bgp"]
|
||||
fail-fast: true
|
||||
include:
|
||||
- mode: arp
|
||||
ginkgo-procs: 4
|
||||
- mode: rt
|
||||
ginkgo-procs: 4
|
||||
- mode: bgp
|
||||
ginkgo-procs: 4
|
||||
steps:
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
|
||||
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
|
||||
- name: Ensure fs wont cause issues
|
||||
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
- name: Checkout code
|
||||
@@ -25,21 +40,22 @@ jobs:
|
||||
run: make dockerx86Local
|
||||
- name: Run Manifest generation tests
|
||||
run: make manifest-test
|
||||
if: matrix.mode == 'arp'
|
||||
- name: Run ARP mode tests
|
||||
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-arp
|
||||
if: matrix.mode== 'arp'
|
||||
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-arp.json" make e2e-tests-arp
|
||||
if: matrix.mode == 'arp'
|
||||
- name: Run RT mode tests
|
||||
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-rt
|
||||
if: matrix.mode== 'rt'
|
||||
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-rt.json" make e2e-tests-rt
|
||||
if: matrix.mode == 'rt'
|
||||
- name: Get GoBGP binaries
|
||||
run: make get-gobgp
|
||||
if: matrix.mode== 'bgp'
|
||||
if: matrix.mode == 'bgp'
|
||||
- name: Run BGP mode tests
|
||||
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-bgp
|
||||
if: matrix.mode== 'bgp'
|
||||
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-bgp.json" make e2e-tests-bgp
|
||||
if: matrix.mode == 'bgp'
|
||||
- name: Change log directory permissions
|
||||
run: sudo chmod -R 755 /tmp/kube-vip-test*
|
||||
if: matrix.mode== 'bgp' && always()
|
||||
if: matrix.mode == 'bgp' && always()
|
||||
- name: Save logs
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -49,10 +65,11 @@ jobs:
|
||||
service-e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E service tests
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
|
||||
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
- name: Install Go
|
||||
|
||||
21
.github/workflows/ci.yaml
vendored
21
.github/workflows/ci.yaml
vendored
@@ -1,11 +1,19 @@
|
||||
name: For each commit
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
validation:
|
||||
runs-on: ubuntu-latest
|
||||
name: Checks and linters
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Init
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
@@ -20,11 +28,20 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Verify gofmt
|
||||
run: |
|
||||
unformatted=$(gofmt -l .)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "The following files are not gofmt-formatted:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
- name: All checks
|
||||
run: make check
|
||||
unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: Unit tests
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
@@ -37,6 +54,7 @@ jobs:
|
||||
integration-tests:
|
||||
name: Integration tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
@@ -49,10 +67,11 @@ jobs:
|
||||
image-vul-check:
|
||||
runs-on: ubuntu-latest
|
||||
name: Image vulnerability scan
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
- name: Build image with iptables
|
||||
- name: Build image
|
||||
run: make dockerx86Action
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
|
||||
111
.github/workflows/nightly-e2e.yaml
vendored
Normal file
111
.github/workflows/nightly-e2e.yaml
vendored
Normal file
@@ -0,0 +1,111 @@
|
||||
name: Nightly e2e
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 2 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
unit-coverage:
|
||||
runs-on: ubuntu-latest
|
||||
name: Unit tests with coverage
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Run tests
|
||||
run: make unit-tests
|
||||
- name: Summarize coverage
|
||||
if: always()
|
||||
run: |
|
||||
if test -f coverage.out; then
|
||||
echo "### Unit coverage" >> "$GITHUB_STEP_SUMMARY"
|
||||
go tool cover -func=coverage.out | tail -1 >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "### Unit coverage: report missing" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
- name: Upload coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unit-coverage
|
||||
path: coverage.out
|
||||
if-no-files-found: error
|
||||
if: always()
|
||||
etcd-e2e:
|
||||
runs-on: ubuntu-latest
|
||||
name: Etcd E2E tests
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Ensure fs wont cause issues
|
||||
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Build image locally
|
||||
run: make dockerx86Local
|
||||
- name: Prepare Etcd artifacts
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/kube-vip-etcd-artifacts
|
||||
: > /tmp/kube-vip-etcd-artifacts/suite.log
|
||||
printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
|
||||
- name: Run Etcd tests
|
||||
id: etcd
|
||||
# Scheduled failures are tolerated only during the initial stabilization window.
|
||||
# The enforcement step below makes manual runs and later schedules blocking.
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: |
|
||||
set +e
|
||||
set -o pipefail
|
||||
DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
|
||||
GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
|
||||
make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
|
||||
exit "$exit_code"
|
||||
- name: Summarize Etcd suite
|
||||
if: always()
|
||||
env:
|
||||
OUTCOME: ${{ steps.etcd.outcome }}
|
||||
EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
|
||||
run: |
|
||||
echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
|
||||
printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
|
||||
"${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
|
||||
> /tmp/kube-vip-etcd-artifacts/result.json
|
||||
- name: Save logs
|
||||
uses: actions/upload-artifact@v7
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: etcd-e2e-logs
|
||||
path: |
|
||||
/tmp/kube-vip-etcd-artifacts
|
||||
/tmp/kube-vip-test*
|
||||
if-no-files-found: warn
|
||||
if: always()
|
||||
- name: Enforce Etcd result
|
||||
if: always()
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
OUTCOME: ${{ steps.etcd.outcome }}
|
||||
run: |
|
||||
if test "$OUTCOME" = success; then
|
||||
exit 0
|
||||
fi
|
||||
if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
|
||||
echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
|
||||
exit 1
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -3,6 +3,7 @@ kube-vip
|
||||
.vscode
|
||||
bin
|
||||
testing/e2e/etcd/certs
|
||||
coverage.out
|
||||
pkg/etcd/etcd.pid
|
||||
pkg/etcd/etcd-data
|
||||
testing/e2e/e2e.test
|
||||
|
||||
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- Propagate `bgp_attach_ip_to_interface` into per-service config so it attaches BGP-mode Service VIPs to the interface as configured.
|
||||
- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
|
||||
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
|
||||
- Reintroduce BGP config via node annotations. Fixes #1488.
|
||||
@@ -50,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Added support in ipoib interfaces in ARP mode. Fixes #694
|
||||
|
||||
### Changed
|
||||
- BGP mode now honours `enable_leader_election` for services: a single global services leader advertises the service VIPs instead of every node advertising them. Deployments that enabled `enable_leader_election` for the control plane and relied on ECMP/multipath for services must unset it (or switch to `enable_service_election`) to keep the previous datapath. kube-vip logs a warning on startup when this path is taken.
|
||||
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
|
||||
- wireguard.go now manages a complete wireguard interface on the current network namespace
|
||||
- manager_wireguard.go uses the new wireguard.go implementation
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:experimental
|
||||
|
||||
FROM golang:1.26.6-alpine3.23 as dev
|
||||
FROM golang:1.27.1-alpine3.23 as dev
|
||||
RUN apk add --no-cache git ca-certificates make
|
||||
RUN adduser -D appuser
|
||||
COPY . /src/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:experimental
|
||||
|
||||
FROM golang:1.26.6-alpine3.23 as dev
|
||||
FROM golang:1.27.1-alpine3.23 as dev
|
||||
RUN apk add --no-cache git make
|
||||
RUN adduser -D appuser
|
||||
COPY . /src/
|
||||
|
||||
29
Makefile
29
Makefile
@@ -5,7 +5,7 @@ TARGET := kube-vip
|
||||
.DEFAULT_GOAL := $(TARGET)
|
||||
|
||||
# These will be provided to the target
|
||||
VERSION := v1.2.3
|
||||
VERSION := v1.2.4
|
||||
|
||||
BUILD := `git rev-parse HEAD`
|
||||
|
||||
@@ -16,10 +16,14 @@ TARGETOS=linux
|
||||
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
|
||||
DOCKERTAG ?= $(VERSION)
|
||||
REPOSITORY ?= docker.io/plndr
|
||||
GO_VERSION := 1.25.6
|
||||
GO_VERSION := $(word 2,$(shell grep '^go ' go.mod))
|
||||
K8S_VERSION ?= v1.35.0
|
||||
GINKGO_ARGS ?=
|
||||
GINKGO_PROCS ?=
|
||||
GINKGO_PARALLEL := $(if $(GINKGO_PROCS),--procs=$(GINKGO_PROCS),-p)
|
||||
BUILDX_CACHE_FLAGS ?=
|
||||
|
||||
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker
|
||||
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker e2e-tests-etcd
|
||||
|
||||
all: check install
|
||||
|
||||
@@ -77,17 +81,17 @@ docker:
|
||||
# This will build a local docker image (x86 only), use make dockerLocal for all architectures
|
||||
dockerx86Local:
|
||||
@-rm ./kube-vip
|
||||
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
|
||||
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) $(BUILDX_CACHE_FLAGS) .
|
||||
@echo New Multi Architecture Docker image created
|
||||
|
||||
dockerx86Action:
|
||||
@-rm ./kube-vip
|
||||
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
|
||||
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
|
||||
@echo New Multi Architecture Docker image created
|
||||
|
||||
dockerx86ActionIPTables:
|
||||
@-rm ./kube-vip
|
||||
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
|
||||
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
|
||||
@echo New Multi Architecture Docker image created
|
||||
|
||||
dockerLocal:
|
||||
@@ -129,22 +133,25 @@ manifest-test:
|
||||
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
|
||||
|
||||
unit-tests:
|
||||
go test -race ./...
|
||||
go test -race -coverprofile=coverage.out -covermode=atomic ./...
|
||||
|
||||
unit-tests-docker:
|
||||
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) make unit-tests
|
||||
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) sh -c "make unit-tests; status=$$?; chmod 666 coverage.out 2>/dev/null || true; exit $$status"
|
||||
|
||||
integration-tests:
|
||||
go test -tags=integration,e2e -v ./pkg/etcd
|
||||
|
||||
e2e-tests-arp: get-whoami
|
||||
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
|
||||
|
||||
e2e-tests-rt: get-whoami
|
||||
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
|
||||
|
||||
e2e-tests-bgp: get-whoami get-gobgp
|
||||
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
|
||||
|
||||
e2e-tests-etcd: get-whoami
|
||||
GOMAXPROCS=4 K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e/etcd
|
||||
|
||||
e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ The idea behind `kube-vip` is a small self-contained Highly-Available option for
|
||||
|
||||
**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
|
||||
|
||||
For upgrading an existing install in place (static Pod or DaemonSet), see the [upgrade guide](https://kube-vip.io/docs/upgrade/).
|
||||
|
||||
## Features
|
||||
|
||||
Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
|
||||
@@ -130,7 +132,7 @@ Additionally it is now relatively easy and quick to develop with [skaffold](http
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#kube-vip/kube-vip&Date)
|
||||
[](https://star-history.dera.page/#kube-vip/kube-vip&type=date)
|
||||
[](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_shield)
|
||||
|
||||
|
||||
|
||||
176
cmd/kube-vip.go
176
cmd/kube-vip.go
@@ -4,17 +4,14 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -52,8 +49,9 @@ var (
|
||||
)
|
||||
|
||||
var kubeVipCmd = &cobra.Command{
|
||||
Use: "kube-vip",
|
||||
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
|
||||
Use: "kube-vip",
|
||||
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
|
||||
SilenceErrors: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -187,11 +185,16 @@ func init() {
|
||||
}
|
||||
|
||||
// Execute - starts the command parsing process
|
||||
func Execute() {
|
||||
if err := kubeVipCmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
func Execute() int {
|
||||
cmd, err := kubeVipCmd.ExecuteC()
|
||||
if err != nil {
|
||||
log.Error("command failed", "err", err)
|
||||
if cmd == kubeVipCmd {
|
||||
_ = cmd.Usage()
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var kubeVipVersion = &cobra.Command{
|
||||
@@ -215,26 +218,24 @@ var kubeVipSample = &cobra.Command{
|
||||
var kubeVipService = &cobra.Command{
|
||||
Use: "service",
|
||||
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
|
||||
cmd.SilenceUsage = true
|
||||
|
||||
// Load configuration from file if specified (lowest priority)
|
||||
if initConfig.ConfigFile != "" {
|
||||
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
|
||||
if err != nil {
|
||||
log.Error("loading config file", "err", err)
|
||||
return
|
||||
return fmt.Errorf("loading config file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parse environment variables, these will overwrite anything loaded from config file
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing env", "err", err)
|
||||
return
|
||||
return fmt.Errorf("parsing environment: %w", err)
|
||||
}
|
||||
if err := initConfig.Validate(); err != nil {
|
||||
log.Error("validating configuration", "err", err)
|
||||
return
|
||||
return fmt.Errorf("validating configuration: %w", err)
|
||||
}
|
||||
|
||||
// Change RTN_UNSPEC to default type
|
||||
@@ -246,8 +247,7 @@ var kubeVipService = &cobra.Command{
|
||||
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
|
||||
|
||||
if err := initConfig.CheckInterface(); err != nil {
|
||||
log.Error("checking interface", "err", err)
|
||||
return
|
||||
return fmt.Errorf("checking interface: %w", err)
|
||||
}
|
||||
|
||||
// User Environment variables as an option to make manifest clearer
|
||||
@@ -260,8 +260,7 @@ var kubeVipService = &cobra.Command{
|
||||
if initConfig.EnableControlPlane &&
|
||||
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
|
||||
if err := initConfig.CheckSubnetExists(); err != nil {
|
||||
log.Error("checking subnet exists if vip_address defined", "err", err)
|
||||
return
|
||||
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,8 +268,7 @@ var kubeVipService = &cobra.Command{
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
|
||||
if err != nil {
|
||||
log.Error("generating CIDR", "err", err)
|
||||
return
|
||||
return fmt.Errorf("generating CIDR: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,41 +278,39 @@ var kubeVipService = &cobra.Command{
|
||||
// Define the new service manager
|
||||
mgr, err := manager.New(ctx, configMap, &initConfig)
|
||||
if err != nil {
|
||||
log.Error("new manager", "err", err)
|
||||
return
|
||||
return fmt.Errorf("new manager: %w", err)
|
||||
}
|
||||
|
||||
// Start the service manager, this will watch the config Map and construct kube-vip services for it
|
||||
err = mgr.Start(ctx)
|
||||
if err != nil {
|
||||
log.Error("manager start", "err", err)
|
||||
return
|
||||
return fmt.Errorf("manager start: %w", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var kubeVipManager = &cobra.Command{
|
||||
Use: "manager",
|
||||
Short: "Start the kube-vip manager",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
|
||||
cmd.SilenceUsage = true
|
||||
|
||||
// Load configuration from file if specified (lowest priority)
|
||||
if initConfig.ConfigFile != "" {
|
||||
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
|
||||
if err != nil {
|
||||
log.Error("loading config file", "err", err)
|
||||
return
|
||||
return fmt.Errorf("loading config file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parse environment variables, these will overwrite anything loaded from config file
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
return fmt.Errorf("parsing environment: %w", err)
|
||||
}
|
||||
if err := initConfig.Validate(); err != nil {
|
||||
log.Error("validating configuration", "err", err)
|
||||
return
|
||||
return fmt.Errorf("validating configuration: %w", err)
|
||||
}
|
||||
|
||||
// Change RTN_UNSPEC to default type
|
||||
@@ -329,8 +325,7 @@ var kubeVipManager = &cobra.Command{
|
||||
if initConfig.EnableControlPlane &&
|
||||
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
|
||||
if err := initConfig.CheckSubnetExists(); err != nil {
|
||||
log.Error("checking subnet exists if vip_address defined", "err", err)
|
||||
return
|
||||
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -338,8 +333,7 @@ var kubeVipManager = &cobra.Command{
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
|
||||
if err != nil {
|
||||
log.Error("No interface is specified for kube-vip to bind to")
|
||||
return
|
||||
return fmt.Errorf("generating CIDR: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,12 +347,17 @@ var kubeVipManager = &cobra.Command{
|
||||
ctx, cancel := context.WithCancel(cmd.Context())
|
||||
defer cancel()
|
||||
|
||||
metrics.RegisterPrometheusMetrics()
|
||||
|
||||
// start prometheus server
|
||||
if initConfig.PrometheusHTTPServer != "" {
|
||||
wg.Go(func() {
|
||||
servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
|
||||
if err := metrics.Serve(ctx, metrics.ServerConfig{
|
||||
Addr: initConfig.PrometheusHTTPServer,
|
||||
})
|
||||
}); err != nil {
|
||||
// Continue even if metrics server fails
|
||||
log.Error("prometheus HTTP server", "err", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -388,13 +387,11 @@ var kubeVipManager = &cobra.Command{
|
||||
}
|
||||
|
||||
if mode == "" {
|
||||
log.Error("no valid kube-vip mode detected, ensure a supported mode is configured")
|
||||
return
|
||||
return fmt.Errorf("no valid kube-vip mode detected, ensure a supported mode is configured")
|
||||
}
|
||||
|
||||
if modesEnabled > 1 {
|
||||
log.Error("multiple kube-vip modes detected, ensure only one mode is configured")
|
||||
return
|
||||
return fmt.Errorf("multiple kube-vip modes detected, ensure only one mode is configured")
|
||||
}
|
||||
|
||||
// Provide configuration to output/logging
|
||||
@@ -402,18 +399,15 @@ var kubeVipManager = &cobra.Command{
|
||||
|
||||
// End if nothing is enabled
|
||||
if !initConfig.EnableServices && !initConfig.EnableControlPlane {
|
||||
log.Error("no features are enabled")
|
||||
return
|
||||
return fmt.Errorf("no features are enabled")
|
||||
}
|
||||
|
||||
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
|
||||
log.Error("auto subnet discovery cannot be used outside ARP mode")
|
||||
return
|
||||
return fmt.Errorf("auto subnet discovery cannot be used outside ARP mode")
|
||||
}
|
||||
|
||||
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
|
||||
log.Error("auto subnet discovery cannot be used if VIP address was provided")
|
||||
return
|
||||
return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
|
||||
}
|
||||
|
||||
// If we're using wireguard then all traffic goes through the wg0 interface
|
||||
@@ -430,20 +424,17 @@ var kubeVipManager = &cobra.Command{
|
||||
log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
|
||||
err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
|
||||
if err != nil {
|
||||
log.Error("adding link", "err", err)
|
||||
return
|
||||
return fmt.Errorf("adding link: %w", err)
|
||||
}
|
||||
l, err = netlink.LinkByName(initConfig.Interface)
|
||||
if err != nil {
|
||||
log.Error("finding link", "err", err)
|
||||
return
|
||||
return fmt.Errorf("finding link: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
err = netlink.LinkSetUp(l)
|
||||
if err != nil {
|
||||
log.Error("setting link UP", "err", err)
|
||||
return
|
||||
return fmt.Errorf("setting link UP: %w", err)
|
||||
}
|
||||
|
||||
} else { // if we're not using Wireguard then we'll need to use an actual interface
|
||||
@@ -453,8 +444,7 @@ var kubeVipManager = &cobra.Command{
|
||||
defaultIF, err := vip.GetDefaultGatewayInterface()
|
||||
if err != nil {
|
||||
_ = cmd.Help()
|
||||
log.Error("detecting interface", "err", err)
|
||||
return
|
||||
return fmt.Errorf("detecting interface: %w", err)
|
||||
}
|
||||
initConfig.Interface = defaultIF.Name
|
||||
log.Info("kube-vip bind", "interface", initConfig.Interface)
|
||||
@@ -470,8 +460,7 @@ var kubeVipManager = &cobra.Command{
|
||||
}
|
||||
// Perform a check on the state of the interface
|
||||
if err := initConfig.CheckInterface(); err != nil {
|
||||
log.Error("checking interface", "err", err)
|
||||
return
|
||||
return fmt.Errorf("checking interface: %w", err)
|
||||
}
|
||||
|
||||
// User Environment variables as an option to make manifest clearer
|
||||
@@ -483,81 +472,22 @@ var kubeVipManager = &cobra.Command{
|
||||
// Define the new service manager
|
||||
mgr, err := manager.New(ctx, configMap, &initConfig)
|
||||
if err != nil {
|
||||
log.Error("new manager", "err", err)
|
||||
return
|
||||
return fmt.Errorf("new manager: %w", err)
|
||||
}
|
||||
|
||||
metrics.RegisterPrometheusMetrics()
|
||||
// Label metrics after the call to manager.New, as it may modify the node name
|
||||
// if it was not set in the configuration.
|
||||
metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
|
||||
|
||||
// Start the service manager, this will watch the config Map and construct kube-vip services for it
|
||||
err = mgr.Start(ctx)
|
||||
if err != nil {
|
||||
log.Error("start manager", "err", err)
|
||||
return
|
||||
return fmt.Errorf("start manager: %w", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// PrometheusHTTPServerConfig defines the Prometheus server configuration.
|
||||
type PrometheusHTTPServerConfig struct {
|
||||
// Addr sets the http server address used to expose the metric endpoint
|
||||
Addr string
|
||||
}
|
||||
|
||||
func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerConfig) {
|
||||
var err error
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
|
||||
_, _ = w.Write([]byte(`<html>
|
||||
<head><title>kube-vip</title></head>
|
||||
<body>
|
||||
<h1>kube-vip Metrics</h1>
|
||||
<p><a href="` + "/metrics" + `">Metrics</a></p>
|
||||
</body>
|
||||
</html>`))
|
||||
})
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: config.Addr,
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
wg.Go(func() {
|
||||
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Error("prometheus HTTP server", "err", err)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
log.Info("prometheus HTTP server started")
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
// create prometheus shutdown context (independent of other contexts)
|
||||
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer func() {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
if err = srv.Shutdown(ctxShutDown); err != nil {
|
||||
log.Error("shutting down prometheus HTTP server", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err == http.ErrServerClosed {
|
||||
err = nil
|
||||
}
|
||||
|
||||
log.Info("prometheus HTTP server stopped")
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func GenerateCidrRange(address string, dnsMode string) (string, error) {
|
||||
var cidrs []string
|
||||
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
name: kube-vip-cluster
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kube-vip-cluster
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: "app"
|
||||
operator: In
|
||||
values:
|
||||
- kube-vip-cluster
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
containers:
|
||||
- image: ghcr.io/kube-vip/kube-vip:0.3.7
|
||||
imagePullPolicy: Always
|
||||
name: kube-vip
|
||||
command:
|
||||
- /kube-vip
|
||||
- service
|
||||
- --configMap
|
||||
- plndr-configmap
|
||||
- --arp
|
||||
- --interface
|
||||
- ens192
|
||||
- --log
|
||||
- "5"
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
hostNetwork: true
|
||||
status: {}
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: lease-access
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "list", "put"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configMap"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: lease-access
|
||||
subjects:
|
||||
- kind: User
|
||||
name: system:serviceaccount:default:default
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: lease-access
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -1,83 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: vip
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: vip-role
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "list", "put"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps", "endpoints"]
|
||||
verbs: ["watch", "get"]
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: vip-role-bind
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: vip
|
||||
apiGroup: ""
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: vip-role
|
||||
apiGroup: ""
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
name: kube-vip-cluster
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kube-vip-cluster
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: "app"
|
||||
operator: In
|
||||
values:
|
||||
- kube-vip-cluster
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
containers:
|
||||
- image: ghcr.io/kube-vip/kube-vip:0.3.7
|
||||
imagePullPolicy: Always
|
||||
name: kube-vip
|
||||
command:
|
||||
- /kube-vip
|
||||
- service
|
||||
env:
|
||||
- name: vip_interface
|
||||
value: "ens192"
|
||||
- name: vip_configmap
|
||||
value: "plndr"
|
||||
- name: vip_arp
|
||||
value: "true"
|
||||
- name: vip_loglevel
|
||||
value: "5"
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
hostNetwork: true
|
||||
serviceAccountName: vip
|
||||
status: {}
|
||||
@@ -1,83 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: vip
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: vip-role
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "list", "put"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps", "endpoints"]
|
||||
verbs: ["watch", "get"]
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: vip-role-bind
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: vip
|
||||
apiGroup: ""
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: vip-role
|
||||
apiGroup: ""
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
name: kube-vip-cluster
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kube-vip-cluster
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: kube-vip-cluster
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: "app"
|
||||
operator: In
|
||||
values:
|
||||
- kube-vip-cluster
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
containers:
|
||||
- image: plndr/kube-vip:0.1.4
|
||||
imagePullPolicy: Always
|
||||
name: kube-vip
|
||||
command:
|
||||
- /kube-vip
|
||||
- service
|
||||
env:
|
||||
- name: vip_interface
|
||||
value: "ens192"
|
||||
- name: vip_configmap
|
||||
value: "plndr"
|
||||
- name: vip_arp
|
||||
value: "true"
|
||||
- name: vip_loglevel
|
||||
value: "5"
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
hostNetwork: true
|
||||
serviceAccountName: vip
|
||||
status: {}
|
||||
@@ -1,55 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kube-vip-ds
|
||||
namespace: kube-system
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
name: kube-vip-ds
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
name: kube-vip-ds
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- manager
|
||||
env:
|
||||
- name: vip_arp
|
||||
value: "true"
|
||||
- name: vip_interface
|
||||
value: eth0
|
||||
- name: port
|
||||
value: "6443"
|
||||
- name: vip_cidr
|
||||
value: "32"
|
||||
- name: svc_enable
|
||||
value: "true"
|
||||
- name: vip_startleader
|
||||
value: "false"
|
||||
- name: vip_addpeerstolb
|
||||
value: "true"
|
||||
- name: vip_localpeer
|
||||
value: ip-172-20-40-207:172.20.40.207:10000
|
||||
- name: vip_address
|
||||
image: plndr/kube-vip:v0.3.5
|
||||
imagePullPolicy: Always
|
||||
name: kube-vip
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
- SYS_TIME
|
||||
hostNetwork: true
|
||||
serviceAccountName: kube-vip
|
||||
updateStrategy: {}
|
||||
status:
|
||||
currentNumberScheduled: 0
|
||||
desiredNumberScheduled: 0
|
||||
numberMisscheduled: 0
|
||||
numberReady: 0
|
||||
43
go.mod
43
go.mod
@@ -5,41 +5,40 @@ go 1.26.4
|
||||
require (
|
||||
github.com/cloudflare/ipvs v0.12.0
|
||||
github.com/containernetworking/plugins v1.9.1
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/docker/docker v28.5.2+incompatible
|
||||
github.com/florianl/go-conntrack v0.7.0
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/google/go-containerregistry v0.21.9
|
||||
github.com/google/go-containerregistry v0.22.1
|
||||
github.com/google/nftables v0.3.0
|
||||
github.com/gookit/slog v0.7.1
|
||||
github.com/huin/goupnp v1.3.0
|
||||
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
|
||||
github.com/jpillora/backoff v1.0.0
|
||||
github.com/mdlayher/ndp v1.1.0
|
||||
github.com/onsi/ginkgo/v2 v2.32.1
|
||||
github.com/onsi/gomega v1.42.1
|
||||
github.com/osrg/gobgp/v4 v4.8.0
|
||||
github.com/onsi/ginkgo/v2 v2.32.2
|
||||
github.com/onsi/gomega v1.43.0
|
||||
github.com/osrg/gobgp/v4 v4.9.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/sirupsen/logrus v1.9.4
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/vishvananda/netlink v1.3.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b
|
||||
github.com/vishvananda/netns v0.0.5
|
||||
go.etcd.io/etcd/api/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/v3 v3.7.1
|
||||
go.uber.org/zap v1.28.0
|
||||
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
|
||||
google.golang.org/grpc v1.83.0
|
||||
google.golang.org/grpc v1.83.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
k8s.io/api v0.36.3
|
||||
k8s.io/apimachinery v0.36.3
|
||||
k8s.io/client-go v0.36.3
|
||||
k8s.io/api v0.36.4
|
||||
k8s.io/apimachinery v0.36.4
|
||||
k8s.io/client-go v0.36.4
|
||||
k8s.io/klog/v2 v2.140.0
|
||||
sigs.k8s.io/kind v0.32.0
|
||||
sigs.k8s.io/kind v0.33.0
|
||||
sigs.k8s.io/yaml v1.6.0
|
||||
)
|
||||
|
||||
@@ -55,6 +54,7 @@ require (
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
@@ -121,7 +121,6 @@ require (
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/vishvananda/netns v0.0.5 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
@@ -132,15 +131,15 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/mod v0.39.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
|
||||
85
go.sum
85
go.sum
@@ -98,8 +98,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-containerregistry v0.21.9 h1:F+D4uZ3iA3DLMJLfhaqMdHJbzeqm/216WGQq2dokuLs=
|
||||
github.com/google/go-containerregistry v0.21.9/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo=
|
||||
github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84=
|
||||
github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
|
||||
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
|
||||
@@ -144,8 +144,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
|
||||
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
|
||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -192,18 +192,18 @@ github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
|
||||
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
|
||||
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
|
||||
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||
github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
|
||||
github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
||||
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/osrg/gobgp/v4 v4.8.0 h1:CoprkOZ2nsH8aTHij6xWj18QOrsBwG9D00l1PGw7394=
|
||||
github.com/osrg/gobgp/v4 v4.8.0/go.mod h1:bJbFm7T2nRANggShfl3I9h0UpPCzu4uAY5J/6dTdRvs=
|
||||
github.com/osrg/gobgp/v4 v4.9.0 h1:pKOw914kwQ4I/lWNVTfEDosEN3FuqPGytMEInXxpTyQ=
|
||||
github.com/osrg/gobgp/v4 v4.9.0/go.mod h1:bJbFm7T2nRANggShfl3I9h0UpPCzu4uAY5J/6dTdRvs=
|
||||
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
|
||||
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
|
||||
@@ -246,11 +246,11 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
|
||||
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
|
||||
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
@@ -265,8 +265,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8
|
||||
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
|
||||
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
|
||||
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b h1:XtEhFJO3IqjQWHJZ3bbNm7LtbDehriJK65KW+6lnw+Q=
|
||||
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b/go.mod h1:lEui7SPMd9fgxzHVGRAvTxsBGCF6PRH81o2kLWLWHgw=
|
||||
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
|
||||
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
@@ -307,46 +307,45 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
|
||||
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
|
||||
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
|
||||
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
|
||||
@@ -358,8 +357,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -373,12 +372,12 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
|
||||
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
|
||||
k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w=
|
||||
k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg=
|
||||
k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM=
|
||||
k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE=
|
||||
k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg=
|
||||
k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30=
|
||||
k8s.io/api v0.36.4 h1:RxrvqCL6vgH5/+UnTeu1IIFqYmGfy0hnyrod1rn35Oo=
|
||||
k8s.io/api v0.36.4/go.mod h1:S2B3orCFBDhrgyWbLeuKcT2QdHIpQesBkCYSlWtwUOw=
|
||||
k8s.io/apimachinery v0.36.4 h1:PT2UzkupGuAx/+xT5XjiMJ1WGpY3fn9/hdAvjweRet4=
|
||||
k8s.io/apimachinery v0.36.4/go.mod h1:p2I2dipt7JHG+quVwQ1d02d28O4GdDi77RByQ13MTpk=
|
||||
k8s.io/client-go v0.36.4 h1:MDvfDNvMSt0Br94SK8neviVlwL9qifw9B26hJCpD1K0=
|
||||
k8s.io/client-go v0.36.4/go.mod h1:pNK4WKELbwlEDvtbE8l22lEZL5THYF61H5EealokZmA=
|
||||
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
|
||||
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
|
||||
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg=
|
||||
@@ -389,8 +388,8 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
|
||||
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
|
||||
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
|
||||
sigs.k8s.io/kind v0.32.0 h1:p9hscbj98u/qyrjVpjId86LI70nQmbSsipV7wCG10Xk=
|
||||
sigs.k8s.io/kind v0.32.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
|
||||
sigs.k8s.io/kind v0.33.0 h1:AjvDv3vOygb/VKLVQW87lfktIBzkxR8Ump9DjxC8+Lk=
|
||||
sigs.k8s.io/kind v0.33.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
|
||||
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
|
||||
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
|
||||
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw=
|
||||
|
||||
4
main.go
4
main.go
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/kube-vip/kube-vip/cmd"
|
||||
)
|
||||
|
||||
@@ -14,5 +16,5 @@ func main() {
|
||||
|
||||
cmd.Release.Version = Version
|
||||
cmd.Release.Build = Build
|
||||
cmd.Execute()
|
||||
os.Exit(cmd.Execute())
|
||||
}
|
||||
|
||||
@@ -25,6 +25,11 @@ const defaultBGPPort uint32 = 179
|
||||
|
||||
// AddPeer will add peers to the BGP configuration
|
||||
func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
|
||||
remotePort := defaultBGPPort
|
||||
if peer.Port != 0 {
|
||||
remotePort = uint32(peer.Port)
|
||||
}
|
||||
|
||||
p := &api.Peer{
|
||||
Conf: &api.PeerConf{
|
||||
NeighborAddress: peer.Address,
|
||||
@@ -50,7 +55,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
|
||||
Transport: &api.Transport{
|
||||
MtuDiscovery: true,
|
||||
RemoteAddress: peer.Address,
|
||||
RemotePort: defaultBGPPort,
|
||||
RemotePort: remotePort,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -85,6 +90,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
|
||||
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
|
||||
if err != nil {
|
||||
log.Error("failed to get MP-BGP addresses, will not us MP-BGP for this host", "error", err)
|
||||
b.setPeerSource(p)
|
||||
} else {
|
||||
p.AfiSafis = []*api.AfiSafi{
|
||||
{
|
||||
@@ -136,13 +142,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if b.c.SourceIP != "" {
|
||||
p.Transport.LocalAddress = b.c.SourceIP
|
||||
}
|
||||
|
||||
if b.c.SourceIF != "" {
|
||||
p.Transport.BindInterface = b.c.SourceIF
|
||||
}
|
||||
b.setPeerSource(p)
|
||||
}
|
||||
|
||||
if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
|
||||
@@ -152,6 +152,16 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Server) setPeerSource(p *api.Peer) {
|
||||
if b.c.SourceIP != "" {
|
||||
p.Transport.LocalAddress = b.c.SourceIP
|
||||
}
|
||||
|
||||
if b.c.SourceIF != "" {
|
||||
p.Transport.BindInterface = b.c.SourceIF
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Server) getPath(ip net.IP) *apiutil.Path {
|
||||
isV6 := ip.To4() == nil
|
||||
|
||||
|
||||
146
pkg/bgp/peers_config_test.go
Normal file
146
pkg/bgp/peers_config_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package bgp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
api "github.com/osrg/gobgp/v4/api"
|
||||
gobgp "github.com/osrg/gobgp/v4/pkg/server"
|
||||
)
|
||||
|
||||
func TestAddPeerConfiguresTransportOptions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
newServer func(*testing.T) *Server
|
||||
peer kubevip.BGPPeer
|
||||
wantPort uint32
|
||||
wantLocalAddr string
|
||||
wantInterface string
|
||||
}{
|
||||
{
|
||||
name: "configured remote port",
|
||||
newServer: func(t *testing.T) *Server {
|
||||
return newStartedTestBGPServer(t, kubevip.BGPConfig{
|
||||
AS: 65000,
|
||||
RouterID: "192.0.2.1",
|
||||
Peers: []kubevip.BGPPeer{{Address: "192.0.2.10", AS: 65001}},
|
||||
})
|
||||
},
|
||||
peer: kubevip.BGPPeer{Address: "192.0.2.10", AS: 65001, Port: 180},
|
||||
wantPort: 180,
|
||||
},
|
||||
{
|
||||
name: "configured source interface after MP-BGP fallback",
|
||||
newServer: func(t *testing.T) *Server {
|
||||
return newPeerTestServer(t, kubevip.BGPConfig{
|
||||
AS: 65000,
|
||||
RouterID: "192.0.2.1",
|
||||
SourceIF: "lo",
|
||||
MpbgpNexthop: "fixed",
|
||||
Peers: []kubevip.BGPPeer{{Address: "192.0.2.20", AS: 65001}},
|
||||
MpbgpIPv4: "",
|
||||
MpbgpIPv6: "",
|
||||
})
|
||||
},
|
||||
peer: kubevip.BGPPeer{Address: "192.0.2.20", AS: 65001},
|
||||
wantInterface: "lo",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server := tt.newServer(t)
|
||||
if err := server.AddPeer(context.Background(), tt.peer); err != nil {
|
||||
t.Fatalf("AddPeer() error = %v", err)
|
||||
}
|
||||
|
||||
peer := listTestPeer(t, server, tt.peer.Address)
|
||||
if peer.GetTransport() == nil {
|
||||
t.Fatal("configured peer has no transport")
|
||||
}
|
||||
if tt.wantPort != 0 && peer.GetTransport().GetRemotePort() != tt.wantPort {
|
||||
t.Fatalf("remote port = %d, want %d", peer.GetTransport().GetRemotePort(), tt.wantPort)
|
||||
}
|
||||
if tt.wantLocalAddr != "" && peer.GetTransport().GetLocalAddress() != tt.wantLocalAddr {
|
||||
t.Fatalf("local address = %q, want %q", peer.GetTransport().GetLocalAddress(), tt.wantLocalAddr)
|
||||
}
|
||||
if tt.wantInterface != "" && peer.GetTransport().GetBindInterface() != tt.wantInterface {
|
||||
t.Fatalf("bind interface = %q, want %q", peer.GetTransport().GetBindInterface(), tt.wantInterface)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newStartedTestBGPServer(t *testing.T, config kubevip.BGPConfig) *Server {
|
||||
t.Helper()
|
||||
|
||||
server, err := NewBGPServer(config, log.LevelError)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBGPServer() error = %v", err)
|
||||
}
|
||||
|
||||
go server.s.Serve()
|
||||
if err := server.s.StartBgp(context.Background(), &api.StartBgpRequest{
|
||||
Global: &api.Global{
|
||||
Asn: config.AS,
|
||||
RouterId: config.RouterID,
|
||||
ListenPort: -1,
|
||||
},
|
||||
}); err != nil {
|
||||
server.s.Stop()
|
||||
t.Fatalf("StartBgp() error = %v", err)
|
||||
}
|
||||
t.Cleanup(server.s.Stop)
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
func listTestPeer(t *testing.T, server *Server, address string) *api.Peer {
|
||||
t.Helper()
|
||||
|
||||
var got *api.Peer
|
||||
if err := server.s.ListPeer(context.Background(), &api.ListPeerRequest{Address: address}, func(peer *api.Peer) {
|
||||
got = peer
|
||||
}); err != nil {
|
||||
t.Fatalf("ListPeer() error = %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatalf("ListPeer() returned no peer for %s", address)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func newPeerTestServer(t *testing.T, cfg kubevip.BGPConfig) *Server {
|
||||
t.Helper()
|
||||
raw := startEmbeddedRawBGP(t)
|
||||
return &Server{s: raw, c: &cfg, tracker: make(map[string]map[string]bool)}
|
||||
}
|
||||
|
||||
func startEmbeddedRawBGP(t *testing.T) *gobgp.BgpServer {
|
||||
t.Helper()
|
||||
raw := gobgp.NewBgpServer()
|
||||
go raw.Serve()
|
||||
if err := raw.StartBgp(context.Background(), &api.StartBgpRequest{
|
||||
Global: &api.Global{
|
||||
Asn: 65000,
|
||||
RouterId: "192.0.2.1",
|
||||
ListenPort: -1,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("starting embedded BGP server: %v", err)
|
||||
}
|
||||
var stopOnce sync.Once
|
||||
t.Cleanup(func() {
|
||||
stopOnce.Do(func() {
|
||||
if err := raw.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
|
||||
t.Logf("stopping embedded BGP server: %v", err)
|
||||
}
|
||||
})
|
||||
})
|
||||
return raw
|
||||
}
|
||||
@@ -15,6 +15,11 @@ import (
|
||||
gobgp "github.com/osrg/gobgp/v4/pkg/server"
|
||||
)
|
||||
|
||||
type BGPManager interface {
|
||||
AddHost(ctx context.Context, addr string, object string) error
|
||||
DelHost(ctx context.Context, addr string, object string) error
|
||||
}
|
||||
|
||||
// Server manages a server object
|
||||
type Server struct {
|
||||
s *gobgp.BgpServer
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
// Cluster - The Cluster object manages the state of the cluster for a particular node
|
||||
type Cluster struct {
|
||||
stop chan bool
|
||||
stopMutex sync.Mutex
|
||||
Network []vip.Network
|
||||
arpMgr *arp.Manager
|
||||
routeMgr *route.Manager
|
||||
@@ -93,6 +95,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
|
||||
|
||||
// Stop - Will stop the Cluster and release VIP if needed
|
||||
func (cluster *Cluster) Stop() {
|
||||
cluster.stopMutex.Lock()
|
||||
defer cluster.stopMutex.Unlock()
|
||||
|
||||
// Close the stop channel, which will shut down the VIP (if needed)
|
||||
if cluster.stop != nil {
|
||||
close(cluster.stop)
|
||||
|
||||
32
pkg/cluster/cluster_stop_test.go
Normal file
32
pkg/cluster/cluster_stop_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStopConcurrentDoesNotRaceOrPanic(t *testing.T) {
|
||||
c := &Cluster{stop: make(chan bool)}
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
var panics atomic.Int64
|
||||
|
||||
for range 128 {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
panics.Add(1)
|
||||
}
|
||||
}()
|
||||
c.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if got := panics.Load(); got != 0 {
|
||||
t.Fatalf("concurrent Stop panicked %d time(s)", got)
|
||||
}
|
||||
}
|
||||
@@ -29,14 +29,7 @@ import (
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// BGPRouteManager allows to manage the routes announced by the BGP server.
|
||||
type BGPRouteManager interface {
|
||||
AddHost(ctx context.Context, addr string, object string) error
|
||||
DelHost(ctx context.Context, addr string, object string) error
|
||||
}
|
||||
|
||||
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager,
|
||||
bgpServer BGPRouteManager, killFunc func()) error {
|
||||
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager, bgpServer bgp.BGPManager, killFunc func()) error {
|
||||
|
||||
var err error
|
||||
|
||||
@@ -113,7 +106,7 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
|
||||
err = em.NodeWatcher(ctx, lb, c.Port)
|
||||
if err != nil {
|
||||
log.Error("Error watching node labels", "err", err)
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
if utils.IsPanicError(err) {
|
||||
killFunc()
|
||||
return
|
||||
}
|
||||
@@ -287,7 +280,28 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer BGPRouteManager, vipCIDR string) {
|
||||
func (cluster *Cluster) bgpHealthCheck(ctx context.Context, c *kubevip.Config) (bool, error) {
|
||||
statusCode := 0
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("building request %v: %w", req, err)
|
||||
} else {
|
||||
resp, err := cluster.healthCheckHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("checking control-plane: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
healthy := statusCode == http.StatusOK
|
||||
if !healthy {
|
||||
return healthy, fmt.Errorf("wrong status code: %d", statusCode)
|
||||
}
|
||||
return healthy, nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer bgp.BGPManager, vipCIDR string) {
|
||||
period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
|
||||
|
||||
consecutiveFailures := 0
|
||||
@@ -304,24 +318,7 @@ func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Confi
|
||||
)
|
||||
|
||||
for {
|
||||
statusCode := 0
|
||||
var healthErr error
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
|
||||
if err != nil {
|
||||
healthErr = err
|
||||
} else {
|
||||
resp, err := cluster.healthCheckHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
healthErr = err
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
}
|
||||
|
||||
healthy := healthErr == nil && statusCode == http.StatusOK
|
||||
|
||||
healthy, healthErr := cluster.bgpHealthCheck(ctx, c)
|
||||
if healthy {
|
||||
consecutiveFailures = 0
|
||||
if !routeAnnounced {
|
||||
@@ -336,10 +333,7 @@ func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Confi
|
||||
consecutiveFailures++
|
||||
if healthErr != nil {
|
||||
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
|
||||
} else {
|
||||
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "status", statusCode)
|
||||
}
|
||||
|
||||
if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
|
||||
log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
|
||||
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
|
||||
@@ -399,7 +393,7 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
|
||||
}
|
||||
|
||||
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
|
||||
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, wg *sync.WaitGroup) error {
|
||||
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp bgp.BGPManager, name string, wg *sync.WaitGroup) error {
|
||||
// use a Go context so we can tell the arp loop code when we
|
||||
// want to step down
|
||||
//nolint
|
||||
@@ -429,7 +423,7 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
|
||||
if err := network.SetMask(c.VIPSubnet); err != nil {
|
||||
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
|
||||
lbCancel()
|
||||
return utils.NewPanicError(fmt.Sprintf("failed to set mask for subnet %q: %s", c.VIPSubnet, err.Error()))
|
||||
return utils.WrapPanicError(err, "failed to set mask for subnet %q", c.VIPSubnet)
|
||||
}
|
||||
_, err := network.DeleteIP()
|
||||
if err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/cluster"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/route"
|
||||
@@ -28,7 +29,7 @@ func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
|
||||
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
|
||||
"route should be announced")
|
||||
@@ -40,7 +41,7 @@ func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
|
||||
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
|
||||
2*time.Second, "route should not be announced while unhealthy")
|
||||
@@ -56,7 +57,7 @@ func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg.ControlPlaneHealthCheck.FailureThreshold = 3
|
||||
startVipService(t, cfg, bgpManager)
|
||||
|
||||
@@ -78,7 +79,7 @@ func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
|
||||
startVipService(t, cfg, bgpManager)
|
||||
|
||||
@@ -100,7 +101,7 @@ func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
cancelContext, vipServiceDone := startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
cancelContext, vipServiceDone := startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
|
||||
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
|
||||
"route should be announced")
|
||||
@@ -121,7 +122,7 @@ func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
bgpManager.setAddErr(errTestAddHost)
|
||||
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
|
||||
|
||||
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
|
||||
2*time.Second, "route should not be announced while AddHost errors")
|
||||
@@ -137,7 +138,7 @@ func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
|
||||
t.Cleanup(healthcheck.server.Close)
|
||||
|
||||
bgpManager := newMockBGPRouteManager()
|
||||
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
|
||||
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
|
||||
startVipService(t, cfg, bgpManager)
|
||||
|
||||
@@ -195,9 +196,8 @@ func (e *testError) Error() string { return e.msg }
|
||||
// startVipService launches vipService in a goroutine with a mock network and
|
||||
// registers a cleanup to cancel the context and wait for it to finish.
|
||||
// Uses InitCluster so the real code parses certs for the BGP health check client.
|
||||
func startVipService(t *testing.T, cfg *kubevip.Config, bgpManager *mockBGPRouteManager) (context.CancelFunc, <-chan struct{}) {
|
||||
func startVipService(t *testing.T, cfg *kubevip.Config, bgpServer bgp.BGPManager) (context.CancelFunc, <-chan struct{}) {
|
||||
t.Helper()
|
||||
|
||||
c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("InitCluster: %v", err)
|
||||
@@ -208,7 +208,7 @@ func startVipService(t *testing.T, cfg *kubevip.Config, bgpManager *mockBGPRoute
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
_ = c.StartVipService(ctx, cfg, nil, bgpManager, func() {})
|
||||
_ = c.StartVipService(ctx, cfg, nil, bgpServer, func() {})
|
||||
close(done)
|
||||
}()
|
||||
|
||||
@@ -247,14 +247,14 @@ func startRoutingTableVipService(t *testing.T, cfg *kubevip.Config, network *moc
|
||||
}
|
||||
|
||||
func newRoutingTableConfig(url, caPath string) *kubevip.Config {
|
||||
cfg := newTestConfig(url, caPath)
|
||||
cfg := newBGPConfig(url, caPath)
|
||||
cfg.EnableBGP = false
|
||||
cfg.EnableRoutingTable = true
|
||||
cfg.BackendHealthCheckInterval = 1
|
||||
return cfg
|
||||
}
|
||||
|
||||
func newTestConfig(url, caPath string) *kubevip.Config {
|
||||
func newBGPConfig(url, caPath string) *kubevip.Config {
|
||||
return &kubevip.Config{
|
||||
EnableBGP: true,
|
||||
ControlPlaneHealthCheck: kubevip.HealthCheck{
|
||||
|
||||
@@ -52,9 +52,8 @@ func (n *ns) add(name string, output chan<- watch.Event) *object {
|
||||
return i
|
||||
}
|
||||
|
||||
func (n *ns) del(name string) {
|
||||
if _, exists := n.Load(name); exists {
|
||||
n.Delete(name)
|
||||
func (n *ns) del(name string, object *object) {
|
||||
if n.CompareAndDelete(name, object) {
|
||||
n.cnt.Add(-1)
|
||||
}
|
||||
}
|
||||
@@ -119,35 +118,57 @@ func (d *debouncer) Start(ctx context.Context) error {
|
||||
return fmt.Errorf("objects of type %T are not supported", v)
|
||||
}
|
||||
|
||||
eventNs, exists := d.getNs(namespace)
|
||||
if !exists {
|
||||
// if not, create new map for the namespace
|
||||
eventNs = d.addNs(namespace)
|
||||
}
|
||||
processEvent:
|
||||
for {
|
||||
eventNs, exists := d.getNs(namespace)
|
||||
if !exists {
|
||||
// if not, create new map for the namespace
|
||||
eventNs = d.addNs(namespace)
|
||||
}
|
||||
|
||||
// check if the object was previously reconciled
|
||||
eventObject, exists := eventNs.get(name)
|
||||
// check if the object was previously reconciled
|
||||
eventObject, exists := eventNs.get(name)
|
||||
|
||||
// if not and the event is not of type 'Deleted', create new object
|
||||
if !exists && tmp.Type != watch.Deleted {
|
||||
eventObject = eventNs.add(name, d.output)
|
||||
// if not and the event is not of type 'Deleted', create new object
|
||||
if !exists && tmp.Type != watch.Deleted {
|
||||
eventObject = eventNs.add(name, d.output)
|
||||
|
||||
wg.Go(func() {
|
||||
// start deboucing events for this object
|
||||
eventObject.start(debouncerCtx, d.debounceTime)
|
||||
// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
|
||||
eventObject = nil
|
||||
eventNs.del(name)
|
||||
// if namespace is empty, delete the namespace map
|
||||
if eventNs.cnt.Load() == 0 {
|
||||
d.delNs(namespace)
|
||||
workerObject := eventObject
|
||||
workerNs := eventNs
|
||||
workerName := name
|
||||
workerNamespace := namespace
|
||||
workerObject.onStop = func() {
|
||||
// Remove the object before its worker can become receiver-less.
|
||||
workerNs.del(workerName, workerObject)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if eventObject != nil {
|
||||
wg.Go(func() {
|
||||
// start deboucing events for this object
|
||||
workerObject.start(debouncerCtx, d.debounceTime)
|
||||
// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
|
||||
workerNs.del(workerName, workerObject)
|
||||
// if namespace is empty, delete the namespace map
|
||||
if workerNs.cnt.Load() == 0 {
|
||||
d.delNs(workerNamespace, workerNs)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if eventObject == nil {
|
||||
break processEvent
|
||||
}
|
||||
|
||||
// pass the watch event to the debouncer object
|
||||
eventObject.input <- tmp
|
||||
select {
|
||||
case eventObject.input <- tmp:
|
||||
break processEvent
|
||||
case <-eventObject.stopChan:
|
||||
// The object stopped after the map lookup. Retry the event
|
||||
// against the newly-created object instead of dropping it.
|
||||
continue processEvent
|
||||
case <-debouncerCtx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -181,8 +202,8 @@ func (d *debouncer) addNs(namespace string) *ns {
|
||||
return &n
|
||||
}
|
||||
|
||||
func (d *debouncer) delNs(namespace string) {
|
||||
d.namespaces.Delete(namespace)
|
||||
func (d *debouncer) delNs(namespace string, ns *ns) {
|
||||
d.namespaces.CompareAndDelete(namespace, ns)
|
||||
}
|
||||
|
||||
type object struct {
|
||||
@@ -190,6 +211,7 @@ type object struct {
|
||||
output chan<- watch.Event
|
||||
stopChan chan any
|
||||
stopOnce sync.Once
|
||||
onStop func()
|
||||
}
|
||||
|
||||
func newObject(output chan<- watch.Event) *object {
|
||||
@@ -247,6 +269,9 @@ func (o *object) start(ctx context.Context, debounceTime time.Duration) {
|
||||
|
||||
func (o *object) stop() {
|
||||
o.stopOnce.Do(func() {
|
||||
if o.onStop != nil {
|
||||
o.onStop()
|
||||
}
|
||||
close(o.stopChan)
|
||||
})
|
||||
}
|
||||
|
||||
162
pkg/debouncer/debouncer_deadlock_test.go
Normal file
162
pkg/debouncer/debouncer_deadlock_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package debouncer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
func TestStartReturnsWhenCancellationInterruptsObjectForwarding(t *testing.T) {
|
||||
input := make(chan watch.Event)
|
||||
d := &debouncer{
|
||||
input: input,
|
||||
output: make(chan watch.Event),
|
||||
stopChan: make(chan any),
|
||||
debounceTime: 200 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Leave the object without a receiver. This is the state reached when its
|
||||
// worker exits on context cancellation just before Start forwards an event.
|
||||
ns := d.addNs("default")
|
||||
ns.add("example", d.output)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- d.Start(ctx) }()
|
||||
|
||||
event := watch.Event{
|
||||
Type: watch.Modified,
|
||||
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "example", Namespace: "default",
|
||||
}},
|
||||
}
|
||||
sent := make(chan struct{})
|
||||
go func() {
|
||||
input <- event
|
||||
close(sent)
|
||||
}()
|
||||
select {
|
||||
case <-sent:
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
cancel()
|
||||
t.Fatal("debouncer did not receive the test event")
|
||||
}
|
||||
cancel()
|
||||
|
||||
// Nothing ever receives from the object, so Start can only return by
|
||||
// abandoning the blocked forward when the context is cancelled. Receiving
|
||||
// here instead would make the forward succeed and the assertion racy.
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer remained blocked forwarding an event after context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartRecreatesObjectAfterDeletionWithoutCancellation(t *testing.T) {
|
||||
previousProcs := runtime.GOMAXPROCS(1)
|
||||
t.Cleanup(func() { runtime.GOMAXPROCS(previousProcs) })
|
||||
|
||||
input := make(chan watch.Event)
|
||||
d, err := New(input, "200ms")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create debouncer: %s", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- d.Start(ctx) }()
|
||||
|
||||
service := func(eventType watch.EventType, resourceVersion string) watch.Event {
|
||||
return watch.Event{
|
||||
Type: eventType,
|
||||
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "example",
|
||||
Namespace: "default",
|
||||
ResourceVersion: resourceVersion,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
send := func(event watch.Event) {
|
||||
t.Helper()
|
||||
select {
|
||||
case input <- event:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer did not receive the test event")
|
||||
}
|
||||
}
|
||||
|
||||
send(service(watch.Added, "initial"))
|
||||
select {
|
||||
case event := <-d.output:
|
||||
if event.Type != watch.Added {
|
||||
t.Fatalf("expected initial Added event, got %s", event.Type)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer did not emit the initial event")
|
||||
}
|
||||
|
||||
eventNs, exists := d.getNs("default")
|
||||
if !exists {
|
||||
t.Fatal("debouncer did not create the namespace map")
|
||||
}
|
||||
oldObject, exists := eventNs.get("example")
|
||||
if !exists {
|
||||
t.Fatal("debouncer did not create the object")
|
||||
}
|
||||
|
||||
send(service(watch.Deleted, "deleted"))
|
||||
select {
|
||||
case event := <-d.output:
|
||||
if event.Type != watch.Deleted {
|
||||
t.Fatalf("expected Deleted event, got %s", event.Type)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer did not emit the Deleted event")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-oldObject.stopChan:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("object did not self-terminate")
|
||||
}
|
||||
if _, exists := eventNs.get("example"); exists {
|
||||
t.Fatal("self-terminated object remained in the namespace map")
|
||||
}
|
||||
|
||||
send(service(watch.Modified, "fresh"))
|
||||
select {
|
||||
case event := <-d.output:
|
||||
if event.Type != watch.Modified {
|
||||
t.Fatalf("expected fresh Modified event, got %s", event.Type)
|
||||
}
|
||||
service, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
t.Fatalf("expected a Service event, got %T", event.Object)
|
||||
}
|
||||
if service.ResourceVersion != "fresh" {
|
||||
t.Fatalf("expected the fresh event, got resource version %q", service.ResourceVersion)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer did not process the fresh event after object deletion")
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("debouncer returned an error: %s", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("debouncer did not stop")
|
||||
}
|
||||
}
|
||||
@@ -2,14 +2,12 @@ package election
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/kube-vip/kube-vip/pkg/etcd"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
@@ -17,7 +15,6 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -62,7 +59,7 @@ func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Cl
|
||||
func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
|
||||
switch c.LeaderElectionType {
|
||||
case "kubernetes", "":
|
||||
runKubernetesLeaderElectionOrDie(ctx, run)
|
||||
return runKubernetesLeaderElectionOrDie(ctx, run)
|
||||
case "etcd":
|
||||
if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
|
||||
return err
|
||||
@@ -74,20 +71,25 @@ func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
|
||||
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
|
||||
annotations, err := kubevip.WithLeaseVIPs(run.LeaseAnnotations, run.Config.InstanceName, run.Config.RoutingProtocol, run.VIPs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
leaseClient := run.Mgr.KubernetesClient.CoordinationV1().Leases(run.LeaseID.Namespace())
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
baseLock := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{
|
||||
Name: run.LeaseID.Name(),
|
||||
Namespace: run.LeaseID.Namespace(),
|
||||
Annotations: run.LeaseAnnotations,
|
||||
Name: run.LeaseID.Name(),
|
||||
Namespace: run.LeaseID.Namespace(),
|
||||
},
|
||||
Client: run.Mgr.KubernetesClient.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: run.Config.NodeName,
|
||||
},
|
||||
}
|
||||
lock := newAnnotatedLeaseLock(baseLock, leaseClient, run.LeaseID.Name(), annotations)
|
||||
|
||||
// start the leader election code loop
|
||||
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
|
||||
@@ -108,6 +110,7 @@ func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
|
||||
OnNewLeader: run.OnNewLeader,
|
||||
},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
|
||||
@@ -138,6 +141,7 @@ type RunConfig struct {
|
||||
LeaseID lease.ID
|
||||
Mgr *Manager
|
||||
LeaseAnnotations map[string]string
|
||||
VIPs []string
|
||||
|
||||
// onStartedLeading is called when this member starts leading.
|
||||
OnStartedLeading func(context.Context)
|
||||
@@ -199,7 +203,7 @@ func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBal
|
||||
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
|
||||
if err != nil {
|
||||
log.Error("adding node to load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
if utils.IsPanicError(err) {
|
||||
return fmt.Errorf("add IPVS backend: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -233,23 +237,20 @@ func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBal
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes Nodes")
|
||||
|
||||
// This round trip allows us to handle unstructured status
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error("watcher", "status", status)
|
||||
watchErr = fmt.Errorf("node watcher error, status: %s", status.String())
|
||||
watchErr = fmt.Errorf("node watcher error: %w", utils.WatchError(event.Object))
|
||||
log.Error("watcher", "err", watchErr)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Exiting Node watcher")
|
||||
return watchErr
|
||||
if watchErr != nil {
|
||||
return watchErr
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return utils.NewPanicError("node watcher channel closed unexpectedly")
|
||||
}
|
||||
|
||||
func checkIfNodeIsReady(node *v1.Node) bool {
|
||||
|
||||
92
pkg/election/lease_lock.go
Normal file
92
pkg/election/lease_lock.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package election
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
|
||||
type annotatedLeaseLock struct {
|
||||
resourcelock.Interface
|
||||
leases coordinationv1client.LeaseInterface
|
||||
name string
|
||||
annotations map[string]string
|
||||
}
|
||||
|
||||
func newAnnotatedLeaseLock(lock resourcelock.Interface, leases coordinationv1client.LeaseInterface,
|
||||
name string, annotations map[string]string) resourcelock.Interface {
|
||||
return &annotatedLeaseLock{Interface: lock, leases: leases, name: name, annotations: annotations}
|
||||
}
|
||||
|
||||
func (lock *annotatedLeaseLock) Create(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
|
||||
if err := lock.Interface.Create(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
lock.ensure(ctx, record)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lock *annotatedLeaseLock) Update(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
|
||||
if err := lock.Interface.Update(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
lock.ensure(ctx, record)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensure applies the configured annotations once this process holds the lease. Failures are
|
||||
// logged rather than returned: the lease write already succeeded, so reporting an error would
|
||||
// make the elector stand down while it still holds the lease.
|
||||
func (lock *annotatedLeaseLock) ensure(ctx context.Context, record resourcelock.LeaderElectionRecord) {
|
||||
if record.HolderIdentity != lock.Identity() {
|
||||
return
|
||||
}
|
||||
changed, err := lock.ensureAnnotations(ctx)
|
||||
if err != nil {
|
||||
log.Warn("failed to annotate lease", "lease", lock.name, "err", err)
|
||||
return
|
||||
}
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
// Annotating out of band bumps the resourceVersion, so refresh the wrapped lock's
|
||||
// cached lease or its next optimistic Update conflicts.
|
||||
if _, _, err := lock.Interface.Get(ctx); err != nil {
|
||||
log.Warn("failed to refresh lease after annotating", "lease", lock.name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (lock *annotatedLeaseLock) ensureAnnotations(ctx context.Context) (bool, error) {
|
||||
changed := false
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
resource, err := lock.leases.Get(ctx, lock.name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resource.Annotations == nil {
|
||||
resource.Annotations = make(map[string]string, len(lock.annotations))
|
||||
}
|
||||
resourceChanged := false
|
||||
for key, value := range lock.annotations {
|
||||
if resource.Annotations[key] == value {
|
||||
continue
|
||||
}
|
||||
resource.Annotations[key] = value
|
||||
resourceChanged = true
|
||||
}
|
||||
if !resourceChanged {
|
||||
return nil
|
||||
}
|
||||
_, err = lock.leases.Update(ctx, resource, metav1.UpdateOptions{})
|
||||
if err == nil {
|
||||
changed = true
|
||||
}
|
||||
return err
|
||||
})
|
||||
return changed, err
|
||||
}
|
||||
177
pkg/election/lease_lock_test.go
Normal file
177
pkg/election/lease_lock_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package election
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
k8stesting "k8s.io/client-go/testing"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
)
|
||||
|
||||
func TestAnnotatedLeaseLockPersistsAnnotationsOnCreateAndUpdate(t *testing.T) {
|
||||
client := fake.NewSimpleClientset()
|
||||
leaseClient := client.CoordinationV1().Leases("default")
|
||||
base := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
|
||||
Client: client.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: "node-a",
|
||||
},
|
||||
}
|
||||
annotations, err := kubevip.WithLeaseVIPs(map[string]string{"example.test/preserved": "true"},
|
||||
"release_a", 248, []string{"192.0.2.10"})
|
||||
if err != nil {
|
||||
t.Fatalf("WithLeaseVIPs() error = %v", err)
|
||||
}
|
||||
lock := newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
|
||||
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
|
||||
if err := lock.Create(context.Background(), record); err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
if err := lock.Update(context.Background(), record); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get Lease: %v", err)
|
||||
}
|
||||
value, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
|
||||
if err != nil {
|
||||
t.Fatalf("ParseLeaseVIPs() error = %v", err)
|
||||
}
|
||||
if value.InstanceName != "release_a" || value.IFAProto != 248 || len(value.VIPs) != 1 ||
|
||||
value.VIPs[0] != (kubevip.LeaseVIP{Index: 0, Value: "192.0.2.10", Kind: kubevip.LeaseVIPKindAddress}) {
|
||||
t.Fatalf("Lease VIP metadata = %+v", value)
|
||||
}
|
||||
if resource.Annotations["example.test/preserved"] != "true" {
|
||||
t.Fatal("Lease update dropped a configured annotation")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed annotation write must not be reported to the leader elector: the lease itself
|
||||
// was already written, and an error makes the elector stand down while it still holds it.
|
||||
func TestAnnotatedLeaseLockAnnotationFailureDoesNotSurfaceToElector(t *testing.T) {
|
||||
client := fake.NewSimpleClientset()
|
||||
base := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
|
||||
Client: client.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{Identity: "node-a"},
|
||||
}
|
||||
|
||||
failing := fake.NewSimpleClientset()
|
||||
failing.PrependReactor("get", "leases", func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("annotation backend unavailable")
|
||||
})
|
||||
|
||||
annotations, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lock := newAnnotatedLeaseLock(base, failing.CoordinationV1().Leases("default"), "lease", annotations)
|
||||
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
|
||||
|
||||
if err := lock.Create(context.Background(), record); err != nil {
|
||||
t.Fatalf("Create() error = %v, want nil so the elector keeps the lease", err)
|
||||
}
|
||||
if err := lock.Update(context.Background(), record); err != nil {
|
||||
t.Fatalf("Update() error = %v, want nil so the elector keeps the lease", err)
|
||||
}
|
||||
|
||||
if _, err := client.CoordinationV1().Leases("default").Get(context.Background(), "lease",
|
||||
metav1.GetOptions{}); err != nil {
|
||||
t.Fatalf("wrapped lock did not write the lease: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnotatedLeaseLockFollowerDoesNotOverwriteAnnotations(t *testing.T) {
|
||||
client := fake.NewSimpleClientset()
|
||||
leaseClient := client.CoordinationV1().Leases("default")
|
||||
newBase := func(identity string) *resourcelock.LeaseLock {
|
||||
return &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
|
||||
Client: client.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
|
||||
}
|
||||
}
|
||||
ownerBase := newBase("node-a")
|
||||
followerBase := newBase("node-b")
|
||||
active, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
creator := newAnnotatedLeaseLock(ownerBase, leaseClient, "lease", active)
|
||||
if err := creator.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
follower, err := kubevip.WithLeaseVIPs(nil, "release_b", 249, []string{"192.0.2.20"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observer := newAnnotatedLeaseLock(followerBase, leaseClient, "lease", follower)
|
||||
if _, _, err := observer.Get(context.Background()); err != nil {
|
||||
t.Fatalf("Get() error = %v", err)
|
||||
}
|
||||
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if metadata.InstanceName != "release_a" || metadata.IFAProto != 248 {
|
||||
t.Fatalf("follower overwrote active metadata: %+v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnotatedLeaseLockReleaseDoesNotOverwriteSuccessorMetadata(t *testing.T) {
|
||||
client := fake.NewSimpleClientset()
|
||||
leaseClient := client.CoordinationV1().Leases("default")
|
||||
newLock := func(identity, instanceName string, protocol int, vip string) resourcelock.Interface {
|
||||
base := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
|
||||
Client: client.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
|
||||
}
|
||||
annotations, err := kubevip.WithLeaseVIPs(nil, instanceName, protocol, []string{vip})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
|
||||
}
|
||||
|
||||
first := newLock("node-a", "release_a", 248, "192.0.2.10")
|
||||
if err := first.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := first.Update(context.Background(), resourcelock.LeaderElectionRecord{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second := newLock("node-b", "release_b", 249, "192.0.2.20")
|
||||
if _, _, err := second.Get(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := second.Update(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if metadata.InstanceName != "release_b" || metadata.IFAProto != 249 || metadata.VIPs[0].Value != "192.0.2.20" {
|
||||
t.Fatalf("successor metadata = %+v", metadata)
|
||||
}
|
||||
}
|
||||
@@ -46,20 +46,24 @@ func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, b
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Event,
|
||||
// Reconcile applies a watch event to the provider and reconciles the service
|
||||
// against the endpoints that remain afterwards. A deleted object is only one of
|
||||
// potentially several backing the service, so deletions are recomputed rather
|
||||
// than assumed to empty it. It reports whether the caller should skip this event
|
||||
// and wait for the next one.
|
||||
func (p *Processor) Reconcile(svcCtx *servicecontext.Context, event watch.Event,
|
||||
lastKnownGoodEndpoint *string, service *v1.Service, id string,
|
||||
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup,
|
||||
clientSet *kubernetes.Clientset,
|
||||
egressUpdateFunc func(context.Context, *v1.Service) error) (bool, error) {
|
||||
|
||||
var err error
|
||||
if err = p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
|
||||
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
|
||||
if err := p.applyEvent(svcCtx, event); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
endpoints, err := p.worker.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, fmt.Errorf("[%s] error getting endpoints: %w", p.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
if err := p.worker.setInstanceEndpointsStatus(svcCtx.Ctx, service, endpoints); err != nil {
|
||||
@@ -89,11 +93,7 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
|
||||
|
||||
svcCtx.SignalReadiness()
|
||||
|
||||
// There are local endpoints available on the node
|
||||
// Process immediately if:
|
||||
// - No services/leader election is enabled, OR
|
||||
// - WireGuard is enabled (it always needs immediate DNAT rule updates)
|
||||
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
|
||||
if p.shouldProcessInstance() {
|
||||
if err := p.worker.processInstance(svcCtx, service); err != nil {
|
||||
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
|
||||
}
|
||||
@@ -106,21 +106,13 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
|
||||
}
|
||||
svcCtx.SignalReadiness()
|
||||
|
||||
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
|
||||
if p.shouldProcessInstance() {
|
||||
if err := p.worker.processInstance(svcCtx, service); err != nil {
|
||||
return false, fmt.Errorf("failed to process endpointless instance: %w", err)
|
||||
}
|
||||
}
|
||||
} else if svcCtx.Signalled.Load() {
|
||||
// There are no local endpoints
|
||||
svcCtx.ResetReadiness()
|
||||
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
|
||||
if p.config.EnableARP && !p.config.EnableServicesElection {
|
||||
i := instance.FindServiceInstance(service, *p.instances)
|
||||
for _, c := range i.Clusters {
|
||||
c.Stop()
|
||||
}
|
||||
}
|
||||
p.handleNoEndpoints(svcCtx, service, lastKnownGoodEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,13 +125,42 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Delete(ctx context.Context, service *v1.Service, id string) error {
|
||||
if err := p.worker.delete(ctx, service, id); err != nil {
|
||||
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
|
||||
// applyEvent updates the provider's view of the objects backing this service.
|
||||
func (p *Processor) applyEvent(svcCtx *servicecontext.Context, event watch.Event) error {
|
||||
if event.Type == watch.Deleted {
|
||||
if err := p.provider.DeleteObject(event.Object); err != nil {
|
||||
return fmt.Errorf("[%s] error deleting k8s object: %w", p.provider.GetLabel(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
|
||||
return fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldProcessInstance reports whether this node has to program the datapath
|
||||
// itself, rather than waiting to be told to by a leader election callback.
|
||||
// WireGuard always reprograms, because its DNAT rules are per-endpoint.
|
||||
func (p *Processor) shouldProcessInstance() bool {
|
||||
return (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard
|
||||
}
|
||||
|
||||
// handleNoEndpoints tears down everything backing a service that no longer has
|
||||
// any usable endpoints.
|
||||
func (p *Processor) handleNoEndpoints(svcCtx *servicecontext.Context, service *v1.Service, lastKnownGoodEndpoint *string) {
|
||||
svcCtx.ResetReadiness()
|
||||
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
|
||||
if p.config.EnableARP && !p.config.EnableServicesElection && p.instances != nil {
|
||||
if i := instance.FindServiceInstance(service, *p.instances); i != nil {
|
||||
for _, c := range i.Clusters {
|
||||
c.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
|
||||
// if we haven't populated one, then do so
|
||||
family := utils.IPv4Family
|
||||
@@ -179,28 +200,41 @@ func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint
|
||||
egressUpdateFunc func(context.Context, *v1.Service) error) {
|
||||
// Set the service accordingly
|
||||
if service.Annotations[kubevip.Egress] == "true" {
|
||||
ip := net.ParseIP(*lastKnownGoodEndpoint)
|
||||
if *lastKnownGoodEndpoint != "" {
|
||||
ip := net.ParseIP(*lastKnownGoodEndpoint)
|
||||
expectIPv6 := service.Annotations[kubevip.EgressIPv6] == "true"
|
||||
if ip == nil || (ip.To4() == nil) != expectIPv6 {
|
||||
log.Warn("ignoring active endpoint with unexpected address family",
|
||||
"service", service.Name,
|
||||
"namespace", service.Namespace,
|
||||
"endpoint", *lastKnownGoodEndpoint,
|
||||
"expected_ipv6", expectIPv6)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Store old values from ServiceSnapshot to detect if annotation actually changed
|
||||
// We use the ServiceSnapshot instead of the service parameter because the service parameter
|
||||
// may have stale annotations if the last update failed
|
||||
var oldEndpoint, oldEndpointIPv6 string
|
||||
snapshotFound := false
|
||||
if p.instances != nil {
|
||||
serviceInstance := instance.FindServiceInstance(service, *p.instances)
|
||||
if serviceInstance != nil {
|
||||
if serviceInstance != nil && serviceInstance.ServiceSnapshot != nil {
|
||||
snapshotFound = true
|
||||
oldEndpoint = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
|
||||
oldEndpointIPv6 = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
}
|
||||
}
|
||||
// Fall back to service annotations if we couldn't find the instance
|
||||
if oldEndpoint == "" && oldEndpointIPv6 == "" {
|
||||
// Empty annotations in an existing snapshot are meaningful after a zero-endpoint transition.
|
||||
if !snapshotFound {
|
||||
oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
|
||||
oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
}
|
||||
|
||||
// Determine which annotation to update based on IP version
|
||||
var endpoint, endpointIPv6 string
|
||||
if ip.To4() == nil && !p.config.EnableEndpoints {
|
||||
if service.Annotations[kubevip.EgressIPv6] == "true" && !p.config.EnableEndpoints {
|
||||
// IPv6
|
||||
endpointIPv6 = *lastKnownGoodEndpoint
|
||||
endpoint = oldEndpoint // Preserve existing IPv4 if any
|
||||
@@ -290,7 +324,12 @@ func (p *Processor) startLeaderElection(svcCtx *servicecontext.Context, service
|
||||
default:
|
||||
leaseNamespace, serviceLease := lease.ServiceName(service)
|
||||
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
|
||||
// The lease is retired once its last service is gone, so an absent one means
|
||||
// this loop has nothing left to elect for.
|
||||
l := p.leaseMgr.Get(id)
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
l.Lock()
|
||||
|
||||
if !l.Elected.Load() {
|
||||
|
||||
@@ -2,7 +2,6 @@ package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
@@ -67,41 +66,13 @@ func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *strin
|
||||
|
||||
b.clearEgress(lastKnownGoodEndpoint, service)
|
||||
|
||||
if svcCtx.LeaderCancel != nil {
|
||||
svcCtx.LeaderCancel()
|
||||
}
|
||||
svcCtx.CallLeaderCancel()
|
||||
}
|
||||
|
||||
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
return b.getAllEndpoints(service, id)
|
||||
}
|
||||
|
||||
func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error {
|
||||
// When no-leader-elecition mode
|
||||
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
|
||||
// find all existing local endpoints
|
||||
endpoints, err := b.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
// If there were local endpoints deleted
|
||||
if len(endpoints) > 0 {
|
||||
b.deleteAction(ctx, service)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BGP) deleteAction(ctx context.Context, service *v1.Service) {
|
||||
b.clearBGPHosts(ctx, service)
|
||||
}
|
||||
|
||||
func (b *BGP) clearBGPHosts(ctx context.Context, service *v1.Service) {
|
||||
ClearBGPHosts(ctx, service, b.instances, b.bgpServer)
|
||||
}
|
||||
|
||||
func (b *BGP) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ type endpointWorker interface {
|
||||
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service)
|
||||
getEndpoints(service *v1.Service, id string) ([]string, error)
|
||||
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
|
||||
delete(ctx context.Context, service *v1.Service, id string) error
|
||||
setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error
|
||||
}
|
||||
|
||||
@@ -65,9 +64,7 @@ func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service) erro
|
||||
|
||||
func (g *generic) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
|
||||
g.clearEgress(lastKnownGoodEndpoint, service)
|
||||
if svcCtx.LeaderCancel != nil {
|
||||
svcCtx.LeaderCancel()
|
||||
}
|
||||
svcCtx.CallLeaderCancel()
|
||||
}
|
||||
|
||||
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
|
||||
@@ -105,10 +102,6 @@ func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, err
|
||||
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
|
||||
}
|
||||
|
||||
func (g *generic) delete(_ context.Context, _ *v1.Service, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *generic) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -66,9 +66,7 @@ func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpo
|
||||
|
||||
rt.clearEgress(lastKnownGoodEndpoint, service)
|
||||
|
||||
if svcCtx.LeaderCancel != nil {
|
||||
svcCtx.LeaderCancel()
|
||||
}
|
||||
svcCtx.CallLeaderCancel()
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
@@ -82,28 +80,6 @@ func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) delete(_ context.Context, service *v1.Service, id string) error {
|
||||
// When no-leader-elecition mode
|
||||
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
|
||||
// find all existing local endpoints
|
||||
endpoints, err := rt.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
// If there were local endpoints deleted
|
||||
if len(endpoints) > 0 {
|
||||
rt.deleteAction(service)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) deleteAction(service *v1.Service) {
|
||||
ClearRoutes(service, rt.instances, rt.routeMgr)
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error {
|
||||
inst := instance.FindServiceInstance(service, *rt.instances)
|
||||
if inst == nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/metrics"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func TestShouldAllowReconcileWithoutEndpoints(t *testing.T) {
|
||||
@@ -47,6 +49,191 @@ type fakeWorker struct {
|
||||
processCalled bool
|
||||
}
|
||||
|
||||
type annotationUpdate struct {
|
||||
endpoint string
|
||||
endpointIPv6 string
|
||||
}
|
||||
|
||||
type recordingProvider struct {
|
||||
providers.Provider
|
||||
updates []annotationUpdate
|
||||
}
|
||||
|
||||
func (p *recordingProvider) UpdateServiceAnnotation(_ context.Context, endpoint, endpointIPv6 string,
|
||||
_ *v1.Service, _ *kubernetes.Clientset) error {
|
||||
p.updates = append(p.updates, annotationUpdate{endpoint: endpoint, endpointIPv6: endpointIPv6})
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestUpdateAnnotationsZeroEndpointsThenSameEndpoint(t *testing.T) {
|
||||
for _, enableEndpoints := range []bool{true, false} {
|
||||
providerName := "EndpointSlices"
|
||||
provider := providers.NewEndpointslices()
|
||||
if enableEndpoints {
|
||||
providerName = "Endpoints"
|
||||
provider = providers.NewEndpoints()
|
||||
}
|
||||
|
||||
for _, family := range []struct {
|
||||
name string
|
||||
endpoint string
|
||||
other string
|
||||
egressIPv6 bool
|
||||
}{
|
||||
{name: "IPv4", endpoint: "10.0.0.1", other: "fd00::1"},
|
||||
{name: "IPv6", endpoint: "fd00::1", other: "10.0.0.1", egressIPv6: true},
|
||||
} {
|
||||
t.Run(providerName+"/"+family.name, func(t *testing.T) {
|
||||
annotations := map[string]string{kubevip.Egress: "true"}
|
||||
if family.egressIPv6 {
|
||||
annotations[kubevip.EgressIPv6] = "true"
|
||||
}
|
||||
if !enableEndpoints {
|
||||
if family.egressIPv6 {
|
||||
annotations[kubevip.ActiveEndpoint] = family.other
|
||||
annotations[kubevip.ActiveEndpointIPv6] = family.endpoint
|
||||
} else {
|
||||
annotations[kubevip.ActiveEndpoint] = family.endpoint
|
||||
annotations[kubevip.ActiveEndpointIPv6] = family.other
|
||||
}
|
||||
} else {
|
||||
annotations[kubevip.ActiveEndpoint] = family.endpoint
|
||||
}
|
||||
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
|
||||
}}
|
||||
serviceInstance := &instance.Instance{ServiceSnapshot: service.DeepCopy()}
|
||||
instances := []*instance.Instance{serviceInstance}
|
||||
recorder := &recordingProvider{Provider: provider}
|
||||
processor := &Processor{
|
||||
config: &kubevip.Config{EnableEndpoints: enableEndpoints},
|
||||
provider: recorder,
|
||||
instances: &instances,
|
||||
}
|
||||
|
||||
updateSnapshot := func(_ context.Context, updated *v1.Service) error {
|
||||
serviceInstance.ServiceSnapshot = updated
|
||||
return nil
|
||||
}
|
||||
|
||||
noEndpoint := ""
|
||||
processor.updateAnnotations(service, &noEndpoint, nil, updateSnapshot)
|
||||
repopulatedEndpoint := family.endpoint
|
||||
processor.updateAnnotations(service, &repopulatedEndpoint, nil, updateSnapshot)
|
||||
|
||||
cleared := annotationUpdate{}
|
||||
repopulated := annotationUpdate{endpoint: family.endpoint}
|
||||
if !enableEndpoints {
|
||||
if family.egressIPv6 {
|
||||
cleared = annotationUpdate{endpoint: family.other}
|
||||
repopulated = annotationUpdate{endpoint: family.other, endpointIPv6: family.endpoint}
|
||||
} else {
|
||||
cleared = annotationUpdate{endpointIPv6: family.other}
|
||||
repopulated = annotationUpdate{endpoint: family.endpoint, endpointIPv6: family.other}
|
||||
}
|
||||
}
|
||||
want := []annotationUpdate{cleared, repopulated}
|
||||
if len(recorder.updates) != len(want) {
|
||||
t.Fatalf("annotation updates = %+v, want %+v", recorder.updates, want)
|
||||
}
|
||||
for index := range want {
|
||||
if recorder.updates[index] != want[index] {
|
||||
t.Errorf("annotation update %d = %+v, want %+v", index, recorder.updates[index], want[index])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateAnnotationsEndpointSlicesClearsConfiguredFamily(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
egressIPv6 bool
|
||||
want annotationUpdate
|
||||
}{
|
||||
{name: "IPv4", want: annotationUpdate{endpointIPv6: "fd00::1"}},
|
||||
{name: "IPv6", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
annotations := map[string]string{
|
||||
kubevip.Egress: "true",
|
||||
kubevip.ActiveEndpoint: "10.0.0.1",
|
||||
kubevip.ActiveEndpointIPv6: "fd00::1",
|
||||
}
|
||||
if test.egressIPv6 {
|
||||
annotations[kubevip.EgressIPv6] = "true"
|
||||
}
|
||||
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
|
||||
}}
|
||||
instances := []*instance.Instance{{ServiceSnapshot: service.DeepCopy()}}
|
||||
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
|
||||
processor := &Processor{
|
||||
config: &kubevip.Config{EnableEndpoints: false},
|
||||
provider: recorder,
|
||||
instances: &instances,
|
||||
}
|
||||
|
||||
noEndpoint := ""
|
||||
processor.updateAnnotations(service, &noEndpoint, nil, func(context.Context, *v1.Service) error { return nil })
|
||||
|
||||
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
|
||||
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateAnnotationsValidatesEndpointFamily(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
egressIPv6 bool
|
||||
want annotationUpdate
|
||||
wantUpdate bool
|
||||
}{
|
||||
{name: "invalid address", endpoint: "not-an-ip"},
|
||||
{name: "IPv6 endpoint for IPv4 egress", endpoint: "fd00::1"},
|
||||
{name: "IPv4 endpoint for IPv6 egress", endpoint: "10.0.0.1", egressIPv6: true},
|
||||
{name: "IPv4 endpoint", endpoint: "10.0.0.2", want: annotationUpdate{endpoint: "10.0.0.2", endpointIPv6: "fd00::1"}, wantUpdate: true},
|
||||
{name: "IPv6 endpoint", endpoint: "fd00::2", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1", endpointIPv6: "fd00::2"}, wantUpdate: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
annotations := map[string]string{
|
||||
kubevip.Egress: "true",
|
||||
kubevip.ActiveEndpoint: "10.0.0.1",
|
||||
kubevip.ActiveEndpointIPv6: "fd00::1",
|
||||
}
|
||||
if test.egressIPv6 {
|
||||
annotations[kubevip.EgressIPv6] = "true"
|
||||
}
|
||||
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-service", Namespace: "default", Annotations: annotations,
|
||||
}}
|
||||
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
|
||||
processor := &Processor{
|
||||
config: &kubevip.Config{EnableEndpoints: false},
|
||||
provider: recorder,
|
||||
}
|
||||
|
||||
processor.updateAnnotations(service, &test.endpoint, nil, nil)
|
||||
|
||||
if !test.wantUpdate {
|
||||
if len(recorder.updates) != 0 {
|
||||
t.Fatalf("annotation updates = %+v, want none", recorder.updates)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
|
||||
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeWorker) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
|
||||
f.processCalled = true
|
||||
return nil
|
||||
@@ -58,14 +245,101 @@ func (f *fakeWorker) clear(_ *servicecontext.Context, _ *string, _ *v1.Service)
|
||||
|
||||
func (f *fakeWorker) getEndpoints(_ *v1.Service, _ string) ([]string, error) { return f.endpoints, nil }
|
||||
func (f *fakeWorker) removeEgress(_ *v1.Service, _ *string) {}
|
||||
func (f *fakeWorker) delete(_ context.Context, _ *v1.Service, _ string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeWorker) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
|
||||
// TestReconcile_RecomputesRemainingEndpoints asserts that deleting one EndpointSlice
|
||||
// reconciles against the endpoints that remain, instead of assuming the service
|
||||
// lost all of them.
|
||||
func TestReconcile_RecomputesRemainingEndpoints(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
remaining []string
|
||||
lastKnown string
|
||||
expectReady bool
|
||||
expectClear bool
|
||||
expectProcess bool
|
||||
expectedLastKnown string
|
||||
}{
|
||||
{
|
||||
name: "remaining endpoints keep the service up",
|
||||
remaining: []string{"10.0.0.2"},
|
||||
lastKnown: "10.0.0.2",
|
||||
expectReady: true,
|
||||
expectProcess: true,
|
||||
expectedLastKnown: "10.0.0.2",
|
||||
},
|
||||
{
|
||||
name: "stale last known endpoint moves to a survivor",
|
||||
remaining: []string{"10.0.0.2"},
|
||||
lastKnown: "10.0.0.1",
|
||||
expectReady: true,
|
||||
expectProcess: true,
|
||||
expectedLastKnown: "10.0.0.2",
|
||||
},
|
||||
{
|
||||
name: "last endpoint removed tears the service down",
|
||||
remaining: nil,
|
||||
lastKnown: "10.0.0.1",
|
||||
expectReady: false,
|
||||
expectClear: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
worker := &fakeWorker{endpoints: test.remaining}
|
||||
p := &Processor{
|
||||
config: &kubevip.Config{},
|
||||
provider: providers.NewEndpointslices(),
|
||||
worker: worker,
|
||||
}
|
||||
|
||||
svcCtx := servicecontext.New(context.Background())
|
||||
svcCtx.SignalReadiness()
|
||||
|
||||
lastKnown := test.lastKnown
|
||||
restart, err := p.Reconcile(
|
||||
svcCtx,
|
||||
watch.Event{
|
||||
Type: watch.Deleted,
|
||||
Object: &discoveryv1.EndpointSlice{ObjectMeta: metav1.ObjectMeta{Name: "slice-1"}},
|
||||
},
|
||||
&lastKnown,
|
||||
&v1.Service{Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal}},
|
||||
"node-1",
|
||||
func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error { return nil },
|
||||
&sync.WaitGroup{},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile returned error: %v", err)
|
||||
}
|
||||
if restart {
|
||||
t.Fatal("Reconcile unexpectedly requested restart")
|
||||
}
|
||||
|
||||
if ready := svcCtx.Signalled.Load(); ready != test.expectReady {
|
||||
t.Fatalf("readiness mismatch: expected %v, got %v", test.expectReady, ready)
|
||||
}
|
||||
if worker.clearCalled != test.expectClear {
|
||||
t.Fatalf("clearCalled mismatch: expected %v, got %v", test.expectClear, worker.clearCalled)
|
||||
}
|
||||
if worker.processCalled != test.expectProcess {
|
||||
t.Fatalf("processCalled mismatch: expected %v, got %v", test.expectProcess, worker.processCalled)
|
||||
}
|
||||
if test.expectedLastKnown != "" && lastKnown != test.expectedLastKnown {
|
||||
t.Fatalf("lastKnownGoodEndpoint mismatch: expected %q, got %q", test.expectedLastKnown, lastKnown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcile_ZeroEndpointsBehavior(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
run := func(t *testing.T, service *v1.Service, presetSignalled bool, expectReady bool, expectClear bool, expectProcess bool) {
|
||||
@@ -83,7 +357,7 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
|
||||
svcCtx.SignalReadiness()
|
||||
}
|
||||
|
||||
restart, err := p.AddOrModify(
|
||||
restart, err := p.Reconcile(
|
||||
svcCtx,
|
||||
watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
|
||||
new(string),
|
||||
@@ -95,10 +369,10 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("AddOrModify returned error: %v", err)
|
||||
t.Fatalf("Reconcile returned error: %v", err)
|
||||
}
|
||||
if restart {
|
||||
t.Fatal("AddOrModify unexpectedly requested restart")
|
||||
t.Fatal("Reconcile unexpectedly requested restart")
|
||||
}
|
||||
|
||||
if ready := svcCtx.Signalled.Load(); ready != expectReady {
|
||||
@@ -137,15 +411,15 @@ func TestAddOrModify_ZeroEndpointsBehavior(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestAddOrModify_ServicesElectionStartsOnce asserts that repeated endpoint events
|
||||
// TestReconcile_ServicesElectionStartsOnce asserts that repeated endpoint events
|
||||
// for the same service start the leader-election restart loop exactly once.
|
||||
//
|
||||
// AddOrModify runs on every EndpointSlice add/modify/resync event, and the loop it
|
||||
// Reconcile runs on every EndpointSlice add/modify/resync event, and the loop it
|
||||
// starts only returns once the service context is cancelled. Starting it per event
|
||||
// therefore accumulates duplicate goroutines that all contend on the same lease.
|
||||
//
|
||||
// See https://github.com/kube-vip/kube-vip/issues/1665.
|
||||
func TestAddOrModify_ServicesElectionStartsOnce(t *testing.T) {
|
||||
func TestReconcile_ServicesElectionStartsOnce(t *testing.T) {
|
||||
config := &kubevip.Config{
|
||||
EnableServicesElection: true,
|
||||
LeaderElectionType: "kubernetes",
|
||||
@@ -189,13 +463,13 @@ func TestAddOrModify_ServicesElectionStartsOnce(t *testing.T) {
|
||||
|
||||
// Three endpoint events, as a flapping backend pod would produce.
|
||||
for range 3 {
|
||||
restart, err := p.AddOrModify(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
|
||||
restart, err := p.Reconcile(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
|
||||
new(string), service, "node-1", serviceFunc, wg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("AddOrModify returned error: %v", err)
|
||||
t.Fatalf("Reconcile returned error: %v", err)
|
||||
}
|
||||
if restart {
|
||||
t.Fatal("AddOrModify unexpectedly requested restart")
|
||||
t.Fatal("Reconcile unexpectedly requested restart")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -212,8 +212,8 @@ func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEnd
|
||||
}
|
||||
}
|
||||
|
||||
if svcCtx.LeaderCancel != nil {
|
||||
svcCtx.LeaderCancel()
|
||||
if svcCtx != nil {
|
||||
svcCtx.CallLeaderCancel()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,14 +242,6 @@ func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoin
|
||||
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
|
||||
}
|
||||
|
||||
// delete removes all DNAT rules for a service
|
||||
func (w *wireguardWorker) delete(ctx context.Context, service *v1.Service, id string) error {
|
||||
log.Info("[wireguard] deleting DNAT rules for service", "service", service.Name, "namespace", service.Namespace)
|
||||
|
||||
w.clear(nil, nil, service)
|
||||
return nil
|
||||
}
|
||||
|
||||
// setInstanceEndpointsStatus updates the endpoint status on the service instance
|
||||
func (w *wireguardWorker) setInstanceEndpointsStatus(_ context.Context, service *v1.Service, endpoints []string) error {
|
||||
hasEndpoints := len(endpoints) > 0
|
||||
|
||||
14
pkg/endpoints/endpoints_wireguard_test.go
Normal file
14
pkg/endpoints/endpoints_wireguard_test.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestWireguardClearDoesNotDereferenceNilServiceContext(t *testing.T) {
|
||||
worker := &wireguardWorker{}
|
||||
service := &v1.Service{}
|
||||
|
||||
worker.clear(nil, nil, service)
|
||||
}
|
||||
@@ -61,6 +61,18 @@ func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelF
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteObject drops the tracked object. A service is backed by exactly one
|
||||
// v1.Endpoints object, so there is nothing to match on and the cache is reset.
|
||||
func (ep *Endpoints) DeleteObject(endpoints runtime.Object) error {
|
||||
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
|
||||
if _, ok := endpoints.(*v1.Endpoints); !ok {
|
||||
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
|
||||
}
|
||||
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
|
||||
ep.endpoints = &v1.Endpoints{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
|
||||
result := []string{}
|
||||
for subset := range ep.endpoints.Subsets {
|
||||
@@ -87,7 +99,7 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
|
||||
continue
|
||||
}
|
||||
// 2. Compare the Hostname (only useful if address.NodeName is not available)
|
||||
if id == address.Hostname {
|
||||
if address.NodeName == nil && id == address.Hostname {
|
||||
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
|
||||
localEndpoints = append(localEndpoints, address.IP)
|
||||
continue
|
||||
|
||||
@@ -20,15 +20,14 @@ import (
|
||||
)
|
||||
|
||||
type Endpointslices struct {
|
||||
label string
|
||||
endpointsv4 []discoveryv1.Endpoint
|
||||
endpointsv6 []discoveryv1.Endpoint
|
||||
ports []discoveryv1.EndpointPort
|
||||
label string
|
||||
slices map[string]*discoveryv1.EndpointSlice
|
||||
}
|
||||
|
||||
func NewEndpointslices() Provider {
|
||||
return &Endpointslices{
|
||||
label: "endpointslices",
|
||||
label: "endpointslices",
|
||||
slices: make(map[string]*discoveryv1.EndpointSlice),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,56 +58,71 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
|
||||
return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
|
||||
}
|
||||
|
||||
if eps.AddressType == discoveryv1.AddressTypeIPv6 {
|
||||
ep.endpointsv6 = eps.Endpoints
|
||||
} else {
|
||||
ep.endpointsv4 = eps.Endpoints
|
||||
if ep.slices == nil {
|
||||
ep.slices = make(map[string]*discoveryv1.EndpointSlice)
|
||||
}
|
||||
|
||||
// Store ports for resolving named ports
|
||||
ep.ports = eps.Ports
|
||||
ep.slices[eps.Name] = eps.DeepCopy()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) DeleteObject(endpoints runtime.Object) error {
|
||||
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
|
||||
if !ok {
|
||||
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
|
||||
}
|
||||
delete(ep.slices, eps.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// isServing reports whether an endpoint should receive traffic. Per the
|
||||
// EndpointConditions godoc a nil Serving defers to Ready, and a nil Ready is an
|
||||
// unknown state that consumers should interpret as ready.
|
||||
func isServing(conditions discoveryv1.EndpointConditions) bool {
|
||||
serving := conditions.Serving
|
||||
if serving == nil {
|
||||
serving = conditions.Ready
|
||||
}
|
||||
return serving == nil || *serving
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
|
||||
result := []string{}
|
||||
for _, e := range ep.endpointsv4 {
|
||||
result = append(result, e.Addresses...)
|
||||
}
|
||||
for _, e := range ep.endpointsv6 {
|
||||
result = append(result, e.Addresses...)
|
||||
for _, eps := range ep.slices {
|
||||
for _, e := range eps.Endpoints {
|
||||
if !isServing(e.Conditions) {
|
||||
continue
|
||||
}
|
||||
result = append(result, e.Addresses...)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
|
||||
var localEndpoints []string
|
||||
tmpEps := []discoveryv1.Endpoint{}
|
||||
|
||||
tmpEps = append(tmpEps, ep.endpointsv4...)
|
||||
tmpEps = append(tmpEps, ep.endpointsv6...)
|
||||
|
||||
for _, endpoint := range tmpEps {
|
||||
if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
|
||||
continue
|
||||
}
|
||||
for _, address := range endpoint.Addresses {
|
||||
// 1. Compare the Nodename
|
||||
if endpoint.NodeName != nil && id == *endpoint.NodeName {
|
||||
if endpoint.Hostname != nil {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
|
||||
} else {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
|
||||
}
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
for _, eps := range ep.slices {
|
||||
for _, endpoint := range eps.Endpoints {
|
||||
if !isServing(endpoint.Conditions) {
|
||||
continue
|
||||
}
|
||||
for _, address := range endpoint.Addresses {
|
||||
// 1. Compare the Nodename
|
||||
if endpoint.NodeName != nil && id == *endpoint.NodeName {
|
||||
if endpoint.Hostname != nil {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
|
||||
} else {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
|
||||
}
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
continue
|
||||
}
|
||||
|
||||
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
|
||||
if endpoint.Hostname != nil && id == *endpoint.Hostname {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
|
||||
if endpoint.NodeName == nil && endpoint.Hostname != nil && id == *endpoint.Hostname {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -153,9 +167,11 @@ func (ep *Endpointslices) GetLabel() string {
|
||||
|
||||
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
|
||||
return ResolvePortWithLookup(servicePort, func(name string) int32 {
|
||||
for _, p := range ep.ports {
|
||||
if p.Name != nil && *p.Name == name && p.Port != nil {
|
||||
return *p.Port
|
||||
for _, eps := range ep.slices {
|
||||
for _, p := range eps.Ports {
|
||||
if p.Name != nil && *p.Name == name && p.Port != nil {
|
||||
return *p.Port
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
|
||||
149
pkg/endpoints/providers/endpointslices_test.go
Normal file
149
pkg/endpoints/providers/endpointslices_test.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestEndpointslicesTracksAndDeletesSlices(t *testing.T) {
|
||||
provider := NewEndpointslices().(*Endpointslices)
|
||||
serving := true
|
||||
nodeName := "node-1"
|
||||
|
||||
slice1 := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{{
|
||||
Addresses: []string{"10.0.0.1"},
|
||||
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
|
||||
NodeName: &nodeName,
|
||||
}},
|
||||
}
|
||||
slice2 := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "slice-2"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{{
|
||||
Addresses: []string{"10.0.0.2"},
|
||||
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
|
||||
NodeName: &nodeName,
|
||||
}},
|
||||
}
|
||||
|
||||
for _, slice := range []*discoveryv1.EndpointSlice{slice1, slice2} {
|
||||
if err := provider.LoadObject(slice, func() {}); err != nil {
|
||||
t.Fatalf("LoadObject returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
assertEndpoints(t, provider, []string{"10.0.0.1", "10.0.0.2"})
|
||||
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.1", "10.0.0.2"})
|
||||
|
||||
if err := provider.DeleteObject(slice1); err != nil {
|
||||
t.Fatalf("DeleteObject returned error: %v", err)
|
||||
}
|
||||
assertEndpoints(t, provider, []string{"10.0.0.2"})
|
||||
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.2"})
|
||||
|
||||
if err := provider.DeleteObject(slice2); err != nil {
|
||||
t.Fatalf("DeleteObject returned error: %v", err)
|
||||
}
|
||||
assertEndpoints(t, provider, nil)
|
||||
assertLocalEndpoints(t, provider, nodeName, nil)
|
||||
}
|
||||
|
||||
func TestEndpointslicesReplacingSliceUpdatesState(t *testing.T) {
|
||||
provider := NewEndpointslices().(*Endpointslices)
|
||||
first := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.1"}}},
|
||||
}
|
||||
replacement := first.DeepCopy()
|
||||
replacement.Endpoints[0].Addresses = []string{"10.0.0.2"}
|
||||
|
||||
if err := provider.LoadObject(first, context.CancelFunc(func() {})); err != nil {
|
||||
t.Fatalf("LoadObject returned error: %v", err)
|
||||
}
|
||||
if err := provider.LoadObject(replacement, context.CancelFunc(func() {})); err != nil {
|
||||
t.Fatalf("LoadObject returned error: %v", err)
|
||||
}
|
||||
|
||||
assertEndpoints(t, provider, []string{"10.0.0.2"})
|
||||
}
|
||||
|
||||
func TestEndpointslicesEndpointConditions(t *testing.T) {
|
||||
yes, no := true, false
|
||||
nodeName := "node-1"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
conditions discoveryv1.EndpointConditions
|
||||
want []string
|
||||
}{
|
||||
{"serving true", discoveryv1.EndpointConditions{Serving: &yes}, []string{"10.0.0.1"}},
|
||||
{"serving false", discoveryv1.EndpointConditions{Serving: &no}, nil},
|
||||
{"serving false overrides ready true", discoveryv1.EndpointConditions{Serving: &no, Ready: &yes}, nil},
|
||||
{"nil serving defers to ready true", discoveryv1.EndpointConditions{Ready: &yes}, []string{"10.0.0.1"}},
|
||||
{"nil serving defers to ready false", discoveryv1.EndpointConditions{Ready: &no}, nil},
|
||||
{"both nil is treated as ready", discoveryv1.EndpointConditions{}, []string{"10.0.0.1"}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
provider := NewEndpointslices().(*Endpointslices)
|
||||
slice := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{{
|
||||
Addresses: []string{"10.0.0.1"},
|
||||
Conditions: test.conditions,
|
||||
NodeName: &nodeName,
|
||||
}},
|
||||
}
|
||||
if err := provider.LoadObject(slice, func() {}); err != nil {
|
||||
t.Fatalf("LoadObject returned error: %v", err)
|
||||
}
|
||||
// Cluster and Local policy have to agree on which endpoints are usable.
|
||||
assertEndpoints(t, provider, test.want)
|
||||
assertLocalEndpoints(t, provider, nodeName, test.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertEndpoints(t *testing.T, provider *Endpointslices, want []string) {
|
||||
t.Helper()
|
||||
got, err := provider.GetAllEndpoints()
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllEndpoints returned error: %v", err)
|
||||
}
|
||||
assertStringSet(t, got, want)
|
||||
}
|
||||
|
||||
func assertLocalEndpoints(t *testing.T, provider *Endpointslices, nodeName string, want []string) {
|
||||
t.Helper()
|
||||
got, err := provider.GetLocalEndpoints(nodeName, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("GetLocalEndpoints returned error: %v", err)
|
||||
}
|
||||
assertStringSet(t, got, want)
|
||||
}
|
||||
|
||||
func assertStringSet(t *testing.T, got, want []string) {
|
||||
t.Helper()
|
||||
counts := map[string]int{}
|
||||
for _, value := range got {
|
||||
counts[value]++
|
||||
}
|
||||
for _, value := range want {
|
||||
counts[value]--
|
||||
}
|
||||
for value, count := range counts {
|
||||
if count != 0 {
|
||||
t.Fatalf("endpoint set mismatch for %q: got %v, want %v", value, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ type Provider interface {
|
||||
GetLabel() string
|
||||
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
|
||||
LoadObject(runtime.Object, context.CancelFunc) error
|
||||
DeleteObject(runtime.Object) error
|
||||
// ResolvePort resolves a service port to the actual target port.
|
||||
// For named ports, it looks up the port number from the endpoint.
|
||||
// For numeric ports, it returns the port as-is.
|
||||
|
||||
319
pkg/endpoints/providers/providers_test.go
Normal file
319
pkg/endpoints/providers/providers_test.go
Normal file
@@ -0,0 +1,319 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestEndpointProvidersParityForLocalAndAllEndpoints(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
nodeA := "node-a"
|
||||
nodeB := "node-b"
|
||||
serving := true
|
||||
v4Addresses := []discoveryv1.Endpoint{
|
||||
{Addresses: []string{"10.0.0.1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
|
||||
{Addresses: []string{"10.0.0.2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
|
||||
}
|
||||
v6Addresses := []discoveryv1.Endpoint{
|
||||
{Addresses: []string{"2001:db8::1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
|
||||
{Addresses: []string{"2001:db8::2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
|
||||
}
|
||||
|
||||
legacy := NewEndpoints()
|
||||
//nolint:staticcheck // this test covers the deprecated legacy Endpoints provider on purpose
|
||||
if err := legacy.LoadObject(&v1.Endpoints{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
|
||||
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
|
||||
Subsets: []v1.EndpointSubset{{
|
||||
Addresses: []v1.EndpointAddress{
|
||||
{IP: "10.0.0.1", NodeName: &nodeA},
|
||||
{IP: "10.0.0.2", NodeName: &nodeB},
|
||||
{IP: "2001:db8::1", NodeName: &nodeA},
|
||||
{IP: "2001:db8::2", NodeName: &nodeB},
|
||||
},
|
||||
}},
|
||||
}, func() {}); err != nil {
|
||||
t.Fatalf("loading legacy Endpoints: %v", err)
|
||||
}
|
||||
|
||||
slices := NewEndpointslices()
|
||||
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service-v4", Namespace: "default"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: v4Addresses,
|
||||
}, func() {}); err != nil {
|
||||
t.Fatalf("loading IPv4 EndpointSlice: %v", err)
|
||||
}
|
||||
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service-v6", Namespace: "default"},
|
||||
AddressType: discoveryv1.AddressTypeIPv6,
|
||||
Endpoints: v6Addresses,
|
||||
}, func() {}); err != nil {
|
||||
t.Fatalf("loading IPv6 EndpointSlice: %v", err)
|
||||
}
|
||||
|
||||
legacyAll, err := legacy.GetAllEndpoints()
|
||||
if err != nil {
|
||||
t.Fatalf("legacy GetAllEndpoints() error = %v", err)
|
||||
}
|
||||
sliceAll, err := slices.GetAllEndpoints()
|
||||
if err != nil {
|
||||
t.Fatalf("EndpointSlice GetAllEndpoints() error = %v", err)
|
||||
}
|
||||
wantAll := endpointSet([]string{"10.0.0.1", "10.0.0.2", "2001:db8::1", "2001:db8::2"})
|
||||
if got := endpointSet(legacyAll); !reflect.DeepEqual(got, wantAll) {
|
||||
t.Errorf("legacy all endpoints = %v, want %v", got, wantAll)
|
||||
}
|
||||
if got := endpointSet(sliceAll); !reflect.DeepEqual(got, wantAll) {
|
||||
t.Errorf("EndpointSlice all endpoints = %v, want %v", got, wantAll)
|
||||
}
|
||||
|
||||
legacyLocal, err := legacy.GetLocalEndpoints(nodeA, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("legacy GetLocalEndpoints() error = %v", err)
|
||||
}
|
||||
sliceLocal, err := slices.GetLocalEndpoints(nodeA, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("EndpointSlice GetLocalEndpoints() error = %v", err)
|
||||
}
|
||||
wantLocal := endpointSet([]string{"10.0.0.1", "2001:db8::1"})
|
||||
if got := endpointSet(legacyLocal); !reflect.DeepEqual(got, wantLocal) {
|
||||
t.Errorf("legacy local endpoints = %v, want %v", got, wantLocal)
|
||||
}
|
||||
if got := endpointSet(sliceLocal); !reflect.DeepEqual(got, wantLocal) {
|
||||
t.Errorf("EndpointSlice local endpoints = %v, want %v", got, wantLocal)
|
||||
}
|
||||
|
||||
assertEndpointFamilies(t, legacyAll, 2, 2)
|
||||
assertEndpointFamilies(t, sliceAll, 2, 2)
|
||||
}
|
||||
|
||||
func TestEndpointSlicesLocalFilteringRequiresServingEndpoint(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node := "node-a"
|
||||
serving := true
|
||||
notServing := false
|
||||
provider := NewEndpointslices()
|
||||
if err := provider.LoadObject(&discoveryv1.EndpointSlice{
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{
|
||||
{Addresses: []string{"10.0.0.1"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
|
||||
{Addresses: []string{"10.0.0.2"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: ¬Serving}},
|
||||
},
|
||||
}, func() {}); err != nil {
|
||||
t.Fatalf("loading EndpointSlice: %v", err)
|
||||
}
|
||||
|
||||
local, err := provider.GetLocalEndpoints(node, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("GetLocalEndpoints() error = %v", err)
|
||||
}
|
||||
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("local endpoints = %v, want serving endpoints %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePortFromFakeClientObjects(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
|
||||
legacyObject := &v1.Endpoints{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
|
||||
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
|
||||
Subsets: []v1.EndpointSubset{{
|
||||
Ports: []v1.EndpointPort{{Name: "web", Port: 8080}},
|
||||
}},
|
||||
}
|
||||
legacyClient := fake.NewSimpleClientset(legacyObject)
|
||||
legacyLoaded, err := legacyClient.CoreV1().Endpoints("default").Get(t.Context(), "service", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("getting fake legacy Endpoints: %v", err)
|
||||
}
|
||||
legacy := NewEndpoints()
|
||||
if err := legacy.LoadObject(legacyLoaded, func() {}); err != nil {
|
||||
t.Fatalf("loading fake legacy Endpoints: %v", err)
|
||||
}
|
||||
|
||||
portName := "web"
|
||||
port := int32(8081)
|
||||
sliceObject := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service-slice", Namespace: "default"},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Ports: []discoveryv1.EndpointPort{{Name: &portName, Port: &port}},
|
||||
}
|
||||
sliceClient := fake.NewSimpleClientset(sliceObject)
|
||||
sliceLoaded, err := sliceClient.DiscoveryV1().EndpointSlices("default").Get(t.Context(), "service-slice", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("getting fake EndpointSlice: %v", err)
|
||||
}
|
||||
slices := NewEndpointslices()
|
||||
if err := slices.LoadObject(sliceLoaded, func() {}); err != nil {
|
||||
t.Fatalf("loading fake EndpointSlice: %v", err)
|
||||
}
|
||||
|
||||
namedPort := v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")}
|
||||
if got := legacy.ResolvePort(namedPort); got != 8080 {
|
||||
t.Errorf("legacy ResolvePort() = %d, want 8080", got)
|
||||
}
|
||||
if got := slices.ResolvePort(namedPort); got != 8081 {
|
||||
t.Errorf("EndpointSlice ResolvePort() = %d, want 8081", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePortWithLookup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
port v1.ServicePort
|
||||
lookup func(string) int32
|
||||
want int32
|
||||
}{
|
||||
{
|
||||
name: "numeric target port wins",
|
||||
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromInt(8080)},
|
||||
lookup: func(string) int32 { return 9090 },
|
||||
want: 8080,
|
||||
},
|
||||
{
|
||||
name: "named target port is looked up",
|
||||
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")},
|
||||
lookup: func(name string) int32 {
|
||||
if name == "web" {
|
||||
return 8081
|
||||
}
|
||||
return 0
|
||||
},
|
||||
want: 8081,
|
||||
},
|
||||
{
|
||||
name: "missing named target falls back to service port",
|
||||
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("missing")},
|
||||
lookup: func(string) int32 { return 0 },
|
||||
want: 80,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := ResolvePortWithLookup(tt.port, tt.lookup); got != tt.want {
|
||||
t.Errorf("ResolvePortWithLookup() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func endpointSet(endpoints []string) map[string]struct{} {
|
||||
result := make(map[string]struct{}, len(endpoints))
|
||||
for _, endpoint := range endpoints {
|
||||
result[endpoint] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func assertEndpointFamilies(t *testing.T, endpoints []string, wantIPv4, wantIPv6 int) {
|
||||
t.Helper()
|
||||
ipv4, ipv6 := 0, 0
|
||||
for _, endpoint := range endpoints {
|
||||
ip := net.ParseIP(endpoint)
|
||||
if ip == nil {
|
||||
t.Errorf("endpoint %q is not an IP address", endpoint)
|
||||
continue
|
||||
}
|
||||
if ip.To4() != nil {
|
||||
ipv4++
|
||||
} else {
|
||||
ipv6++
|
||||
}
|
||||
}
|
||||
if ipv4 != wantIPv4 || ipv6 != wantIPv6 {
|
||||
t.Errorf("endpoint families = IPv4 %d, IPv6 %d; want IPv4 %d, IPv6 %d", ipv4, ipv6, wantIPv4, wantIPv6)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointProvidersPreferNodeNameOverHostname(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
nodeA := "node-a"
|
||||
nodeB := "node-b"
|
||||
serving := true
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
load func(Provider) error
|
||||
}{
|
||||
{
|
||||
name: "legacy Endpoints",
|
||||
load: func(provider Provider) error {
|
||||
//nolint:staticcheck // the legacy provider is deliberately under test
|
||||
return provider.LoadObject(&v1.Endpoints{
|
||||
Subsets: []v1.EndpointSubset{{
|
||||
Addresses: []v1.EndpointAddress{{
|
||||
IP: "10.0.0.1",
|
||||
NodeName: &nodeB,
|
||||
Hostname: nodeA,
|
||||
}},
|
||||
}},
|
||||
}, func() {})
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "EndpointSlice",
|
||||
load: func(provider Provider) error {
|
||||
hostname := nodeA
|
||||
return provider.LoadObject(&discoveryv1.EndpointSlice{
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
Endpoints: []discoveryv1.Endpoint{{
|
||||
Addresses: []string{"10.0.0.1"},
|
||||
NodeName: &nodeB,
|
||||
Hostname: &hostname,
|
||||
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
|
||||
}},
|
||||
}, func() {})
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var provider Provider
|
||||
if tt.name == "legacy Endpoints" {
|
||||
provider = NewEndpoints()
|
||||
} else {
|
||||
provider = NewEndpointslices()
|
||||
}
|
||||
if err := tt.load(provider); err != nil {
|
||||
t.Fatalf("LoadObject() error = %v", err)
|
||||
}
|
||||
|
||||
local, err := provider.GetLocalEndpoints(nodeA, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeA, err)
|
||||
}
|
||||
if len(local) != 0 {
|
||||
t.Fatalf("GetLocalEndpoints(%q) = %v, want no endpoints", nodeA, local)
|
||||
}
|
||||
|
||||
local, err = provider.GetLocalEndpoints(nodeB, &kubevip.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeB, err)
|
||||
}
|
||||
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("GetLocalEndpoints(%q) = %v, want %v", nodeB, got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -28,7 +27,6 @@ const (
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
expectSuccess(startEtcd(ctx), "starting etcd")
|
||||
|
||||
@@ -203,6 +203,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
|
||||
SingleNode: true,
|
||||
EnableARP: config.EnableARP,
|
||||
EnableBGP: config.EnableBGP,
|
||||
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
|
||||
VIPSubnet: subnet,
|
||||
EnableRoutingTable: config.EnableRoutingTable,
|
||||
RoutingTableID: config.RoutingTableID,
|
||||
@@ -269,6 +270,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
|
||||
SingleNode: true,
|
||||
EnableARP: config.EnableARP,
|
||||
EnableBGP: config.EnableBGP,
|
||||
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
|
||||
VIPSubnet: config.VIPSubnet,
|
||||
EnableRoutingTable: config.EnableRoutingTable,
|
||||
RoutingTableID: config.RoutingTableID,
|
||||
@@ -341,6 +343,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("context error while starting DHCPv4 for %s/%s: error: %w",
|
||||
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
|
||||
case err := <-instance.DHCPv4Client.ErrorChannel():
|
||||
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
|
||||
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
|
||||
@@ -356,6 +361,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("context error while starting DHCPv6 for %s/%s: error: %w",
|
||||
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
|
||||
case err := <-instance.DHCPv6Client.ErrorChannel():
|
||||
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
|
||||
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
|
||||
@@ -652,7 +660,8 @@ func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uin
|
||||
|
||||
wg.Go(func() {
|
||||
if err := client.Start(ctx); err != nil {
|
||||
log.Error("[instance] DHCP client error: %w")
|
||||
log.Error("[instance] DHCP client", "error", err)
|
||||
client.Stop()
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
62
pkg/instance/instance_bgp_attach_test.go
Normal file
62
pkg/instance/instance_bgp_attach_test.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package instance_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/route"
|
||||
)
|
||||
|
||||
func TestNewInstance_PropagatesBGPAttachIPToInterface(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
attach bool
|
||||
}{
|
||||
{name: "attach enabled is propagated", attach: true},
|
||||
{name: "attach disabled is propagated", attach: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
globalConfig := &kubevip.Config{
|
||||
Interface: "lo",
|
||||
VIPSubnet: "32",
|
||||
EnableBGP: true,
|
||||
BGPAttachIPToInterface: tt.attach,
|
||||
}
|
||||
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-svc",
|
||||
Namespace: "default",
|
||||
Annotations: map[string]string{
|
||||
kubevip.LoadbalancerIPAnnotation: "10.0.1.2",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
inst, err := instance.NewInstance(context.Background(), svc, globalConfig,
|
||||
networkinterface.NewManager(), arp.NewManager(globalConfig), route.NewManager(),
|
||||
nil, &sync.WaitGroup{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewInstance() error = %v", err)
|
||||
}
|
||||
|
||||
if len(inst.VIPConfigs) != 1 {
|
||||
t.Fatalf("VIPConfigs len = %d, want 1", len(inst.VIPConfigs))
|
||||
}
|
||||
|
||||
if got := inst.VIPConfigs[0].BGPAttachIPToInterface; got != tt.attach {
|
||||
t.Fatalf("BGPAttachIPToInterface = %t, want %t", got, tt.attach)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -87,21 +86,6 @@ type IPTables struct {
|
||||
|
||||
nftables bool
|
||||
}
|
||||
|
||||
// Stat represents a structured statistic entry.
|
||||
type Stat struct {
|
||||
Packets uint64 `json:"pkts"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
Target string `json:"target"`
|
||||
Protocol string `json:"prot"`
|
||||
Opt string `json:"opt"`
|
||||
Input string `json:"in"`
|
||||
Output string `json:"out"`
|
||||
Source *net.IPNet `json:"source"`
|
||||
Destination *net.IPNet `json:"destination"`
|
||||
Options string `json:"options"`
|
||||
}
|
||||
|
||||
type Option func(*IPTables)
|
||||
|
||||
func IPFamily(proto Protocol) Option {
|
||||
@@ -251,16 +235,6 @@ func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) err
|
||||
return err
|
||||
}
|
||||
|
||||
// List rules in specified table/chain
|
||||
func (ipt *IPTables) ListByID(table, chain string, id int) (string, error) {
|
||||
args := []string{"-t", table, "-S", chain, strconv.Itoa(id)}
|
||||
rule, err := ipt.executeList(args)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rule[0], nil
|
||||
}
|
||||
|
||||
// List rules in specified table/chain
|
||||
func (ipt *IPTables) List(table, chain string) ([]string, error) {
|
||||
args := []string{"-t", table, "-S", chain}
|
||||
@@ -313,129 +287,6 @@ func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Stats lists rules including the byte and packet counts
|
||||
func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
|
||||
args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
|
||||
lines, err := ipt.executeList(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
appendSubnet := func(addr string) string {
|
||||
if strings.IndexByte(addr, byte('/')) < 0 {
|
||||
if strings.IndexByte(addr, '.') < 0 {
|
||||
return addr + "/128"
|
||||
}
|
||||
return addr + "/32"
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
ipv6 := ipt.proto == ProtocolIPv6
|
||||
|
||||
rows := [][]string{}
|
||||
for i, line := range lines {
|
||||
// Skip over chain name and field header
|
||||
if i < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Fields:
|
||||
// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
|
||||
line = strings.TrimSpace(line)
|
||||
fields := strings.Fields(line)
|
||||
|
||||
// The ip6tables verbose output cannot be naively split due to the default "opt"
|
||||
// field containing 2 single spaces.
|
||||
if ipv6 {
|
||||
// Check if field 6 is "opt" or "source" address
|
||||
dest := fields[6]
|
||||
ip, _, _ := net.ParseCIDR(dest)
|
||||
if ip == nil {
|
||||
ip = net.ParseIP(dest)
|
||||
}
|
||||
|
||||
// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
|
||||
if ip != nil {
|
||||
f := []string{}
|
||||
f = append(f, fields[:4]...)
|
||||
f = append(f, " ") // Empty "opt" field for ip6tables
|
||||
f = append(f, fields[4:]...)
|
||||
fields = f
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust "source" and "destination" to include netmask, to match regular
|
||||
// List output
|
||||
fields[7] = appendSubnet(fields[7])
|
||||
fields[8] = appendSubnet(fields[8])
|
||||
|
||||
// Combine "options" fields 9... into a single space-delimited field.
|
||||
options := fields[9:]
|
||||
fields = fields[:9]
|
||||
fields = append(fields, strings.Join(options, " "))
|
||||
rows = append(rows, fields)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// ParseStat parses a single statistic row into a Stat struct. The input should
|
||||
// be a string slice that is returned from calling the Stat method.
|
||||
func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
|
||||
// For forward-compatibility, expect at least 10 fields in the stat
|
||||
if len(stat) < 10 {
|
||||
return parsed, fmt.Errorf("stat contained fewer fields than expected")
|
||||
}
|
||||
|
||||
// Convert the fields that are not plain strings
|
||||
parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
|
||||
if err != nil {
|
||||
return parsed, fmt.Errorf(err.Error(), "could not parse packets")
|
||||
}
|
||||
parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
|
||||
if err != nil {
|
||||
return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
|
||||
}
|
||||
_, parsed.Source, err = net.ParseCIDR(stat[7])
|
||||
if err != nil {
|
||||
return parsed, fmt.Errorf(err.Error(), "could not parse source")
|
||||
}
|
||||
_, parsed.Destination, err = net.ParseCIDR(stat[8])
|
||||
if err != nil {
|
||||
return parsed, fmt.Errorf(err.Error(), "could not parse destination")
|
||||
}
|
||||
|
||||
// Put the fields that are strings
|
||||
parsed.Target = stat[2]
|
||||
parsed.Protocol = stat[3]
|
||||
parsed.Opt = stat[4]
|
||||
parsed.Input = stat[5]
|
||||
parsed.Output = stat[6]
|
||||
parsed.Options = stat[9]
|
||||
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// StructuredStats returns statistics as structured data which may be further
|
||||
// parsed and marshaled.
|
||||
func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
|
||||
rawStats, err := ipt.Stats(table, chain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
structStats := []Stat{}
|
||||
for _, rawStat := range rawStats {
|
||||
stat, err := ipt.ParseStat(rawStat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
structStats = append(structStats, stat)
|
||||
}
|
||||
|
||||
return structStats, nil
|
||||
}
|
||||
|
||||
func (ipt *IPTables) executeList(args []string) ([]string, error) {
|
||||
var stdout bytes.Buffer
|
||||
if err := ipt.runWithOutput(args, &stdout); err != nil {
|
||||
|
||||
@@ -77,6 +77,9 @@ const (
|
||||
// Name of the service lease object
|
||||
ServiceLease = "kube-vip.io/leaseName"
|
||||
|
||||
// Versioned kube-vip ownership metadata stored on Kubernetes election Leases
|
||||
LeaseVIPs = "kube-vip.io/lease-vips"
|
||||
|
||||
// Forces kube-vip to use per service election for this particular service
|
||||
ForcePerServiceElection = "kube-vip.io/forcePerServiceElection"
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package kubevip
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -260,12 +259,12 @@ func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, s
|
||||
}
|
||||
|
||||
if ipv4 != "" {
|
||||
if net.ParseIP(ipv4) == nil {
|
||||
if !utils.IsIPv4(ipv4) {
|
||||
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", ipv4)
|
||||
}
|
||||
}
|
||||
if ipv6 != "" {
|
||||
if net.ParseIP(ipv6) == nil {
|
||||
if !utils.IsIPv6(ipv6) {
|
||||
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", ipv6)
|
||||
}
|
||||
}
|
||||
|
||||
38
pkg/kubevip/config_bgp_family_test.go
Normal file
38
pkg/kubevip/config_bgp_family_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package kubevip
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
api "github.com/osrg/gobgp/v4/api"
|
||||
)
|
||||
|
||||
func TestFindMpbgpAddressesRejectsFixedAddressFamilyMismatches(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
peer BGPPeer
|
||||
}{
|
||||
{
|
||||
name: "IPv6 value in IPv4 field",
|
||||
peer: BGPPeer{
|
||||
MpbgpNexthop: "fixed",
|
||||
MpbgpIPv4: "2001:db8::20",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IPv4 value in IPv6 field",
|
||||
peer: BGPPeer{
|
||||
MpbgpNexthop: "fixed",
|
||||
MpbgpIPv6: "192.0.2.20",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, _, err := tt.peer.FindMpbgpAddresses(&api.Peer{Transport: &api.Transport{}}, &BGPConfig{})
|
||||
if err == nil {
|
||||
t.Fatal("FindMpbgpAddresses() error = nil, want address-family error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -125,6 +125,13 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
|
||||
},
|
||||
},
|
||||
}
|
||||
if !role {
|
||||
newManifest.Rules = append(newManifest.Rules, applyRbacV1.PolicyRuleApplyConfiguration{
|
||||
APIGroups: []string{"networking.k8s.io"},
|
||||
Resources: []string{"servicecidrs"},
|
||||
Verbs: []string{"list", "get", "watch"},
|
||||
})
|
||||
}
|
||||
return newManifest
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,38 @@ package kubevip
|
||||
|
||||
import (
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
|
||||
)
|
||||
|
||||
func TestGenerateRoleServiceCIDRAccess(t *testing.T) {
|
||||
clusterRole := GenerateRole(&Config{}, false)
|
||||
if !hasServiceCIDRRule(clusterRole) {
|
||||
t.Fatal("generated ClusterRole is missing ServiceCIDR access")
|
||||
}
|
||||
|
||||
role := GenerateRole(&Config{ServiceNamespace: "kube-vip"}, true)
|
||||
if hasServiceCIDRRule(role) {
|
||||
t.Fatal("generated namespaced Role contains ineffective ServiceCIDR access")
|
||||
}
|
||||
}
|
||||
|
||||
func hasServiceCIDRRule(role *applyRbacV1.RoleApplyConfiguration) bool {
|
||||
for _, rule := range role.Rules {
|
||||
if slices.Contains(rule.APIGroups, "networking.k8s.io") &&
|
||||
slices.Contains(rule.Resources, "servicecidrs") &&
|
||||
slices.Contains(rule.Verbs, "get") &&
|
||||
slices.Contains(rule.Verbs, "list") &&
|
||||
slices.Contains(rule.Verbs, "watch") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestParseEnvironment(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
|
||||
134
pkg/kubevip/lease_annotations.go
Normal file
134
pkg/kubevip/lease_annotations.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package kubevip
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const LeaseVIPsVersion = "v1"
|
||||
|
||||
// LeaseVIPKind distinguishes literal addresses from names that resolve to one.
|
||||
type LeaseVIPKind string
|
||||
|
||||
const (
|
||||
LeaseVIPKindAddress LeaseVIPKind = "address"
|
||||
LeaseVIPKindName LeaseVIPKind = "name"
|
||||
)
|
||||
|
||||
type LeaseVIPsValue struct {
|
||||
Version string `json:"version"`
|
||||
InstanceName string `json:"instance_name"`
|
||||
IFAProto int `json:"ifa_proto"`
|
||||
VIPs []LeaseVIP `json:"vips"`
|
||||
}
|
||||
|
||||
type LeaseVIP struct {
|
||||
Index int `json:"index"`
|
||||
Value string `json:"value"`
|
||||
Kind LeaseVIPKind `json:"kind"`
|
||||
}
|
||||
|
||||
func WithLeaseVIPs(annotations map[string]string, instanceName string, ifaProto int, vips []string) (map[string]string, error) {
|
||||
result := make(map[string]string, len(annotations)+1)
|
||||
for key, value := range annotations {
|
||||
result[key] = value
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(LeaseVIPsValue{
|
||||
Version: LeaseVIPsVersion,
|
||||
InstanceName: instanceName,
|
||||
IFAProto: ifaProto,
|
||||
VIPs: normalizeLeaseVIPs(vips),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode %s annotation: %w", LeaseVIPs, err)
|
||||
}
|
||||
result[LeaseVIPs] = string(encoded)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ParseLeaseVIPs(value string) (LeaseVIPsValue, error) {
|
||||
var parsed LeaseVIPsValue
|
||||
if err := json.Unmarshal([]byte(value), &parsed); err != nil {
|
||||
return LeaseVIPsValue{}, fmt.Errorf("decode %s annotation: %w", LeaseVIPs, err)
|
||||
}
|
||||
if parsed.Version != LeaseVIPsVersion {
|
||||
return LeaseVIPsValue{}, fmt.Errorf("unsupported %s annotation version %q", LeaseVIPs, parsed.Version)
|
||||
}
|
||||
for index, vip := range parsed.VIPs {
|
||||
if vip.Index != index {
|
||||
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP index %d at position %d", LeaseVIPs, vip.Index, index)
|
||||
}
|
||||
switch vip.Kind {
|
||||
case LeaseVIPKindAddress, LeaseVIPKindName:
|
||||
default:
|
||||
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP kind %q at index %d", LeaseVIPs, vip.Kind, vip.Index)
|
||||
}
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func normalizeLeaseVIPs(values []string) []LeaseVIP {
|
||||
unique := make(map[string]struct{}, len(values))
|
||||
addresses := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for candidate := range strings.SplitSeq(value, ",") {
|
||||
candidate = strings.TrimSpace(candidate)
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := unique[candidate]; exists {
|
||||
continue
|
||||
}
|
||||
unique[candidate] = struct{}{}
|
||||
addresses = append(addresses, candidate)
|
||||
}
|
||||
}
|
||||
// Sorting keeps the annotation byte-identical however callers happen to order VIPs.
|
||||
slices.SortFunc(addresses, compareLeaseVIPs)
|
||||
|
||||
result := make([]LeaseVIP, 0, len(addresses))
|
||||
for _, address := range addresses {
|
||||
kind := LeaseVIPKindName
|
||||
if _, isAddress := leaseVIPAddress(address); isAddress {
|
||||
kind = LeaseVIPKindAddress
|
||||
}
|
||||
result = append(result, LeaseVIP{Index: len(result), Value: address, Kind: kind})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// compareLeaseVIPs orders addresses numerically and ahead of names, which keeps VIPs like
|
||||
// 10.0.0.2 and 10.0.0.10 in the order an operator expects. Values that are not addresses,
|
||||
// such as DNS records, are kept and ordered lexically.
|
||||
func compareLeaseVIPs(a, b string) int {
|
||||
addressA, isAddressA := leaseVIPAddress(a)
|
||||
addressB, isAddressB := leaseVIPAddress(b)
|
||||
switch {
|
||||
case isAddressA && isAddressB:
|
||||
if order := addressA.Compare(addressB); order != 0 {
|
||||
return order
|
||||
}
|
||||
// Distinct spellings of one address still need a stable order.
|
||||
return strings.Compare(a, b)
|
||||
case isAddressA:
|
||||
return -1
|
||||
case isAddressB:
|
||||
return 1
|
||||
default:
|
||||
return strings.Compare(a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func leaseVIPAddress(value string) (netip.Addr, bool) {
|
||||
if address, err := netip.ParseAddr(value); err == nil {
|
||||
return address.Unmap(), true
|
||||
}
|
||||
if prefix, err := netip.ParsePrefix(value); err == nil {
|
||||
return prefix.Addr().Unmap(), true
|
||||
}
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
94
pkg/kubevip/lease_annotations_test.go
Normal file
94
pkg/kubevip/lease_annotations_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package kubevip
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWithLeaseVIPsEncodesVersionedInstanceOwnership(t *testing.T) {
|
||||
base := map[string]string{"example.test/preserved": "true", LeaseVIPs: "stale"}
|
||||
annotations, err := WithLeaseVIPs(base, "release_a", 248, []string{
|
||||
"2001:db8::10/128", "192.0.2.10", "192.0.2.10/32", "api.example.test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WithLeaseVIPs() error = %v", err)
|
||||
}
|
||||
if annotations["example.test/preserved"] != "true" {
|
||||
t.Fatal("WithLeaseVIPs() dropped an existing annotation")
|
||||
}
|
||||
if base[LeaseVIPs] != "stale" {
|
||||
t.Fatal("WithLeaseVIPs() mutated the input annotations")
|
||||
}
|
||||
|
||||
value, err := ParseLeaseVIPs(annotations[LeaseVIPs])
|
||||
if err != nil {
|
||||
t.Fatalf("ParseLeaseVIPs() error = %v", err)
|
||||
}
|
||||
if value.Version != LeaseVIPsVersion || value.InstanceName != "release_a" || value.IFAProto != 248 {
|
||||
t.Fatalf("Lease VIP metadata = %+v", value)
|
||||
}
|
||||
// Values are stored verbatim so DNS records survive alongside addresses.
|
||||
want := []LeaseVIP{
|
||||
{Index: 0, Value: "192.0.2.10", Kind: LeaseVIPKindAddress},
|
||||
{Index: 1, Value: "192.0.2.10/32", Kind: LeaseVIPKindAddress},
|
||||
{Index: 2, Value: "2001:db8::10/128", Kind: LeaseVIPKindAddress},
|
||||
{Index: 3, Value: "api.example.test", Kind: LeaseVIPKindName},
|
||||
}
|
||||
if !slices.Equal(value.VIPs, want) {
|
||||
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The annotation is rewritten whenever a node starts campaigning, so the encoding
|
||||
// has to be stable even when callers collect the same VIPs in a different order.
|
||||
func TestWithLeaseVIPsIsIndependentOfInputOrder(t *testing.T) {
|
||||
first, err := WithLeaseVIPs(nil, "release_a", 248, []string{
|
||||
"2001:db8::10", "192.0.2.10", "10.0.0.2", "10.0.0.10",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WithLeaseVIPs() error = %v", err)
|
||||
}
|
||||
second, err := WithLeaseVIPs(nil, "release_a", 248, []string{
|
||||
"10.0.0.10", "192.0.2.10", "2001:db8::10", "10.0.0.2",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WithLeaseVIPs() error = %v", err)
|
||||
}
|
||||
if first[LeaseVIPs] != second[LeaseVIPs] {
|
||||
t.Fatalf("annotation changed with input order:\n%s\n%s", first[LeaseVIPs], second[LeaseVIPs])
|
||||
}
|
||||
|
||||
value, err := ParseLeaseVIPs(first[LeaseVIPs])
|
||||
if err != nil {
|
||||
t.Fatalf("ParseLeaseVIPs() error = %v", err)
|
||||
}
|
||||
want := []string{"10.0.0.2", "10.0.0.10", "192.0.2.10", "2001:db8::10"}
|
||||
if len(value.VIPs) != len(want) {
|
||||
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
|
||||
}
|
||||
for index, address := range want {
|
||||
if value.VIPs[index] != (LeaseVIP{Index: index, Value: address, Kind: LeaseVIPKindAddress}) {
|
||||
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLeaseVIPsRejectsUnknownVersion(t *testing.T) {
|
||||
if _, err := ParseLeaseVIPs(`{"version":"v2","instance_name":"release_a","ifa_proto":248,"vips":[]}`); err == nil {
|
||||
t.Fatal("ParseLeaseVIPs() accepted an unknown version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLeaseVIPsRejectsUnknownKind(t *testing.T) {
|
||||
if _, err := ParseLeaseVIPs(
|
||||
`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":0,"value":"192.0.2.10","kind":"cidr"}]}`,
|
||||
); err == nil {
|
||||
t.Fatal("ParseLeaseVIPs() accepted an unknown VIP kind")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLeaseVIPsRejectsOutOfOrderIndexes(t *testing.T) {
|
||||
if _, err := ParseLeaseVIPs(`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":1,"value":"192.0.2.10"}]}`); err == nil {
|
||||
t.Fatal("ParseLeaseVIPs() accepted an out-of-order VIP index")
|
||||
}
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
|
||||
// Fatal error at this point as IPVS is probably not working
|
||||
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
|
||||
log.Error("IPVS service", "err", err)
|
||||
return utils.NewPanicError(fmt.Sprintf("unable to create an IPVS service - %s", err))
|
||||
return utils.WrapPanicError(err, "unable to create an IPVS service")
|
||||
|
||||
}
|
||||
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
|
||||
|
||||
@@ -2,7 +2,6 @@ package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -33,7 +32,6 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/upnp"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
@@ -58,10 +56,6 @@ type Manager struct {
|
||||
|
||||
svcProcessor *services.Processor
|
||||
|
||||
// This is a prometheus counter used to count the number of events received
|
||||
// from the service watcher
|
||||
countServiceWatchEvent *prometheus.CounterVec
|
||||
|
||||
// This mutex is to protect calls from various goroutines
|
||||
mutex sync.Mutex
|
||||
|
||||
@@ -260,16 +254,10 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
|
||||
intfMgr, arpMgr, nodeLabelManager, electionMgr, leaseMgr, routeMgr)
|
||||
|
||||
return &Manager{
|
||||
clientSet: clientset,
|
||||
rwClientSet: rwClientSet,
|
||||
configMap: configMap,
|
||||
config: config,
|
||||
countServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "kube_vip",
|
||||
Subsystem: "manager",
|
||||
Name: "all_services_events",
|
||||
Help: "Count all events fired by the service watcher categorised by event type",
|
||||
}, []string{"type"}),
|
||||
clientSet: clientset,
|
||||
rwClientSet: rwClientSet,
|
||||
configMap: configMap,
|
||||
config: config,
|
||||
signalChan: signalChan,
|
||||
svcProcessor: svcProcessor,
|
||||
intfMgr: intfMgr,
|
||||
@@ -414,7 +402,7 @@ func (sm *Manager) startMode(ctx context.Context) error {
|
||||
return nil
|
||||
default:
|
||||
if err = w.StartServices(modeCtx); err != nil {
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
if utils.IsPanicError(err) {
|
||||
sm.Kill()
|
||||
return fmt.Errorf("failed to reconcile services, non-recoverable error: %w", err)
|
||||
} else {
|
||||
|
||||
@@ -11,10 +11,9 @@ import (
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -42,6 +41,9 @@ func annotationsWatcher(ctx context.Context, clientSet,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(nodeList.Items) == 0 {
|
||||
return fmt.Errorf("no node found with hostname %q", config.NodeName)
|
||||
}
|
||||
|
||||
// We'll assume there's only one node with the hostname annotation. If that's not true,
|
||||
// there's probably bigger problems
|
||||
@@ -108,22 +110,15 @@ func annotationsWatcher(ctx context.Context, clientSet,
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes Nodes")
|
||||
|
||||
// This round trip allows us to handle unstructured status
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error(status.String())
|
||||
log.Error("annotations watcher failed", "err", utils.WatchError(event.Object))
|
||||
default:
|
||||
}
|
||||
}
|
||||
log.Info("[annotations] exiting annotations watcher")
|
||||
return nil
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return utils.NewPanicError("annotations watcher channel closed unexpectedly")
|
||||
}
|
||||
|
||||
// parseNodeAnnotations parses the annotations on the node and updates the configuration
|
||||
|
||||
21
pkg/manager/watch_annotations_test.go
Normal file
21
pkg/manager/watch_annotations_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestAnnotationsWatcherHandlesEmptyNodeList(t *testing.T) {
|
||||
client := fake.NewSimpleClientset()
|
||||
config := &kubevip.Config{
|
||||
NodeName: "node-a",
|
||||
Annotations: "kube-vip.io",
|
||||
}
|
||||
|
||||
if err := annotationsWatcher(context.Background(), client, client, config); err == nil {
|
||||
t.Fatal("annotationsWatcher() error = nil, want empty node-list error")
|
||||
}
|
||||
}
|
||||
@@ -113,6 +113,10 @@ func (b *BGP) StartServices(ctx context.Context) error {
|
||||
if err := b.PerServiceLeader(ctx, false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if b.config.EnableLeaderElection {
|
||||
log.Warn("leader election is enabled, only the elected leader will advertise service VIPs; unset enable_leader_election to keep advertising from every node (ECMP)",
|
||||
"lease", b.config.ServicesLeaseName)
|
||||
b.GlobalLeader(ctx, b.config.ServicesLeaseName)
|
||||
} else {
|
||||
if err := b.ServicesNoLeader(ctx); err != nil {
|
||||
return err
|
||||
@@ -121,10 +125,6 @@ func (b *BGP) StartServices(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BGP) ServicesGlobalLeader(ctx context.Context, id string) {
|
||||
// NOT IMPLEMENTED
|
||||
}
|
||||
|
||||
func (b *BGP) Name() string {
|
||||
return "BGP"
|
||||
}
|
||||
|
||||
@@ -43,7 +43,11 @@ func (t *Table) Configure(ctx context.Context, wg *sync.WaitGroup) error {
|
||||
if t.config.CleanRoutingTable {
|
||||
wg.Go(func() {
|
||||
// we assume that after 10s all services should be configured so we can delete redundant routes
|
||||
time.Sleep(time.Second * 10)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
if err := t.cleanRoutes(); err != nil {
|
||||
log.Error("error checking for old routes", "err", err)
|
||||
}
|
||||
|
||||
47
pkg/manager/worker/table_ctx_test.go
Normal file
47
pkg/manager/worker/table_ctx_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
)
|
||||
|
||||
func TestConfigureCleanRoutingTableStopsWithContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
table := &Table{Common: Common{
|
||||
config: &kubevip.Config{
|
||||
CleanRoutingTable: true,
|
||||
EnableControlPlane: true,
|
||||
Address: "10.254.254.254",
|
||||
RoutingTableID: 0x7fffffff,
|
||||
RoutingProtocol: 255,
|
||||
},
|
||||
mutex: &sync.Mutex{},
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
if err := table.Configure(ctx, &wg); err != nil {
|
||||
t.Fatalf("Configure returned an error: %v", err)
|
||||
}
|
||||
cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
// DEFECT: pkg/manager/worker/table.go:43-48 uses an unconditional
|
||||
// 10-second sleep for cleanRoutingTable and ignores the canceled RT
|
||||
// worker context, delaying shutdown/recovery.
|
||||
t.Fatal("cleanRoutingTable worker did not stop after context cancellation")
|
||||
}
|
||||
}
|
||||
@@ -102,12 +102,15 @@ func (w *WireGuard) ConfigureServices() {
|
||||
}
|
||||
|
||||
func (w *WireGuard) StartServices(ctx context.Context) error {
|
||||
// WireGuard has no multipath mechanism, so every service must be advertised by
|
||||
// exactly one node: leader election (per-service or global) is required.
|
||||
if w.config.EnableServicesElection {
|
||||
log.Info("beginning watching services, leaderelection will happen for every service")
|
||||
err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx, false)
|
||||
if err != nil {
|
||||
if err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx, false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
w.GlobalLeader(ctx, w.config.ServicesLeaseName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -147,12 +150,6 @@ func (w *WireGuard) OnStartedLeading(ctx context.Context) {
|
||||
w.endpointWatcherWg.Go(func() {
|
||||
w.watchKubernetesEndpoints(w.endpointWatcherCtx, tunnelConfig)
|
||||
})
|
||||
|
||||
if w.config.EnableServices && !w.config.EnableServicesElection {
|
||||
if err := w.svcProcessor.ServicesWatcher(ctx, services.NewCallback(w.svcProcessor.SyncServices, false), false); err != nil {
|
||||
log.Error("failed to start services watcher", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watchKubernetesEndpoints watches the kubernetes service EndpointSlices for changes
|
||||
@@ -185,8 +182,15 @@ func (w *WireGuard) watchKubernetesEndpoints(ctx context.Context, tunnelConfig *
|
||||
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified, watch.Deleted:
|
||||
if err := provider.LoadObject(event.Object, func() {}); err != nil {
|
||||
log.Error("failed to load endpoint object", "err", err)
|
||||
// A deleted slice has to be dropped so it stops counting toward the endpoint set.
|
||||
var err error
|
||||
if event.Type == watch.Deleted {
|
||||
err = provider.DeleteObject(event.Object)
|
||||
} else {
|
||||
err = provider.LoadObject(event.Object, func() {})
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("failed to update endpoint object", "eventType", event.Type, "err", err)
|
||||
continue
|
||||
}
|
||||
endpoints, _ := provider.GetAllEndpoints()
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package metrics
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var (
|
||||
// Service / VIP Lifecycle
|
||||
@@ -68,19 +72,25 @@ var (
|
||||
)
|
||||
)
|
||||
|
||||
var registerOnce sync.Once
|
||||
|
||||
// RegisterPrometheusMetrics registers all kube-vip metrics with the default
|
||||
// Prometheus registry.
|
||||
func RegisterPrometheusMetrics() {
|
||||
// Register all metrics with Prometheus
|
||||
prometheus.MustRegister(
|
||||
ActiveServices,
|
||||
ServiceReconcileErrorsTotal,
|
||||
ServiceReconcileDuration,
|
||||
LeaderTransitionsTotal,
|
||||
IsLeader,
|
||||
ServiceElectionLoops,
|
||||
ServiceElectionAttemptsTotal,
|
||||
ServiceElectionErrorsTotal,
|
||||
BGPSessionInfoGauge,
|
||||
BuildInfo,
|
||||
CountServiceWatchEvent,
|
||||
)
|
||||
registerOnce.Do(func() {
|
||||
// Register all metrics with Prometheus
|
||||
prometheus.MustRegister(
|
||||
ActiveServices,
|
||||
ServiceReconcileErrorsTotal,
|
||||
ServiceReconcileDuration,
|
||||
LeaderTransitionsTotal,
|
||||
IsLeader,
|
||||
ServiceElectionLoops,
|
||||
ServiceElectionAttemptsTotal,
|
||||
ServiceElectionErrorsTotal,
|
||||
BGPSessionInfoGauge,
|
||||
BuildInfo,
|
||||
CountServiceWatchEvent,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
94
pkg/metrics/server.go
Normal file
94
pkg/metrics/server.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// shutdownTimeout bounds how long the server waits for in-flight requests to
|
||||
// finish once the context is cancelled.
|
||||
const shutdownTimeout = 5 * time.Second
|
||||
|
||||
// ServerConfig defines the Prometheus server configuration.
|
||||
type ServerConfig struct {
|
||||
// Addr sets the http server address used to expose the metric endpoint
|
||||
Addr string
|
||||
}
|
||||
|
||||
// Serve exposes the Prometheus metrics endpoint on the configured address.
|
||||
func Serve(ctx context.Context, config ServerConfig) error {
|
||||
ln, err := net.Listen("tcp", config.Addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listening on %q: %w", config.Addr, err)
|
||||
}
|
||||
|
||||
return serve(ctx, ln)
|
||||
}
|
||||
|
||||
// serve starts the metrics endpoint on the provided listener
|
||||
func serve(ctx context.Context, ln net.Listener) error {
|
||||
srv := &http.Server{
|
||||
Handler: newServeMux(),
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
defer wg.Wait()
|
||||
|
||||
serveErr := make(chan error, 1)
|
||||
wg.Go(func() {
|
||||
err := srv.Serve(ln)
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
err = nil
|
||||
}
|
||||
serveErr <- err
|
||||
})
|
||||
|
||||
log.Info("prometheus HTTP server started", "addr", ln.Addr().String())
|
||||
|
||||
select {
|
||||
case err := <-serveErr:
|
||||
if err != nil {
|
||||
return fmt.Errorf("serving prometheus metrics: %w", err)
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
}
|
||||
|
||||
// create prometheus shutdown context (independent of other contexts)
|
||||
ctxShutDown, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := srv.Shutdown(ctxShutDown); err != nil {
|
||||
return fmt.Errorf("shutting down prometheus HTTP server: %w", err)
|
||||
}
|
||||
|
||||
log.Info("prometheus HTTP server stopped")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newServeMux() *http.ServeMux {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`<html>
|
||||
<head><title>kube-vip</title></head>
|
||||
<body>
|
||||
<h1>kube-vip Metrics</h1>
|
||||
<p><a href="/metrics">Metrics</a></p>
|
||||
</body>
|
||||
</html>`))
|
||||
})
|
||||
|
||||
return mux
|
||||
}
|
||||
213
pkg/metrics/server_test.go
Normal file
213
pkg/metrics/server_test.go
Normal file
@@ -0,0 +1,213 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
func TestServeExposesKubeVipMetrics(t *testing.T) {
|
||||
RegisterPrometheusMetrics()
|
||||
version, build, node := "v1.2.3", "test-build", "node-1"
|
||||
BuildInfo.WithLabelValues(version, build, node)
|
||||
|
||||
base, stop := startServer(t, newTestListener(t))
|
||||
|
||||
body, code := get(t, base+"/metrics")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET /metrics status = %d, want %d", code, http.StatusOK)
|
||||
}
|
||||
|
||||
// Label names are exposed in alphabetical order.
|
||||
want := fmt.Sprintf("kube_vip_build_info{build=\"%s\",node=\"%s\",version=\"%s\"}", build, node, version)
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("GET /metrics body does not contain %s, got:\n%s", want, body)
|
||||
}
|
||||
|
||||
if err := stop(); err != nil {
|
||||
t.Errorf("serve returned an error on shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeRootPageLinksToMetrics(t *testing.T) {
|
||||
base, stop := startServer(t, newTestListener(t))
|
||||
|
||||
body, code := get(t, base+"/")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET / status = %d, want %d", code, http.StatusOK)
|
||||
}
|
||||
|
||||
if !strings.Contains(body, `href="/metrics"`) {
|
||||
t.Errorf("GET / body does not link to /metrics, got:\n%s", body)
|
||||
}
|
||||
|
||||
if err := stop(); err != nil {
|
||||
t.Errorf("serve returned an error on shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStopsOnContextCancellation(t *testing.T) {
|
||||
ln := newTestListener(t)
|
||||
addr := ln.Addr().String()
|
||||
|
||||
_, stop := startServer(t, ln)
|
||||
|
||||
// stop blocks until serve returns, and serve waits on its serving
|
||||
// goroutine, so a clean return means nothing was left running.
|
||||
if err := stop(); err != nil {
|
||||
t.Fatalf("serve returned an error on shutdown: %v", err)
|
||||
}
|
||||
|
||||
// Shutdown must have closed the listener, freeing the port.
|
||||
reopened, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
t.Fatalf("listener still bound to %s after shutdown: %v", addr, err)
|
||||
}
|
||||
_ = reopened.Close()
|
||||
}
|
||||
|
||||
func TestServeWithAlreadyCancelledContext(t *testing.T) {
|
||||
ln := newTestListener(t)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// Shutdown can win the race against the serving goroutine here. That is
|
||||
// safe: a server already told to shut down makes Serve return
|
||||
// ErrServerClosed straight away, so nothing blocks.
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- serve(ctx, ln)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("serve on an already cancelled context returned: %v", err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("serve hung on an already cancelled context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeReturnsErrorWhenAddressUnavailable(t *testing.T) {
|
||||
ln := newTestListener(t)
|
||||
defer ln.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// The port is already used in newTestListener, so Serve should return an error
|
||||
if err := Serve(ctx, ServerConfig{Addr: ln.Addr().String()}); err == nil {
|
||||
t.Fatal("Serve on an address already in use returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterPrometheusMetricsIsIdempotent(t *testing.T) {
|
||||
RegisterPrometheusMetrics()
|
||||
|
||||
// Registering a collector that is already registered is an error,
|
||||
// RegisterPrometheusMetrics should be guarded with sync.Once.
|
||||
mustNotPanic(t, "repeated RegisterPrometheusMetrics call", RegisterPrometheusMetrics)
|
||||
|
||||
// Confirm the collectors were really registered.
|
||||
err := prometheus.DefaultRegisterer.Register(ActiveServices)
|
||||
|
||||
var alreadyRegistered prometheus.AlreadyRegisteredError
|
||||
if !errors.As(err, &alreadyRegistered) {
|
||||
t.Fatalf("Register(ActiveServices) error = %v, want AlreadyRegisteredError", err)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestListener binds a loopback listener on an arbitrary free port.
|
||||
func newTestListener(t *testing.T) net.Listener {
|
||||
t.Helper()
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listening on a free loopback port: %v", err)
|
||||
}
|
||||
|
||||
return ln
|
||||
}
|
||||
|
||||
// startServer runs serve on ln and returns the base URL along with a stop
|
||||
// function that cancels the context and reports what serve returned.
|
||||
func startServer(t *testing.T, ln net.Listener) (string, func() error) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
serveErr := make(chan error, 1)
|
||||
go func() {
|
||||
serveErr <- serve(ctx, ln)
|
||||
}()
|
||||
|
||||
var (
|
||||
once sync.Once
|
||||
err error
|
||||
)
|
||||
stop := func() error {
|
||||
once.Do(func() {
|
||||
cancel()
|
||||
select {
|
||||
case err = <-serveErr:
|
||||
case <-time.After(10 * time.Second):
|
||||
err = errors.New("serve did not return after the context was cancelled")
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = stop()
|
||||
})
|
||||
|
||||
return "http://" + ln.Addr().String(), stop
|
||||
}
|
||||
|
||||
func get(t *testing.T, url string) (string, int) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("building request for %s: %v", url, err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("reading body of %s: %v", url, err)
|
||||
}
|
||||
|
||||
return string(body), resp.StatusCode
|
||||
}
|
||||
|
||||
// mustNotPanic reports a panic in fn as a test failure describing what
|
||||
// panicked, rather than letting it take down the test binary.
|
||||
func mustNotPanic(t *testing.T, what string, fn func()) {
|
||||
t.Helper()
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("%s panicked: %v", what, r)
|
||||
}
|
||||
}()
|
||||
|
||||
fn()
|
||||
}
|
||||
@@ -1,19 +1,19 @@
|
||||
package networkinterface
|
||||
|
||||
import (
|
||||
log "log/slog"
|
||||
"sync"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
lock sync.Mutex
|
||||
interfaces map[string]*Link
|
||||
}
|
||||
|
||||
type Link struct {
|
||||
Lock sync.Mutex
|
||||
Intf netlink.Link
|
||||
mu sync.Mutex
|
||||
intf netlink.Link
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
@@ -23,19 +23,37 @@ func NewManager() *Manager {
|
||||
}
|
||||
|
||||
func (m *Manager) Get(intf netlink.Link) *Link {
|
||||
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
|
||||
updated, err := netlink.LinkByName(l.Intf.Attrs().Name)
|
||||
if err != nil {
|
||||
log.Error("failed to get interface %q: %w", l.Intf.Attrs().Name, err)
|
||||
return nil
|
||||
}
|
||||
l.Intf = updated
|
||||
return l
|
||||
if intf == nil || intf.Attrs() == nil {
|
||||
return nil
|
||||
}
|
||||
result := &Link{
|
||||
Intf: intf,
|
||||
attrs := intf.Attrs()
|
||||
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
if link, ok := m.interfaces[attrs.Name]; ok {
|
||||
link.replace(intf)
|
||||
return link
|
||||
}
|
||||
|
||||
m.interfaces[intf.Attrs().Name] = result
|
||||
return result
|
||||
link := &Link{intf: intf}
|
||||
m.interfaces[attrs.Name] = link
|
||||
return link
|
||||
}
|
||||
|
||||
func (l *Link) WithInterface(run func(netlink.Link) error) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return run(l.intf)
|
||||
}
|
||||
|
||||
func (l *Link) replace(intf netlink.Link) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.intf = intf
|
||||
}
|
||||
|
||||
func (m *Manager) Len() int {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
return len(m.interfaces)
|
||||
}
|
||||
|
||||
65
pkg/networkinterface/networkinterface_instance_test.go
Normal file
65
pkg/networkinterface/networkinterface_instance_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package networkinterface_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/node/noop"
|
||||
"github.com/kube-vip/kube-vip/pkg/route"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestManagerReconstructsProductionInstanceConcurrently(t *testing.T) {
|
||||
config := &kubevip.Config{Interface: "lo", ServicesInterface: "lo", VIPSubnet: "32", DisableServiceUpdates: true}
|
||||
manager := networkinterface.NewManager()
|
||||
service := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default", UID: "service"},
|
||||
Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10"},
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
var ready sync.WaitGroup
|
||||
ready.Add(2)
|
||||
results := make(chan struct {
|
||||
instance *instance.Instance
|
||||
err error
|
||||
}, 2)
|
||||
for range 2 {
|
||||
go func() {
|
||||
ready.Done()
|
||||
<-start
|
||||
instanceConfig := *config
|
||||
created, err := instance.NewInstance(context.Background(), service.DeepCopy(), &instanceConfig, manager,
|
||||
arp.NewManager(&instanceConfig), route.NewManager(), noop.NewManager(), &sync.WaitGroup{})
|
||||
results <- struct {
|
||||
instance *instance.Instance
|
||||
err error
|
||||
}{created, err}
|
||||
}()
|
||||
}
|
||||
ready.Wait()
|
||||
close(start)
|
||||
for range 2 {
|
||||
select {
|
||||
case result := <-results:
|
||||
if result.err != nil {
|
||||
t.Fatalf("NewInstance() error = %v", result.err)
|
||||
}
|
||||
if len(result.instance.Clusters) != 1 {
|
||||
t.Fatalf("cluster count = %d, want 1", len(result.instance.Clusters))
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("timed out waiting for concurrent NewInstance calls")
|
||||
}
|
||||
}
|
||||
if got := manager.Len(); got != 1 {
|
||||
t.Fatalf("cached link count = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
74
pkg/networkinterface/networkinterface_test.go
Normal file
74
pkg/networkinterface/networkinterface_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package networkinterface
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
func TestManagerGetReplacesChangedInterfaceIndex(t *testing.T) {
|
||||
manager := NewManager()
|
||||
firstInterface := dummyLink("eth0", 1)
|
||||
first := manager.Get(firstInterface)
|
||||
|
||||
if got := manager.Get(dummyLink("eth0", 1)); got != first {
|
||||
t.Fatal("Get returned a new link for the same interface generation")
|
||||
}
|
||||
|
||||
secondInterface := dummyLink("eth0", 2)
|
||||
second := manager.Get(secondInterface)
|
||||
if second != first {
|
||||
t.Fatal("Get replaced the shared link after the interface index changed")
|
||||
}
|
||||
var current netlink.Link
|
||||
if err := first.WithInterface(func(intf netlink.Link) error {
|
||||
current = intf
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("WithInterface() error = %v", err)
|
||||
}
|
||||
if current != secondInterface {
|
||||
t.Fatal("Get did not retain the new link generation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerGetConcurrent(t *testing.T) {
|
||||
manager := NewManager()
|
||||
interfaces := []netlink.Link{dummyLink("eth0", 1), dummyLink("eth1", 2)}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan struct {
|
||||
index int
|
||||
link *Link
|
||||
}, 64)
|
||||
for index := range cap(results) {
|
||||
interfaceIndex := index % len(interfaces)
|
||||
wg.Go(func() {
|
||||
results <- struct {
|
||||
index int
|
||||
link *Link
|
||||
}{index: interfaceIndex, link: manager.Get(interfaces[interfaceIndex])}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
var cached [2]*Link
|
||||
for result := range results {
|
||||
if result.link == nil {
|
||||
t.Fatal("concurrent interface lookup returned nil")
|
||||
}
|
||||
if cached[result.index] == nil {
|
||||
cached[result.index] = result.link
|
||||
} else if result.link != cached[result.index] {
|
||||
t.Fatalf("interface %d produced multiple cached Link objects", result.index)
|
||||
}
|
||||
}
|
||||
if cached[0] == cached[1] {
|
||||
t.Fatal("different interfaces shared one cached Link object")
|
||||
}
|
||||
}
|
||||
|
||||
func dummyLink(name string, index int) netlink.Link {
|
||||
return &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name, Index: index}}
|
||||
}
|
||||
@@ -49,9 +49,6 @@ func (m *Manager) Add(object string, r route, precheck, update bool) error {
|
||||
itm, exists := m.tracker[key]
|
||||
|
||||
if !exists {
|
||||
m.tracker[key] = newItem(r)
|
||||
itm = m.tracker[key]
|
||||
|
||||
added, err := r.AddRoute(precheck)
|
||||
if err != nil {
|
||||
if update && errors.Is(err, syscall.EEXIST) && update {
|
||||
@@ -70,9 +67,13 @@ func (m *Manager) Add(object string, r route, precheck, update bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
itm = newItem(r)
|
||||
m.tracker[key] = itm
|
||||
|
||||
if added {
|
||||
log.Debug("[RT] added route", "path", key, "object", object)
|
||||
}
|
||||
m.tracker[key] = itm
|
||||
}
|
||||
|
||||
itm.objects[object] = true
|
||||
@@ -112,6 +113,8 @@ func (m *Manager) Delete(object string, r route) error {
|
||||
}
|
||||
|
||||
func (m *Manager) Clear() {
|
||||
m.mtx.Lock()
|
||||
defer m.mtx.Unlock()
|
||||
for _, itm := range m.tracker {
|
||||
if err := itm.route.DeleteRoute(); err != nil {
|
||||
log.Warn("[RT] failed to delete route", "err", err.Error())
|
||||
@@ -121,6 +124,8 @@ func (m *Manager) Clear() {
|
||||
}
|
||||
|
||||
func (m *Manager) Check(key string) bool {
|
||||
m.mtx.Lock()
|
||||
defer m.mtx.Unlock()
|
||||
_, exists := m.tracker[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
29
pkg/route/manager_atomic_test.go
Normal file
29
pkg/route/manager_atomic_test.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package route
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var errTransientRouteAdd = errors.New("transient route add failure")
|
||||
|
||||
func TestManagerRetriesRouteAfterInitialAddFailure(t *testing.T) {
|
||||
m := NewManager()
|
||||
r := &mockRoute{hash: "retry", addErr: errTransientRouteAdd}
|
||||
|
||||
if err := m.Add("service", r, false, true); !errors.Is(err, errTransientRouteAdd) {
|
||||
t.Fatalf("first add error = %v, want %v", err, errTransientRouteAdd)
|
||||
}
|
||||
|
||||
// A transient netlink failure (for example while the link is being recreated)
|
||||
// must not poison the in-memory tracker. The next reconciliation has to retry
|
||||
// the kernel operation.
|
||||
r.addErr = nil
|
||||
r.added = true
|
||||
if err := m.Add("service", r, false, true); err != nil {
|
||||
t.Fatalf("retry add failed: %v", err)
|
||||
}
|
||||
if r.addCalls != 2 {
|
||||
t.Fatalf("AddRoute called %d times, want 2", r.addCalls)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
package route
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -184,8 +186,50 @@ func Test_MultipleRoutesAddDel(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestAddFailureDoesNotTrackRoute(t *testing.T) {
|
||||
m := NewManager()
|
||||
r := &mockRoute{hash: "failed-route", addErr: errors.New("add failed")}
|
||||
|
||||
if err := m.Add("service", r, false, false); err == nil {
|
||||
t.Fatal("Add error = nil, want route failure")
|
||||
}
|
||||
if m.Check(r.RouteHash()) {
|
||||
t.Fatal("failed route was tracked")
|
||||
}
|
||||
|
||||
r.addErr = nil
|
||||
if err := m.Add("service", r, false, false); err != nil {
|
||||
t.Fatalf("retry Add error = %v", err)
|
||||
}
|
||||
if !m.Check(r.RouteHash()) {
|
||||
t.Fatal("successful retry was not tracked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearAndCheckAreSafeWithRouteUpdates(t *testing.T) {
|
||||
manager := NewManager()
|
||||
route := &mockRoute{hash: "concurrent-route", added: true}
|
||||
if err := manager.Add("service", route, false, false); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 20 {
|
||||
wg.Go(func() {
|
||||
manager.Check(route.RouteHash())
|
||||
})
|
||||
}
|
||||
wg.Go(manager.Clear)
|
||||
wg.Wait()
|
||||
|
||||
if manager.Check(route.RouteHash()) {
|
||||
t.Fatal("route remained tracked after concurrent Clear")
|
||||
}
|
||||
}
|
||||
|
||||
type mockRoute struct {
|
||||
added bool
|
||||
addCalls int
|
||||
addErr error
|
||||
updated bool
|
||||
updateErr error
|
||||
@@ -195,6 +239,7 @@ type mockRoute struct {
|
||||
}
|
||||
|
||||
func (mr *mockRoute) AddRoute(_ bool) (bool, error) {
|
||||
mr.addCalls++
|
||||
return mr.added, mr.addErr
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ type Context struct {
|
||||
IsWatched bool
|
||||
ConfiguredNetworks sync.Map
|
||||
EndpointsReady chan any
|
||||
mu sync.Mutex
|
||||
epReady sync.Once
|
||||
leaderElection sync.Once
|
||||
Signalled atomic.Bool
|
||||
@@ -51,6 +52,9 @@ func (ctx *Context) StartLeaderElectionOnce(f func()) {
|
||||
}
|
||||
|
||||
func (ctx *Context) SignalReadiness() {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
ctx.epReady.Do(func() {
|
||||
close(ctx.EndpointsReady)
|
||||
ctx.Signalled.Store(true)
|
||||
@@ -58,9 +62,50 @@ func (ctx *Context) SignalReadiness() {
|
||||
}
|
||||
|
||||
func (ctx *Context) ResetReadiness() {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
if ctx.Signalled.Load() {
|
||||
ctx.EndpointsReady = make(chan any)
|
||||
ctx.epReady = sync.Once{}
|
||||
ctx.Signalled.Store(false)
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) GetEndpointsReady() chan any {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
return ctx.EndpointsReady
|
||||
}
|
||||
|
||||
func (ctx *Context) SetLeaderCancel(cancel context.CancelFunc) {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
ctx.LeaderCancel = cancel
|
||||
}
|
||||
|
||||
func (ctx *Context) CallLeaderCancel() {
|
||||
ctx.mu.Lock()
|
||||
cancel := ctx.LeaderCancel
|
||||
ctx.mu.Unlock()
|
||||
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) SetWatched(watched bool) {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
ctx.IsWatched = watched
|
||||
}
|
||||
|
||||
func (ctx *Context) IsWatchedLocked() bool {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
return ctx.IsWatched
|
||||
}
|
||||
|
||||
56
pkg/servicecontext/servicecontext_race_test.go
Normal file
56
pkg/servicecontext/servicecontext_race_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package servicecontext
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadinessResetConcurrentWithSignal(t *testing.T) {
|
||||
ctx := New(context.Background())
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
ctx.SignalReadiness()
|
||||
ctx.ResetReadiness()
|
||||
}
|
||||
})
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
ready := ctx.GetEndpointsReady()
|
||||
select {
|
||||
case <-ready:
|
||||
default:
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestLeaderCancelConcurrentWithEndpointCleanup(t *testing.T) {
|
||||
ctx := New(context.Background())
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
ctx.SetLeaderCancel(func() {})
|
||||
}
|
||||
})
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
ctx.CallLeaderCancel()
|
||||
}
|
||||
})
|
||||
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
log "log/slog"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
@@ -130,6 +132,22 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
|
||||
return cidrs[0]
|
||||
}
|
||||
|
||||
// getSameFamilyCidrs returns every CIDR in the comma-separated list that has the
|
||||
// same IP family as ip.
|
||||
func getSameFamilyCidrs(sourceCidrs, ip string) []string {
|
||||
if sourceCidrs == "" {
|
||||
return nil
|
||||
}
|
||||
isV6 := utils.IsIPv6(ip)
|
||||
var matching []string
|
||||
for _, cidr := range strings.Split(sourceCidrs, ",") {
|
||||
if (isV6 && utils.IsIPv6CIDR(cidr)) || (!isV6 && utils.IsIPv4CIDR(cidr)) {
|
||||
matching = append(matching, cidr)
|
||||
}
|
||||
}
|
||||
return matching
|
||||
}
|
||||
|
||||
func checkCIDR(ip, cidr string) (string, error) {
|
||||
_, ipnetA, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
@@ -187,12 +205,12 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
|
||||
log.Warn("autodiscover CIDR", "err", discoverErr)
|
||||
}
|
||||
|
||||
if p.config.EgressPodCidr != "" {
|
||||
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
|
||||
} else {
|
||||
if discoverErr == nil {
|
||||
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
|
||||
}
|
||||
podCidrSource := p.config.EgressPodCidr
|
||||
if podCidrSource == "" && discoverErr == nil {
|
||||
podCidrSource = autoPodCIDR
|
||||
}
|
||||
if podCidrSource != "" {
|
||||
podCidr = getSameFamilyCidr(podCidrSource, podIP)
|
||||
}
|
||||
|
||||
if podCidr == "" {
|
||||
@@ -204,6 +222,14 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
|
||||
podCidr = defaultPodCIDR
|
||||
}
|
||||
|
||||
// Pod-to-pod traffic to any pod CIDR must not be SNAT'd. Node auto-discovery
|
||||
// yields one CIDR per node, so collect every same-family CIDR instead of only
|
||||
// the one containing the local pod IP.
|
||||
podCidrs := getSameFamilyCidrs(podCidrSource, podIP)
|
||||
if len(podCidrs) == 0 {
|
||||
podCidrs = []string{podCidr}
|
||||
}
|
||||
|
||||
if p.config.EgressServiceCidr != "" {
|
||||
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
|
||||
} else {
|
||||
@@ -243,10 +269,8 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
|
||||
var ignoreCIDRs []string
|
||||
// Create an array of CIDRs that we wont SNAT to.
|
||||
if noInternalTraffic == "" || strings.Clone(noInternalTraffic) == "false" || p.config.EnableInternalSNAT {
|
||||
ignoreCIDRs = append(ignoreCIDRs, []string{
|
||||
podCidr,
|
||||
serviceCidr,
|
||||
}...)
|
||||
ignoreCIDRs = append(ignoreCIDRs, podCidrs...)
|
||||
ignoreCIDRs = append(ignoreCIDRs, serviceCidr)
|
||||
}
|
||||
|
||||
// Add any specifically denied networks
|
||||
@@ -300,9 +324,11 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
|
||||
return fmt.Errorf("error creating mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, podCidr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
for _, cidr := range podCidrs {
|
||||
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, cidr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, serviceCidr)
|
||||
@@ -392,6 +418,49 @@ func (p *Processor) prepareEgressNftablesTable(serviceUID string, ipv6 bool) err
|
||||
|
||||
func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
|
||||
log.Debug("Trying to automatically discover Service and Pod CIDRs")
|
||||
serviceCIDR, podCIDR = p.discoverCIDRsFromAPI(ctx)
|
||||
if serviceCIDR != "" && podCIDR != "" {
|
||||
return serviceCIDR, podCIDR, nil
|
||||
}
|
||||
|
||||
// Fall back to the kube-controller-manager flags; e.g. CNIs doing their own
|
||||
// IPAM don't allocate Node PodCIDRs and clusters may lack the ServiceCIDR API.
|
||||
legacyServiceCIDR, legacyPodCIDR, legacyErr := p.discoverCIDRsFromControllerManager(ctx)
|
||||
if serviceCIDR == "" {
|
||||
serviceCIDR = legacyServiceCIDR
|
||||
}
|
||||
if podCIDR == "" {
|
||||
podCIDR = legacyPodCIDR
|
||||
}
|
||||
if serviceCIDR == "" || podCIDR == "" {
|
||||
if legacyErr != nil {
|
||||
return serviceCIDR, podCIDR, legacyErr
|
||||
}
|
||||
return serviceCIDR, podCIDR, fmt.Errorf("unable to fully determine cluster CIDR configurations")
|
||||
}
|
||||
|
||||
return serviceCIDR, podCIDR, nil
|
||||
}
|
||||
|
||||
func (p *Processor) discoverCIDRsFromAPI(ctx context.Context) (serviceCIDR, podCIDR string) {
|
||||
serviceCIDRs, err := p.clientSet.NetworkingV1().ServiceCIDRs().List(ctx, metav1.ListOptions{})
|
||||
if err == nil {
|
||||
serviceCIDR = serviceCIDRsFromItems(serviceCIDRs.Items)
|
||||
} else {
|
||||
log.Debug("Unable to discover Service CIDRs from ServiceCIDR API", "err", err)
|
||||
}
|
||||
|
||||
nodes, err := p.clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
||||
if err == nil {
|
||||
podCIDR = podCIDRsFromNodes(nodes.Items)
|
||||
} else {
|
||||
log.Debug("Unable to discover CNI Pod CIDRs from Node API", "err", err)
|
||||
}
|
||||
|
||||
return serviceCIDR, podCIDR
|
||||
}
|
||||
|
||||
func (p *Processor) discoverCIDRsFromControllerManager(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
|
||||
options := metav1.ListOptions{
|
||||
LabelSelector: "component=kube-controller-manager",
|
||||
}
|
||||
@@ -404,19 +473,45 @@ func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR
|
||||
}
|
||||
|
||||
pod := podList.Items[0]
|
||||
for flags := range pod.Spec.Containers[0].Command {
|
||||
if strings.Contains(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=") {
|
||||
podCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=", "")
|
||||
for _, flag := range pod.Spec.Containers[0].Command {
|
||||
if strings.Contains(flag, "--cluster-cidr=") {
|
||||
podCIDR = strings.ReplaceAll(flag, "--cluster-cidr=", "")
|
||||
}
|
||||
if strings.Contains(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=") {
|
||||
serviceCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=", "")
|
||||
if strings.Contains(flag, "--service-cluster-ip-range=") {
|
||||
serviceCIDR = strings.ReplaceAll(flag, "--service-cluster-ip-range=", "")
|
||||
}
|
||||
}
|
||||
if podCIDR == "" || serviceCIDR == "" {
|
||||
err = fmt.Errorf("unable to fully determine cluster CIDR configurations")
|
||||
}
|
||||
return serviceCIDR, podCIDR, nil
|
||||
}
|
||||
|
||||
return
|
||||
func serviceCIDRsFromItems(items []networkingv1.ServiceCIDR) string {
|
||||
var cidrs []string
|
||||
for _, item := range items {
|
||||
cidrs = appendUnique(cidrs, item.Spec.CIDRs...)
|
||||
}
|
||||
return strings.Join(cidrs, ",")
|
||||
}
|
||||
|
||||
func podCIDRsFromNodes(nodes []corev1.Node) string {
|
||||
var cidrs []string
|
||||
for _, node := range nodes {
|
||||
if len(node.Spec.PodCIDRs) > 0 {
|
||||
cidrs = appendUnique(cidrs, node.Spec.PodCIDRs...)
|
||||
} else if node.Spec.PodCIDR != "" {
|
||||
cidrs = appendUnique(cidrs, node.Spec.PodCIDR)
|
||||
}
|
||||
}
|
||||
return strings.Join(cidrs, ",")
|
||||
}
|
||||
|
||||
func appendUnique(values []string, additions ...string) []string {
|
||||
for _, addition := range additions {
|
||||
if addition == "" || slices.Contains(values, addition) {
|
||||
continue
|
||||
}
|
||||
values = append(values, addition)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func (p *Processor) updateEgressNftablesTableAnnotation(ctx context.Context, service *corev1.Service) error {
|
||||
|
||||
79
pkg/services/egress_cidr_test.go
Normal file
79
pkg/services/egress_cidr_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestServiceCIDRsFromItems(t *testing.T) {
|
||||
serviceCIDR := serviceCIDRsFromItems([]networkingv1.ServiceCIDR{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "kubernetes"},
|
||||
Spec: networkingv1.ServiceCIDRSpec{
|
||||
CIDRs: []string{"10.96.0.0/16", "fd00:10:96::/112"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if serviceCIDR != "10.96.0.0/16,fd00:10:96::/112" {
|
||||
t.Fatalf("serviceCIDR = %q", serviceCIDR)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodCIDRsFromNodes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
nodes []corev1.Node
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "dual stack across nodes",
|
||||
nodes: []corev1.Node{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "node-a"},
|
||||
Spec: corev1.NodeSpec{
|
||||
PodCIDRs: []string{"10.244.0.0/24", "fd00:10:244::/64"},
|
||||
},
|
||||
},
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "node-b"},
|
||||
Spec: corev1.NodeSpec{
|
||||
PodCIDRs: []string{"10.244.1.0/24", "fd00:10:244:1::/64"},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: "10.244.0.0/24,fd00:10:244::/64,10.244.1.0/24,fd00:10:244:1::/64",
|
||||
},
|
||||
{
|
||||
name: "legacy singular CIDR",
|
||||
nodes: []corev1.Node{
|
||||
{Spec: corev1.NodeSpec{PodCIDR: "10.244.0.0/24"}},
|
||||
},
|
||||
want: "10.244.0.0/24",
|
||||
},
|
||||
{
|
||||
name: "duplicate CIDRs",
|
||||
nodes: []corev1.Node{
|
||||
{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
|
||||
{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
|
||||
},
|
||||
want: "10.244.0.0/16",
|
||||
},
|
||||
{
|
||||
name: "missing CNI CIDRs",
|
||||
nodes: []corev1.Node{{}},
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := podCIDRsFromNodes(test.nodes); got != test.want {
|
||||
t.Fatalf("podCIDRsFromNodes() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,14 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
return fmt.Errorf("no existing lease found for service %q with UID %q", service.Name, service.UID)
|
||||
}
|
||||
|
||||
// A cancelled service context means this call belongs to a torn-down incarnation of
|
||||
// the service. Its replacement is built as Cancel -> Delete -> Add, so the lease
|
||||
// fetched above may already be the replacement's. Registering on it here would let
|
||||
// the cleanup goroutine below retire a lease that is still in use.
|
||||
if err := svcCtx.Ctx.Err(); err != nil {
|
||||
return fmt.Errorf("service context cancelled before election start: %w", err)
|
||||
}
|
||||
|
||||
isNew := svcLease.Add(objectName)
|
||||
|
||||
svcLease.Lock()
|
||||
@@ -91,7 +99,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
return fmt.Errorf("service context cancelled before election start: %w", svcCtx.Ctx.Err())
|
||||
case <-svcLease.Ctx.Done():
|
||||
return fmt.Errorf("lease context cancelled before election start: %w", svcLease.Ctx.Err())
|
||||
case <-svcCtx.EndpointsReady:
|
||||
case <-svcCtx.GetEndpointsReady():
|
||||
}
|
||||
|
||||
// this service is sharing lease with another service
|
||||
@@ -114,7 +122,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
// Block until service context is cancelled
|
||||
<-svcCtx.Ctx.Done()
|
||||
|
||||
if err := p.onStoppedLeading(svcLease, service); err != nil {
|
||||
if err := p.onStoppedLeading(svcCtx, svcLease, service); err != nil {
|
||||
log.Error("error on stopped leading", "error", err)
|
||||
}
|
||||
|
||||
@@ -127,7 +135,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
|
||||
|
||||
leaderCtx, leaderCancel := context.WithCancel(svcLease.Ctx)
|
||||
svcCtx.LeaderCancel = leaderCancel
|
||||
svcCtx.SetLeaderCancel(leaderCancel)
|
||||
|
||||
run := election.RunConfig{
|
||||
Config: p.config,
|
||||
@@ -151,7 +159,7 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
// we can do cleanup here
|
||||
svcLease.Elected.Store(false)
|
||||
log.Info("leadership lost", "service", service.Name, "uid", service.UID, "leader", p.config.NodeName)
|
||||
if err := p.onStoppedLeading(svcLease, service); err != nil {
|
||||
if err := p.onStoppedLeading(svcCtx, svcLease, service); err != nil {
|
||||
metrics.ServiceReconcileErrorsTotal.WithLabelValues(service.Namespace, service.Name, "delete_service").Inc()
|
||||
leaderCancel()
|
||||
}
|
||||
@@ -177,8 +185,6 @@ func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context,
|
||||
}
|
||||
|
||||
func (p *Processor) onStartedLeading(svcCtx *servicecontext.Context, service *v1.Service, wg *sync.WaitGroup) error {
|
||||
// Mark this service as active (as we've started leading)
|
||||
// we run this in background as it's blocking
|
||||
err := p.SyncServices(svcCtx, service, wg, true)
|
||||
if err != nil {
|
||||
log.Error("service sync", "uid", service.UID, "err", err)
|
||||
@@ -187,9 +193,18 @@ func (p *Processor) onStartedLeading(svcCtx *servicecontext.Context, service *v1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) onStoppedLeading(svcLease *lease.Lease, service *v1.Service) error {
|
||||
func (p *Processor) onStoppedLeading(svcCtx *servicecontext.Context, svcLease *lease.Lease, service *v1.Service) error {
|
||||
currentSvcCtx, err := p.getServiceContext(service.UID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if currentSvcCtx != nil && currentSvcCtx != svcCtx {
|
||||
log.Debug("skipping cleanup from superseded service context", "service", service.Name, "uid", service.UID)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Debug("deleting service due to lost leadership", "uid", service.UID)
|
||||
err := p.deleteService(svcLease.Ctx, service.UID)
|
||||
err = p.deleteService(svcLease.Ctx, service.UID)
|
||||
if err != nil {
|
||||
log.Error("service deletion", "err", err)
|
||||
return err
|
||||
|
||||
@@ -2,7 +2,6 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
"reflect"
|
||||
@@ -91,7 +90,8 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc *Callback, forcedOnly bool, wg *sync.WaitGroup) error {
|
||||
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc *Callback, forcedOnly bool,
|
||||
wg *sync.WaitGroup, cancelWatcher context.CancelCauseFunc) error {
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
|
||||
@@ -105,9 +105,9 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
return nil
|
||||
}
|
||||
|
||||
// We only care about LoadBalancer services
|
||||
// A tracked LoadBalancer must be torn down when its type changes.
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
return nil
|
||||
return p.deleteTrackedService(svc)
|
||||
}
|
||||
|
||||
// Check if we ignore this service
|
||||
@@ -142,16 +142,6 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
}
|
||||
|
||||
svcInstance := instance.FindServiceInstance(svc, p.ServiceInstances)
|
||||
var err error
|
||||
if svcInstance == nil {
|
||||
svcInstance, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
|
||||
if err != nil {
|
||||
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
|
||||
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)
|
||||
}
|
||||
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
|
||||
p.updateActiveServicesMetric()
|
||||
}
|
||||
|
||||
_, usesCommonLease := svc.Annotations[kubevip.ServiceLease]
|
||||
if usesCommonLease && svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
@@ -194,17 +184,16 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "delete_service").Inc()
|
||||
log.Error("(svc) unable to remove", "service", svc.UID)
|
||||
}
|
||||
// in theory this should never fail
|
||||
p.svcMap.Delete(svc.UID)
|
||||
// Drop this service from its lease now, so the replacement context
|
||||
// below is not parented to a lease the pending cleanup is about to
|
||||
// cancel. A lease shared with other services stays alive for them.
|
||||
// Retire the lease before the replacement context is built, so Add below
|
||||
// cannot hand back an instance the pending cleanup is about to cancel.
|
||||
// A lease shared with other services keeps their references and survives.
|
||||
ns, name := lease.ServiceName(svc)
|
||||
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
|
||||
p.leaseMgr.Delete(leaseID, lease.ServiceNamespacedName(svc), nil)
|
||||
// Reset the the svcCtx when it was garbage collected
|
||||
// As the next function will create a new context when nil
|
||||
svcCtx = nil
|
||||
svcInstance = nil
|
||||
p.updateActiveServicesMetric()
|
||||
}
|
||||
}
|
||||
@@ -216,19 +205,31 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
if svcCtx == nil {
|
||||
ns, name := lease.ServiceName(svc)
|
||||
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
|
||||
lease := p.leaseMgr.Add(ctx, leaseID)
|
||||
svcCtx = servicecontext.New(lease.Ctx)
|
||||
p.leaseMgr.Add(ctx, leaseID)
|
||||
// The service context is parented to the watcher, not to the lease: losing a
|
||||
// lease must not tear the service down, it has to let the election restart.
|
||||
svcCtx = servicecontext.New(ctx)
|
||||
p.svcMap.Store(svc.UID, svcCtx)
|
||||
}
|
||||
|
||||
if svcInstance == nil {
|
||||
svcInstance, err = instance.NewInstance(svcCtx.Ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
|
||||
if err != nil {
|
||||
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
|
||||
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)
|
||||
}
|
||||
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
|
||||
p.updateActiveServicesMetric()
|
||||
}
|
||||
|
||||
// this goroutine starts service handling function (with or without leaderelection)
|
||||
if !svcCtx.IsWatched {
|
||||
if !svcCtx.IsWatchedLocked() {
|
||||
wg.Go(func() {
|
||||
watchWg := sync.WaitGroup{}
|
||||
defer func() {
|
||||
// wait for the sub-goroutines and tag service as not watched
|
||||
watchWg.Wait()
|
||||
svcCtx.IsWatched = false
|
||||
svcCtx.SetWatched(false)
|
||||
}()
|
||||
|
||||
watchWg.Go(func() {
|
||||
@@ -238,7 +239,7 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
err = serviceFunc.Run(svcCtx, svc, wg)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
if utils.IsPanicError(err) {
|
||||
// cancel service context on panic error
|
||||
// TODO: should we quit kube-vip altogether here?
|
||||
svcCtx.Cancel()
|
||||
@@ -256,15 +257,18 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
|
||||
log.Error(err.Error())
|
||||
if err := p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider, cancelWatcher); err != nil {
|
||||
log.Error("endpoint watcher failed", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
if utils.IsPanicError(err) {
|
||||
cancelWatcher(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
// tag service as watched
|
||||
svcCtx.IsWatched = true
|
||||
svcCtx.SetWatched(true)
|
||||
}
|
||||
|
||||
if !p.config.EnableServicesElection {
|
||||
@@ -307,23 +311,16 @@ func (p *Processor) Delete(event watch.Event, forcedOnly bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.deleteTrackedService(svc)
|
||||
}
|
||||
|
||||
func (p *Processor) deleteTrackedService(svc *v1.Service) error {
|
||||
svcCtx, err := p.getServiceContext(svc.UID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("(svcs) unable to get context: %w", err)
|
||||
}
|
||||
|
||||
if svcCtx != nil {
|
||||
// We only care about LoadBalancer services
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
return nil
|
||||
}
|
||||
|
||||
// We can ignore this service
|
||||
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
|
||||
log.Info("(svcs) ignore annotation for kube-vip", "service name", svc.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// If no leader election is enabled, delete routes here
|
||||
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
|
||||
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
|
||||
@@ -347,14 +344,17 @@ func (p *Processor) Delete(event watch.Event, forcedOnly bool) error {
|
||||
// Drop the per-service election series so a recreated service starts clean.
|
||||
metrics.ServiceElectionLoops.DeleteLabelValues(svc.Namespace, svc.Name)
|
||||
p.updateActiveServicesMetric()
|
||||
}
|
||||
|
||||
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
|
||||
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) Stop() {
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
for _, instance := range p.ServiceInstances {
|
||||
for _, cluster := range instance.Clusters {
|
||||
cluster.Stop()
|
||||
|
||||
@@ -4,14 +4,70 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
func TestAddOrModifyStopsTrackedServiceWhenTypeChanges(t *testing.T) {
|
||||
for _, ignored := range []bool{false, true} {
|
||||
name := "normal"
|
||||
if ignored {
|
||||
name = "ignored"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
annotations := map[string]string{}
|
||||
if ignored {
|
||||
annotations[kubevip.LoadbalancerIgnore] = "true"
|
||||
}
|
||||
|
||||
uid := types.UID("service-uid")
|
||||
tracked := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "example",
|
||||
Namespace: "default",
|
||||
UID: uid,
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Type: v1.ServiceTypeLoadBalancer,
|
||||
LoadBalancerIP: "192.0.2.10",
|
||||
},
|
||||
}
|
||||
modified := tracked.DeepCopy()
|
||||
modified.Spec.Type = v1.ServiceTypeClusterIP
|
||||
modified.Annotations = annotations
|
||||
|
||||
p := &Processor{
|
||||
config: &kubevip.Config{},
|
||||
leaseMgr: lease.NewManager(),
|
||||
ServiceInstances: []*instance.Instance{{ServiceSnapshot: tracked}},
|
||||
}
|
||||
svcCtx := servicecontext.New(context.Background())
|
||||
p.svcMap.Store(uid, svcCtx)
|
||||
|
||||
if err := p.AddOrModify(context.Background(), watch.Event{Type: watch.Modified, Object: modified}, nil, false, nil, nil); err != nil {
|
||||
t.Fatalf("AddOrModify returned error: %v", err)
|
||||
}
|
||||
|
||||
if svcCtx.Ctx.Err() == nil {
|
||||
t.Fatal("tracked service context was not cancelled")
|
||||
}
|
||||
if _, ok := p.svcMap.Load(uid); ok {
|
||||
t.Fatal("tracked service context was not removed from svcMap")
|
||||
}
|
||||
if len(p.ServiceInstances) != 0 {
|
||||
t.Fatalf("tracked service instance count = %d, want 0", len(p.ServiceInstances))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropCancelledServiceContext is a regression test for the lease/svcMap desync that
|
||||
// permanently stops a LoadBalancer VIP from being advertised.
|
||||
//
|
||||
@@ -117,3 +173,37 @@ func TestDropCancelledServiceContextAllowsLeaseRecreation(t *testing.T) {
|
||||
t.Fatal("expected a new lease to be created once the cancelled service context was dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnStoppedLeadingDoesNotDeleteReplacementContext(t *testing.T) {
|
||||
p := &Processor{
|
||||
config: &kubevip.Config{},
|
||||
leaseMgr: lease.NewManager(),
|
||||
}
|
||||
|
||||
service := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "example",
|
||||
Namespace: "default",
|
||||
UID: types.UID("service-uid"),
|
||||
},
|
||||
}
|
||||
|
||||
oldCtx := servicecontext.New(context.Background())
|
||||
replacementCtx := servicecontext.New(context.Background())
|
||||
p.svcMap.Store(service.UID, replacementCtx)
|
||||
replacementInstance := &instance.Instance{ServiceSnapshot: service.DeepCopy()}
|
||||
p.ServiceInstances = []*instance.Instance{replacementInstance}
|
||||
|
||||
leaseNamespace, serviceLease := lease.ServiceName(service)
|
||||
svcLease := p.leaseMgr.Add(context.Background(), lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease))
|
||||
|
||||
if err := p.onStoppedLeading(oldCtx, svcLease, service); err != nil {
|
||||
t.Fatalf("onStoppedLeading returned an error: %v", err)
|
||||
}
|
||||
if got, err := p.getServiceContext(service.UID); err != nil || got != replacementCtx {
|
||||
t.Fatalf("replacement context was changed: got %v, err %v", got, err)
|
||||
}
|
||||
if len(p.ServiceInstances) != 1 || p.ServiceInstances[0] != replacementInstance {
|
||||
t.Fatal("replacement service instance was removed by superseded cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
33
pkg/services/processor_race_test.go
Normal file
33
pkg/services/processor_race_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
)
|
||||
|
||||
func TestWatchedFlagConcurrentWithWatcherTeardown(t *testing.T) {
|
||||
svcCtx := servicecontext.New(context.Background())
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
svcCtx.SetWatched(false)
|
||||
}
|
||||
})
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 1000 {
|
||||
if !svcCtx.IsWatchedLocked() {
|
||||
svcCtx.SetWatched(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/egress"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
@@ -66,7 +65,7 @@ func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, w
|
||||
select {
|
||||
case <-ctx.Ctx.Done():
|
||||
return nil
|
||||
case <-ctx.EndpointsReady:
|
||||
case <-ctx.GetEndpointsReady():
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +75,15 @@ func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, w
|
||||
|
||||
case ActionNone:
|
||||
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
|
||||
// Egress: when the service reaches ActionNone it means AddCalled is already true.
|
||||
// If ActiveEndpoint is now set (by the endpoint watcher) and the service has a
|
||||
// LB IP, the initial addService call may have missed the SNAT configuration because
|
||||
// ActiveEndpoint was not yet present. Re-run it here.
|
||||
if svc.Annotations[kubevip.Egress] == "true" && svc.Annotations[kubevip.ActiveEndpoint] != "" {
|
||||
if err := p.updateEgressConfiguration(ctx.Ctx, svc); err != nil {
|
||||
log.Warn("[service] egress reconfigure on ActionNone", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
|
||||
return nil
|
||||
@@ -219,19 +227,23 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
|
||||
if index == -1 {
|
||||
log.Error("unable to find proper VIPConfig for the DHCPv4")
|
||||
} else {
|
||||
for ip := range inst.DHCPv4Client.IPChannel() {
|
||||
log.Debug("IP changed", "ip", ip)
|
||||
inst.VIPConfigs[index].VIP = ip
|
||||
inst.DHCPInterfaceIPv4 = ip
|
||||
if !p.config.DisableServiceUpdates {
|
||||
if err := p.updateStatus(ctx, inst); err != nil {
|
||||
log.Warn("updating svc", "err", err)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debug("IPv4 update watcher stopping")
|
||||
return
|
||||
case ip := <-inst.DHCPv4Client.IPChannel():
|
||||
log.Debug("IP changed", "ip", ip)
|
||||
inst.VIPConfigs[index].VIP = ip
|
||||
inst.DHCPInterfaceIPv4 = ip
|
||||
if !p.config.DisableServiceUpdates {
|
||||
if err := p.updateStatus(ctx, inst); err != nil {
|
||||
log.Warn("updating svc", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Debug("IPv4 update channel closed, stopping")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
@@ -248,17 +260,22 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
|
||||
if index == -1 {
|
||||
log.Error("unable to find proper VIPConfig for the DHCPv6")
|
||||
} else {
|
||||
for ip := range inst.DHCPv4Client.IPChannel() {
|
||||
log.Debug("IP changed", "ip", ip)
|
||||
inst.VIPConfigs[index].VIP = ip
|
||||
inst.DHCPInterfaceIPv6 = ip
|
||||
if !p.config.DisableServiceUpdates {
|
||||
if err := p.updateStatus(ctx, inst); err != nil {
|
||||
log.Warn("updating svc", "err", err)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debug("IPv6 update watcher stopping")
|
||||
return
|
||||
case ip := <-inst.DHCPv6Client.IPChannel():
|
||||
log.Debug("IP changed", "ip", ip)
|
||||
inst.VIPConfigs[index].VIP = ip
|
||||
inst.DHCPInterfaceIPv6 = ip
|
||||
if !p.config.DisableServiceUpdates {
|
||||
if err := p.updateStatus(ctx, inst); err != nil {
|
||||
log.Warn("updating svc", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Debug("IPv6 update channel closed, stopping")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -367,17 +384,6 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var provider providers.Provider
|
||||
if p.config.EnableEndpoints {
|
||||
provider = providers.NewEndpoints()
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
err := provider.UpdateServiceAnnotation(ctx, svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
|
||||
if err != nil {
|
||||
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,7 +460,9 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
|
||||
endpoints.ClearBGPHostsByInstance(ctx, serviceInstance, p.bgpServer)
|
||||
}
|
||||
|
||||
if p.config.EnableRoutingTable && (p.config.EnableLeaderElection || p.config.EnableServicesElection) {
|
||||
// ClearRoutesByInstance is reference-counted per route, so calling it here is safe
|
||||
// even when the no-election path in Processor.Delete already cleared it.
|
||||
if p.config.EnableRoutingTable {
|
||||
if errs := endpoints.ClearRoutesByInstance(serviceInstance.ServiceSnapshot, serviceInstance, &p.ServiceInstances, p.routeMgr); len(errs) > 0 {
|
||||
for _, err := range errs {
|
||||
log.Error("unable to clear routes", "err", err)
|
||||
@@ -546,11 +554,24 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
|
||||
oldIPv6 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
newIPv6 := svc.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
|
||||
// Skip update if endpoints haven't changed
|
||||
// Skip update if endpoints haven't changed, without touching the API.
|
||||
if oldIPv4 == newIPv4 && oldIPv6 == newIPv6 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The svc snapshot may have been captured before the LB IP was assigned.
|
||||
// Refresh from the API so FetchServiceAddresses sees the current ingress.
|
||||
if current, err := p.clientSet.CoreV1().Services(svc.Namespace).Get(ctx, svc.Name, metav1.GetOptions{}); err == nil {
|
||||
// Preserve the caller-supplied annotations (ActiveEndpoint etc.) that triggered this call.
|
||||
for k, v := range svc.Annotations {
|
||||
if current.Annotations == nil {
|
||||
current.Annotations = make(map[string]string)
|
||||
}
|
||||
current.Annotations[k] = v
|
||||
}
|
||||
svc = current
|
||||
}
|
||||
|
||||
log.Info("[service] updating egress configuration",
|
||||
"service", svc.Name,
|
||||
"namespace", svc.Namespace,
|
||||
|
||||
@@ -1,14 +1,90 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
)
|
||||
|
||||
func TestConfigureServiceDoesNotOverwriteActiveEndpoint(t *testing.T) {
|
||||
const selectedEndpoint = "172.30.2.40"
|
||||
|
||||
staleService := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-service", Namespace: "default", UID: "test-uid",
|
||||
Annotations: map[string]string{kubevip.Egress: "true", kubevip.ActiveEndpoint: ""},
|
||||
},
|
||||
Spec: v1.ServiceSpec{LoadBalancerIP: "10.114.44.149"},
|
||||
}
|
||||
currentService := staleService.DeepCopy()
|
||||
currentService.Annotations[kubevip.ActiveEndpoint] = selectedEndpoint
|
||||
currentService.ResourceVersion = "2"
|
||||
|
||||
var mutex sync.Mutex
|
||||
updateRequests := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
switch request.Method {
|
||||
case http.MethodGet:
|
||||
if err := json.NewEncoder(writer).Encode(currentService); err != nil {
|
||||
t.Errorf("encode Service response: %v", err)
|
||||
}
|
||||
case http.MethodPut:
|
||||
updatedService := &v1.Service{}
|
||||
if err := json.NewDecoder(request.Body).Decode(updatedService); err != nil {
|
||||
http.Error(writer, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
updateRequests++
|
||||
currentService = updatedService
|
||||
if err := json.NewEncoder(writer).Encode(currentService); err != nil {
|
||||
t.Errorf("encode updated Service response: %v", err)
|
||||
}
|
||||
default:
|
||||
http.Error(writer, "unexpected request", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
clientSet, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL})
|
||||
if err != nil {
|
||||
t.Fatalf("create Kubernetes client: %v", err)
|
||||
}
|
||||
processor := &Processor{
|
||||
config: &kubevip.Config{
|
||||
DisableServiceUpdates: true,
|
||||
EnableServicesElection: true,
|
||||
},
|
||||
clientSet: clientSet,
|
||||
}
|
||||
serviceInstance := &instance.Instance{ServiceSnapshot: staleService}
|
||||
if err := processor.configureService(context.Background(), serviceInstance, staleService, &sync.WaitGroup{}); err != nil {
|
||||
t.Fatalf("configureService returned error: %v", err)
|
||||
}
|
||||
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
if updateRequests != 0 {
|
||||
t.Fatalf("configureService sent %d stale Service updates, want none", updateRequests)
|
||||
}
|
||||
if got := currentService.Annotations[kubevip.ActiveEndpoint]; got != selectedEndpoint {
|
||||
t.Fatalf("active endpoint = %q, want %q", got, selectedEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Test_upnpLeaseDurationForService tests whether the default lease duration is used, and whether the annotation
|
||||
// overrides it correctly.
|
||||
//
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
@@ -10,28 +11,31 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
|
||||
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service,
|
||||
provider providers.Provider, cancelWatcher context.CancelCauseFunc) error {
|
||||
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
|
||||
rw, err := provider.CreateRetryWatcher(svcCtx.Ctx, p.rwClientSet, service)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
|
||||
if svcCtx.Ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return utils.WrapPanicError(err, "[%s] error watching endpoints", provider.GetLabel())
|
||||
}
|
||||
|
||||
d, err := debouncer.New(rw.ResultChan(), p.config.DebounceTime)
|
||||
if err != nil {
|
||||
rw.Stop()
|
||||
return fmt.Errorf("failed to create debouncer for endpoints event: %w", err)
|
||||
return utils.WrapPanicError(err, "failed to create debouncer for endpoints event")
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
stopChan := make(chan any)
|
||||
|
||||
defer func() {
|
||||
@@ -46,6 +50,9 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
|
||||
if d != nil {
|
||||
if err := d.Start(svcCtx.Ctx); err != nil {
|
||||
log.Error("[endpoint watcher] debouncer, cancelling context", "error", err.Error())
|
||||
if svcCtx.Ctx.Err() == nil {
|
||||
cancelWatcher(utils.WrapPanicError(err, "[%s] endpoint debouncer failed", provider.GetLabel()))
|
||||
}
|
||||
svcCtx.Cancel()
|
||||
}
|
||||
}
|
||||
@@ -71,28 +78,31 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
|
||||
case watch.Added, watch.Modified:
|
||||
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id,
|
||||
case watch.Added, watch.Modified, watch.Deleted:
|
||||
if event.Type == watch.Deleted {
|
||||
log.Info("[endpoint watcher] endpoint object deleted", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
|
||||
}
|
||||
|
||||
restart, err := epProcessor.Reconcile(svcCtx, event, &lastKnownGoodEndpoint, service, id,
|
||||
p.StartServicesLeaderElection, &wg, p.clientSet, p.updateEgressConfiguration)
|
||||
if restart {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
|
||||
return fmt.Errorf("[%s] error while processing %s event: %w", provider.GetLabel(), event.Type, err)
|
||||
}
|
||||
|
||||
case watch.Deleted:
|
||||
if err := epProcessor.Delete(svcCtx.Ctx, service, id); err != nil {
|
||||
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
log.Info("[endpoint watcher] stopping watching - endpoint object deleted", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
|
||||
return nil
|
||||
case watch.Error:
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, _ := errObject.(*apierrors.StatusError)
|
||||
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
|
||||
if svcCtx.Ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
watchErr := utils.WatchError(event.Object)
|
||||
log.Error("watch error", "provider", provider.GetLabel(), "err", watchErr)
|
||||
return utils.WrapPanicError(watchErr, "[%s] endpoint watch failed", provider.GetLabel())
|
||||
}
|
||||
}
|
||||
if svcCtx.Ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
log.Info("[endpoint watcher] stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
|
||||
return nil //nolint:govet
|
||||
return utils.NewPanicError("[%s] endpoint watch channel closed unexpectedly for service %s/%s", provider.GetLabel(), service.Namespace, service.Name)
|
||||
}
|
||||
|
||||
@@ -2,13 +2,11 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/kube-vip/kube-vip/pkg/debouncer"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/metrics"
|
||||
@@ -16,7 +14,6 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
@@ -62,7 +59,7 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
|
||||
return fmt.Errorf("failed to create debouncer for endpoints event: %w", err)
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
var wg sync.WaitGroup
|
||||
defer func() {
|
||||
if d != nil {
|
||||
d.Stop()
|
||||
@@ -71,14 +68,14 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
watcherCtx, watcherCancel := context.WithCancel(ctx)
|
||||
defer watcherCancel()
|
||||
watcherCtx, cancelWatcher := context.WithCancelCause(ctx)
|
||||
defer cancelWatcher(nil)
|
||||
|
||||
wg.Go(func() {
|
||||
if d != nil {
|
||||
if err := d.Start(watcherCtx); err != nil {
|
||||
log.Error("(svcs) debouncer, cancelling context", "error", err.Error())
|
||||
watcherCancel()
|
||||
cancelWatcher(utils.WrapPanicError(err, "service debouncer failed"))
|
||||
}
|
||||
}
|
||||
<-watcherCtx.Done()
|
||||
@@ -96,47 +93,52 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
|
||||
}
|
||||
|
||||
// Used for tracking an active endpoint / pod
|
||||
EventLoop:
|
||||
for event := range ch {
|
||||
metrics.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
|
||||
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg); err != nil {
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
return fmt.Errorf("add/modify service error: %w", err)
|
||||
} else {
|
||||
log.Error("service watcher event failed", "type", event.Type, "error", err)
|
||||
}
|
||||
}
|
||||
case watch.Deleted:
|
||||
if err := p.Delete(event, forcedOnly); err != nil {
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
return fmt.Errorf("delete service error: %w", err)
|
||||
} else {
|
||||
log.Error("service watcher event failed", "type", event.Type, "error", err)
|
||||
}
|
||||
}
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes services")
|
||||
|
||||
// This round trip allows us to handle unstructured status
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error("services", "err", status)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info("global context done")
|
||||
case <-watcherCtx.Done():
|
||||
log.Info("WatcheConotext done")
|
||||
break EventLoop
|
||||
default:
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg, cancelWatcher); err != nil {
|
||||
if utils.IsPanicError(err) {
|
||||
return fmt.Errorf("add/modify service error: %w", err)
|
||||
}
|
||||
log.Error("service watcher event failed", "type", event.Type, "error", err)
|
||||
}
|
||||
case watch.Deleted:
|
||||
if err := p.Delete(event, forcedOnly); err != nil {
|
||||
if utils.IsPanicError(err) {
|
||||
return fmt.Errorf("delete service error: %w", err)
|
||||
}
|
||||
log.Error("service watcher event failed", "type", event.Type, "error", err)
|
||||
}
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes services")
|
||||
watchErr := utils.WatchError(event.Object)
|
||||
log.Error("services", "err", watchErr)
|
||||
return utils.WrapPanicError(watchErr, "service watch failed")
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if watcherErr := context.Cause(watcherCtx); watcherErr != nil {
|
||||
return watcherErr
|
||||
}
|
||||
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
|
||||
return nil
|
||||
return utils.NewPanicError("service watch channel closed unexpectedly")
|
||||
}
|
||||
|
||||
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
|
||||
|
||||
@@ -146,7 +146,7 @@ func getQdiscFromInterfaceByType(nicID int, nicName string, qType string) (uint3
|
||||
// get id through tc qdisc show dev fromNICName
|
||||
qs, err := netlink.QdiscList(&netlink.Ifb{LinkAttrs: netlink.LinkAttrs{Index: nicID}})
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to list qdisc for interface %s: %v", nicName, err)
|
||||
log.Error("failed to list qdisc", "interface", nicName, "err", err)
|
||||
return 0, err
|
||||
}
|
||||
for _, q := range qs {
|
||||
|
||||
@@ -1,15 +1,31 @@
|
||||
package utils
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type PanicError struct {
|
||||
cause string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *PanicError) Error() string {
|
||||
return fmt.Sprintf("%s - unrecoverable error", e.cause)
|
||||
}
|
||||
|
||||
func NewPanicError(cause string) error {
|
||||
return &PanicError{cause: cause}
|
||||
func (e *PanicError) Unwrap() error {
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func NewPanicError(format string, args ...any) error {
|
||||
return &PanicError{cause: fmt.Errorf(format, args...)}
|
||||
}
|
||||
|
||||
func WrapPanicError(err error, format string, args ...any) error {
|
||||
return &PanicError{cause: fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), err)}
|
||||
}
|
||||
|
||||
func IsPanicError(err error) bool {
|
||||
var panicErr *PanicError
|
||||
return errors.As(err, &panicErr)
|
||||
}
|
||||
|
||||
41
pkg/utils/panic_test.go
Normal file
41
pkg/utils/panic_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsPanicError(t *testing.T) {
|
||||
panicErr := NewPanicError("endpoint watch stopped")
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "direct", err: panicErr, want: true},
|
||||
{name: "wrapped", err: fmt.Errorf("watch failed: %w", panicErr), want: true},
|
||||
{name: "ordinary", err: errors.New("watch failed"), want: false},
|
||||
{name: "nil", err: nil, want: false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := IsPanicError(test.err); got != test.want {
|
||||
t.Fatalf("IsPanicError() = %t, want %t", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapPanicErrorPreservesCause(t *testing.T) {
|
||||
cause := errors.New("endpointslices is forbidden")
|
||||
err := WrapPanicError(cause, "endpoint watch failed")
|
||||
|
||||
if !IsPanicError(err) {
|
||||
t.Fatal("expected wrapped error to be classified as PanicError")
|
||||
}
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatal("expected wrapped PanicError to preserve its cause")
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,20 @@ import (
|
||||
"time"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
// WatchError converts a Kubernetes watch error object into a safe Go error.
|
||||
func WatchError(object runtime.Object) error {
|
||||
errObject := apierrors.FromObject(object)
|
||||
if statusErr, ok := errObject.(*apierrors.StatusError); ok {
|
||||
return statusErr
|
||||
}
|
||||
return fmt.Errorf("unknown watch error object of type %T: %v", object, object)
|
||||
}
|
||||
|
||||
// watchWithAuthRetry retries watchFn with exponential backoff on transient 403 Forbidden
|
||||
// and 401 Unauthorized errors. On joining control plane nodes with K8s 1.34+, the local
|
||||
// etcd may still be a learner when kube-vip starts, causing RBAC data to be unavailable.
|
||||
@@ -39,7 +49,14 @@ func WatchWithAuthRetry(ctx context.Context, watchFn func(context.Context) (watc
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, NewPanicError(fmt.Sprintf("watch failed after retries: %q (last: %v)", err.Error(), lastErr))
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if lastErr != nil {
|
||||
log.Error("watch auth retries exhausted", "err", lastErr)
|
||||
return nil, lastErr
|
||||
}
|
||||
return nil, WrapPanicError(err, "watch failed after retries (last: %v)", lastErr)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
37
pkg/utils/watcher_test.go
Normal file
37
pkg/utils/watcher_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestWatchErrorPreservesStatusError(t *testing.T) {
|
||||
status := &metav1.Status{
|
||||
Status: metav1.StatusFailure,
|
||||
Reason: metav1.StatusReasonForbidden,
|
||||
Message: "endpointslices is forbidden",
|
||||
}
|
||||
|
||||
err := WatchError(status)
|
||||
var statusErr *apierrors.StatusError
|
||||
if !errors.As(err, &statusErr) {
|
||||
t.Fatalf("expected a status error, got %T: %v", err, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), status.Message) {
|
||||
t.Fatalf("expected error to contain %q, got %q", status.Message, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchErrorHandlesUnexpectedObject(t *testing.T) {
|
||||
err := WatchError(&metav1.APIGroup{})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an unexpected watch object")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unknown watch error object") {
|
||||
t.Fatalf("expected unexpected-object context, got %q", err)
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,9 @@ import (
|
||||
const (
|
||||
defaultValidLft = 60
|
||||
iptablesComment = "%s kube-vip load balancer IP"
|
||||
// Linux reserves IFA_PROTO values 0-3 for unspecified and kernel-generated
|
||||
// addresses; user-space protocols start at 4. This is only a safety floor,
|
||||
firstUserAddressProtocol = 4
|
||||
|
||||
DefaultMaskIPv4 = 32
|
||||
DefaultMaskIPv6 = 128
|
||||
@@ -77,8 +80,7 @@ type Network interface {
|
||||
|
||||
// network - This allows network configuration
|
||||
type network struct {
|
||||
mu sync.Mutex
|
||||
|
||||
mu sync.RWMutex
|
||||
address *netlink.Addr
|
||||
link *networkinterface.Link
|
||||
ports []v1.ServicePort
|
||||
@@ -168,6 +170,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
if err != nil {
|
||||
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
|
||||
}
|
||||
markKubeVIPAddress(result.address, result.routingProtocol)
|
||||
|
||||
// set address as deprecated so it isn't used as source address according to RFC 3484
|
||||
result.address.PreferedLft = 0
|
||||
@@ -265,6 +268,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
|
||||
return networks, err
|
||||
}
|
||||
markKubeVIPAddress(result.address, result.routingProtocol)
|
||||
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
|
||||
result.address.ValidLft = defaultValidLft
|
||||
|
||||
@@ -310,7 +314,17 @@ func ListRoutesByDst(table int, dst *net.IPNet) ([]netlink.Route, error) {
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func (configurator *network) PrepareRoute() *netlink.Route {
|
||||
func (configurator *network) PrepareRoute() (route *netlink.Route) {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
route = configurator.prepareRoute(intf)
|
||||
return nil
|
||||
})
|
||||
return route
|
||||
}
|
||||
|
||||
func (configurator *network) prepareRoute(intf netlink.Link) *netlink.Route {
|
||||
routeScope := netlink.SCOPE_UNIVERSE
|
||||
if configurator.routingTableType == unix.RTN_LOCAL {
|
||||
routeScope = netlink.SCOPE_LINK
|
||||
@@ -318,7 +332,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
|
||||
route := &netlink.Route{
|
||||
Scope: routeScope,
|
||||
Dst: configurator.address.IPNet,
|
||||
LinkIndex: configurator.link.Intf.Attrs().Index,
|
||||
LinkIndex: intf.Attrs().Index,
|
||||
Table: configurator.routeTable,
|
||||
Type: configurator.routingTableType,
|
||||
Protocol: netlink.RouteProtocol(configurator.routingProtocol),
|
||||
@@ -339,32 +353,33 @@ func NetlinkHash(r *netlink.Route) string {
|
||||
}
|
||||
|
||||
// AddRoute - Add an IP address to a route table
|
||||
func (configurator *network) AddRoute(precheck bool) (bool, error) {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
route := configurator.PrepareRoute()
|
||||
|
||||
exists := false
|
||||
var err error
|
||||
if precheck {
|
||||
exists, err = configurator.routeExists(route)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to check route")
|
||||
func (configurator *network) AddRoute(precheck bool) (added bool, err error) {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
err = configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
route := configurator.prepareRoute(intf)
|
||||
exists := false
|
||||
if precheck {
|
||||
var existsErr error
|
||||
exists, existsErr = configurator.routeExists(intf, route)
|
||||
if existsErr != nil {
|
||||
return errors.Wrap(existsErr, "failed to check route")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if err := netlink.RouteAdd(route); err != nil {
|
||||
return false, errors.Wrap(err, "failed to add route")
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
if routeErr := netlink.RouteAdd(route); routeErr != nil {
|
||||
return errors.Wrap(routeErr, "failed to add route")
|
||||
}
|
||||
added = true
|
||||
return nil
|
||||
})
|
||||
return added, err
|
||||
}
|
||||
|
||||
func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
|
||||
routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
|
||||
func (configurator *network) routeExists(intf netlink.Link, route *netlink.Route) (bool, error) {
|
||||
routes, err := netlink.RouteList(intf, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to list routes")
|
||||
}
|
||||
@@ -385,16 +400,18 @@ func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
|
||||
// the same-prefix interface address is processed in the same netlink batch,
|
||||
// leaving the route in the kernel but never redistributed.
|
||||
func (configurator *network) ReplaceRoute() error {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
route := configurator.PrepareRoute()
|
||||
configurator.reassertToggle = !configurator.reassertToggle
|
||||
if configurator.reassertToggle {
|
||||
route.Realm = 1
|
||||
} else {
|
||||
route.Realm = 2
|
||||
}
|
||||
return netlink.RouteReplace(route)
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
return configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
route := configurator.prepareRoute(intf)
|
||||
configurator.reassertToggle = !configurator.reassertToggle
|
||||
if configurator.reassertToggle {
|
||||
route.Realm = 1
|
||||
} else {
|
||||
route.Realm = 2
|
||||
}
|
||||
return netlink.RouteReplace(route)
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteRoute - Delete an IP address from a route table
|
||||
@@ -405,6 +422,8 @@ func (configurator *network) DeleteRoute() error {
|
||||
|
||||
// GetRoutes - Get an IP addresses from a route table
|
||||
func (configurator *network) getRoutes() (*[]netlink.Route, error) {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
routes, err := ListRoutesByDst(configurator.routeTable, configurator.address.IPNet)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting routes: %w", err)
|
||||
@@ -443,12 +462,22 @@ func (configurator *network) shouldSkipDAD(override bool) bool {
|
||||
// precheck: if true, check if the IP already exists before adding
|
||||
// skipDAD: if true, set IFA_F_NODAD flag for IPv6 addresses to skip Duplicate Address Detection
|
||||
func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
var added bool
|
||||
err := configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
var addErr error
|
||||
added, addErr = configurator.addIP(intf, precheck, skipDAD, minLifetime...)
|
||||
return addErr
|
||||
})
|
||||
return added, err
|
||||
}
|
||||
|
||||
func (configurator *network) addIP(intf netlink.Link, precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
|
||||
var existing *netlink.Addr
|
||||
var err error
|
||||
if precheck {
|
||||
if existing, err = configurator.IsSet(); err != nil {
|
||||
if existing, err = configurator.isSet(intf); err != nil {
|
||||
return false, errors.Wrap(err, "could not check if address exists")
|
||||
}
|
||||
}
|
||||
@@ -469,14 +498,18 @@ func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...i
|
||||
// an address that we know should be ours (e.g., after DADFAILED recovery).
|
||||
// We also allow to globally configure NODAD in case user knows they are running in an
|
||||
// environment where multiple nodes may advertise the same VIP (e.g., ECMP routing).
|
||||
if configurator.shouldSkipDAD(skipDAD) && utils.IsIPv6(configurator.address.IP.String()) {
|
||||
configurator.address.Flags |= unix.IFA_F_NODAD
|
||||
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
|
||||
if utils.IsIPv6(configurator.address.IP.String()) {
|
||||
if configurator.shouldSkipDAD(skipDAD) {
|
||||
configurator.address.Flags |= unix.IFA_F_NODAD
|
||||
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
|
||||
} else {
|
||||
configurator.address.Flags &^= unix.IFA_F_NODAD
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("replacing IP", "address", configurator.address)
|
||||
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
|
||||
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", configurator.link.Intf.Attrs().Name))
|
||||
if err := netlink.AddrReplace(intf, configurator.address); err != nil {
|
||||
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", intf.Attrs().Name))
|
||||
}
|
||||
|
||||
if configurator.nftables {
|
||||
@@ -514,7 +547,7 @@ func (configurator *network) configureNFTables() error {
|
||||
"ports", configurator.ports, "service-name", configurator.serviceName)
|
||||
|
||||
opt := nftables.TableFamilyIPv4
|
||||
if utils.IsIPv6(configurator.IP()) {
|
||||
if utils.IsIPv6(configurator.address.IP.String()) {
|
||||
opt = nftables.TableFamilyIPv6
|
||||
}
|
||||
|
||||
@@ -591,12 +624,13 @@ func (configurator *network) addNftablesRulesToLimitTrafficPorts(c *nfinternal.C
|
||||
}
|
||||
}
|
||||
|
||||
firstRule, err := insertCommonNFTablesRules(c, configurator.IP(), comment)
|
||||
vip := configurator.address.IP.String()
|
||||
firstRule, err := insertCommonNFTablesRules(c, vip, comment)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add common nftables rules: %w", err)
|
||||
}
|
||||
|
||||
if err := configurator.insertNFTablesRulesForServicePorts(c, configurator.IP(), comment, firstRule.Handle); err != nil {
|
||||
if err := configurator.insertNFTablesRulesForServicePorts(c, vip, comment, firstRule.Handle); err != nil {
|
||||
return fmt.Errorf("could not add nftables rules for service ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -1053,10 +1087,19 @@ func (configurator *network) removeNftablesRuleToLimitTrafficPorts(c *nfinternal
|
||||
|
||||
// DeleteIP - Remove an IP address from the interface
|
||||
func (configurator *network) DeleteIP() (bool, error) {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
var deleted bool
|
||||
err := configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
var deleteErr error
|
||||
deleted, deleteErr = configurator.deleteIP(intf)
|
||||
return deleteErr
|
||||
})
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
result, err := configurator.IsSet()
|
||||
func (configurator *network) deleteIP(intf netlink.Link) (bool, error) {
|
||||
result, err := configurator.isSet(intf)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "ip check in DeleteIP failed")
|
||||
}
|
||||
@@ -1066,7 +1109,7 @@ func (configurator *network) DeleteIP() (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
|
||||
if err = netlink.AddrDel(intf, configurator.address); err != nil {
|
||||
return false, errors.Wrap(err, "could not delete ip")
|
||||
}
|
||||
|
||||
@@ -1136,7 +1179,8 @@ func (configurator *network) addIptablesRulesForMasquerade() error {
|
||||
|
||||
// TO DO: It seems it is not be possible to use google/nftables with IPVS due to lack of IPVS matcher in nft
|
||||
func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client, comment string) error {
|
||||
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.IP(), configurator.IPVSMark())
|
||||
vip := configurator.address.IP.String()
|
||||
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, vip, configurator.ipvsMark)
|
||||
|
||||
markChain := &nftables.Chain{
|
||||
Name: "ipvs_prerouting",
|
||||
@@ -1150,7 +1194,7 @@ func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client,
|
||||
|
||||
markChain = c.AddChain(markChain)
|
||||
|
||||
ip := net.ParseIP(configurator.IP())
|
||||
ip := net.ParseIP(vip)
|
||||
|
||||
if ip.To4() != nil {
|
||||
ip = ip.To4()
|
||||
@@ -1160,7 +1204,7 @@ func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client,
|
||||
|
||||
port := binaryutil.BigEndian.PutUint16(configurator.ipvsPort)
|
||||
|
||||
mark := binaryutil.NativeEndian.PutUint32(configurator.IPVSMark())
|
||||
mark := binaryutil.NativeEndian.PutUint32(configurator.ipvsMark)
|
||||
|
||||
markRule := &nftables.Rule{
|
||||
Table: markChain.Table,
|
||||
@@ -1282,7 +1326,7 @@ func (configurator *network) removeNftablesRulesForMasquerade(c *nfinternal.Clie
|
||||
if configurator.serviceName != "" {
|
||||
comment = fmt.Sprintf(iptablesComment, configurator.serviceName)
|
||||
}
|
||||
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.IP(), configurator.IPVSMark())
|
||||
cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.address.IP.String(), configurator.ipvsMark)
|
||||
|
||||
r, err := c.FindRuleByComment(chain.Table, chain, cmt)
|
||||
if err != nil {
|
||||
@@ -1345,27 +1389,26 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
|
||||
|
||||
// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
|
||||
func (configurator *network) IsDADFAIL() bool {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
|
||||
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Get all the address
|
||||
addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Find the VIP and check if it is DADFAILED
|
||||
for _, address := range addresses {
|
||||
if address.IP.Equal(configurator.address.IP) && addressHasDADFAILEDFlag(address) {
|
||||
return true
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
var dadFailed bool
|
||||
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
addresses, err := netlink.AddrList(intf, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for _, address := range addresses {
|
||||
if address.IP.Equal(configurator.address.IP) && addressHasDADFAILEDFlag(address) {
|
||||
dadFailed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return dadFailed
|
||||
}
|
||||
|
||||
func addressHasDADFAILEDFlag(address netlink.Addr) bool {
|
||||
@@ -1374,6 +1417,16 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
|
||||
|
||||
// isSet - Check to see if VIP is set
|
||||
func (configurator *network) IsSet() (result *netlink.Addr, err error) {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
err = configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
result, err = configurator.isSet(intf)
|
||||
return err
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (configurator *network) isSet(intf netlink.Link) (result *netlink.Addr, err error) {
|
||||
var addresses []netlink.Addr
|
||||
|
||||
if configurator.address == nil {
|
||||
@@ -1384,7 +1437,7 @@ func (configurator *network) IsSet() (result *netlink.Addr, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
|
||||
addresses, err = netlink.AddrList(intf, 0)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "could not list addresses")
|
||||
|
||||
@@ -1404,10 +1457,10 @@ func (configurator *network) IsSet() (result *netlink.Addr, err error) {
|
||||
func (configurator *network) SetIP(ip string) error {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
return configurator.setIP(ip)
|
||||
}
|
||||
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
|
||||
func (configurator *network) setIP(ip string) error {
|
||||
if strings.Contains("/", ip) {
|
||||
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
|
||||
}
|
||||
@@ -1439,7 +1492,7 @@ func (configurator *network) SetIP(ip string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if configurator.address != nil && configurator.IsDNS() {
|
||||
if configurator.address != nil && configurator.dnsName != "" {
|
||||
addr.ValidLft = defaultValidLft
|
||||
} else {
|
||||
addr.ValidLft = math.MaxInt
|
||||
@@ -1449,9 +1502,94 @@ func (configurator *network) SetIP(ip string) error {
|
||||
addr.PreferedLft = 0
|
||||
|
||||
configurator.address = addr
|
||||
if configurator.routingProtocol != 0 {
|
||||
markKubeVIPAddress(configurator.address, configurator.routingProtocol)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsKubeVIPAddress reports whether an address carries kube-vip's configured
|
||||
// protocol origin. IFA_PROTO is supported for both IPv4 and IPv6 on Linux 5.18+.
|
||||
func IsKubeVIPAddress(address netlink.Addr, protocol int) bool {
|
||||
return protocol >= firstUserAddressProtocol && address.Protocol == protocol
|
||||
}
|
||||
|
||||
// RetainedKubeVIPAddressKeys returns the tagged kernel addresses matching the
|
||||
// supplied VIPs. Callers use the keys with CleanupKubeVIPAddresses after they
|
||||
// determine which VIPs remain locally referenced.
|
||||
func RetainedKubeVIPAddressKeys(protocol int, vips map[string]struct{}) (map[string]struct{}, error) {
|
||||
retained := make(map[string]struct{})
|
||||
err := forEachKubeVIPAddress(protocol, func(_ netlink.Link, address netlink.Addr) error {
|
||||
if _, retain := vips[address.IP.String()]; retain {
|
||||
retained[addressKey(address)] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retained, nil
|
||||
}
|
||||
|
||||
// CleanupKubeVIPAddresses removes kube-vip addresses not reasserted by this
|
||||
// process. The retained keys must come from successful AddrReplace operations.
|
||||
func CleanupKubeVIPAddresses(protocol int, retained map[string]struct{}) (int, error) {
|
||||
removed := 0
|
||||
err := forEachKubeVIPAddress(protocol, func(link netlink.Link, address netlink.Addr) error {
|
||||
key := addressKey(address)
|
||||
if key == "" {
|
||||
return nil
|
||||
}
|
||||
if _, keep := retained[key]; keep {
|
||||
return nil
|
||||
}
|
||||
if err := netlink.AddrDel(link, &address); err != nil {
|
||||
return errors.Wrapf(err, "deleting kube-vip address %q from interface %q", address.IP, link.Attrs().Name)
|
||||
}
|
||||
removed++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func forEachKubeVIPAddress(protocol int, visit func(netlink.Link, netlink.Addr) error) error {
|
||||
links, err := netlink.LinkList()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "listing network links")
|
||||
}
|
||||
for _, link := range links {
|
||||
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "listing addresses on interface %q", link.Attrs().Name)
|
||||
}
|
||||
for _, address := range addresses {
|
||||
if IsKubeVIPAddress(address, protocol) {
|
||||
if err := visit(link, address); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addressKey(address netlink.Addr) string {
|
||||
if address.LinkIndex <= 0 || address.IP == nil {
|
||||
return ""
|
||||
}
|
||||
prefixLength, _ := address.Mask.Size()
|
||||
return fmt.Sprintf("%d/%s/%d", address.LinkIndex, address.IP, prefixLength)
|
||||
}
|
||||
|
||||
func markKubeVIPAddress(address *netlink.Addr, protocol int) {
|
||||
if address != nil {
|
||||
address.Protocol = protocol
|
||||
}
|
||||
}
|
||||
|
||||
// SetServicePorts updates the service ports from the service
|
||||
// If you want to limit traffic to the VIP to only the service ports, add service ports to the network firstly.
|
||||
func (configurator *network) SetServicePorts(service *v1.Service) {
|
||||
@@ -1465,8 +1603,8 @@ func (configurator *network) SetServicePorts(service *v1.Service) {
|
||||
|
||||
// IP - return the IP Address
|
||||
func (configurator *network) IP() string {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
|
||||
if configurator.address == nil || configurator.address.IP == nil {
|
||||
return ""
|
||||
@@ -1476,8 +1614,8 @@ func (configurator *network) IP() string {
|
||||
}
|
||||
|
||||
func (configurator *network) CIDR() string {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
|
||||
if configurator.address == nil || configurator.address.IPNet == nil {
|
||||
return ""
|
||||
@@ -1488,24 +1626,30 @@ func (configurator *network) CIDR() string {
|
||||
|
||||
// IP - return the IP Address
|
||||
func (configurator *network) IPisLinkLocal() bool {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
|
||||
return configurator.address.IP.IsLinkLocalUnicast()
|
||||
}
|
||||
|
||||
// DNSName return the configured dnsName when use DNS
|
||||
func (configurator *network) DNSName() string {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.dnsName
|
||||
}
|
||||
|
||||
// IsDNS - when dnsName is configured
|
||||
func (configurator *network) IsDNS() bool {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.dnsName != ""
|
||||
}
|
||||
|
||||
// IsDDNS - return true if use dynamic dns
|
||||
func (configurator *network) IsDDNS() bool {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.isDDNS
|
||||
}
|
||||
|
||||
@@ -1514,12 +1658,19 @@ func (configurator *network) IsDDNS() bool {
|
||||
// it's expected that dynamic DNS should be configured so
|
||||
// the fqdn for apiserver endpoint is dDNSHostName.{LocalDomain}
|
||||
func (configurator *network) DDNSHostName() string {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return getHostName(configurator.dnsName)
|
||||
}
|
||||
|
||||
// Interface - return the Interface name
|
||||
func (configurator *network) Interface() string {
|
||||
return configurator.link.Intf.Attrs().Name
|
||||
var name string
|
||||
_ = configurator.link.WithInterface(func(intf netlink.Link) error {
|
||||
name = intf.Attrs().Name
|
||||
return nil
|
||||
})
|
||||
return name
|
||||
}
|
||||
|
||||
func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
|
||||
@@ -1530,32 +1681,28 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
|
||||
}
|
||||
|
||||
l := intfMgr.Get(link)
|
||||
|
||||
l.Lock.Lock()
|
||||
defer l.Lock.Unlock()
|
||||
|
||||
// Get addresses on adapter
|
||||
addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Compare all addresses to new service address, and remove if needed
|
||||
for _, existing := range addrs {
|
||||
if existing.IP.String() == address {
|
||||
// We've found the existing address
|
||||
found = true
|
||||
// linting issue
|
||||
existing := existing
|
||||
if err = netlink.AddrDel(l.Intf, &existing); err != nil {
|
||||
return true, errors.Wrap(err, "could not delete ip")
|
||||
err = l.WithInterface(func(intf netlink.Link) error {
|
||||
addrs, listErr := netlink.AddrList(intf, netlink.FAMILY_ALL)
|
||||
if listErr != nil {
|
||||
return listErr
|
||||
}
|
||||
for _, existing := range addrs {
|
||||
if existing.IP.String() == address {
|
||||
found = true
|
||||
existing := existing
|
||||
if deleteErr := netlink.AddrDel(intf, &existing); deleteErr != nil {
|
||||
return errors.Wrap(deleteErr, "could not delete ip")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return // Didn't find the address on the adapter
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
|
||||
func (configurator *network) SetMask(mask string) error {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
selectedMask := mask
|
||||
var err error
|
||||
|
||||
@@ -1563,8 +1710,12 @@ func (configurator *network) SetMask(mask string) error {
|
||||
return fmt.Errorf("no mask provided")
|
||||
}
|
||||
|
||||
if configurator.IP() != "" {
|
||||
selectedMask, err = SelectSubnet(configurator.IP(), mask)
|
||||
ip := ""
|
||||
if configurator.address != nil && configurator.address.IP != nil {
|
||||
ip = configurator.address.IP.String()
|
||||
}
|
||||
if ip != "" {
|
||||
selectedMask, err = SelectSubnet(ip, mask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to select mask %q: %w", mask, err)
|
||||
}
|
||||
@@ -1580,8 +1731,8 @@ func (configurator *network) SetMask(mask string) error {
|
||||
size := DefaultMaskIPv4
|
||||
family := utils.IPv4Family
|
||||
|
||||
if configurator.IP() != "" {
|
||||
if utils.IsIPv6(configurator.IP()) {
|
||||
if ip != "" {
|
||||
if utils.IsIPv6(ip) {
|
||||
size = DefaultMaskIPv6
|
||||
family = utils.IPv6Family
|
||||
}
|
||||
@@ -1596,20 +1747,29 @@ func (configurator *network) SetMask(mask string) error {
|
||||
return fmt.Errorf("failed to create mask /%d", m)
|
||||
}
|
||||
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
|
||||
configurator.address.Mask = toSet
|
||||
return nil
|
||||
}
|
||||
|
||||
func (configurator *network) SetHasEndpoints(value bool) {
|
||||
log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
ip := ""
|
||||
if configurator.address != nil && configurator.address.IP != nil {
|
||||
ip = configurator.address.IP.String()
|
||||
}
|
||||
log.Debug("setting HasEndpoints", "ip", ip, "value", value)
|
||||
configurator.hasEndpoints = value
|
||||
}
|
||||
|
||||
func (configurator *network) HasEndpoints() bool {
|
||||
log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
ip := ""
|
||||
if configurator.address != nil && configurator.address.IP != nil {
|
||||
ip = configurator.address.IP.String()
|
||||
}
|
||||
log.Debug("getting HasEndpoints", "ip", ip, "value", configurator.hasEndpoints)
|
||||
return configurator.hasEndpoints
|
||||
}
|
||||
|
||||
@@ -1618,14 +1778,20 @@ func (configurator *network) ARPName() string {
|
||||
}
|
||||
|
||||
func (configurator *network) GetPossibleSubnets() string {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.possibleSubnets
|
||||
}
|
||||
|
||||
func (configurator *network) DHCPFamily() string {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.dhcpFamily
|
||||
}
|
||||
|
||||
func (configurator *network) IPVSMark() uint32 {
|
||||
configurator.mu.RLock()
|
||||
defer configurator.mu.RUnlock()
|
||||
return configurator.ipvsMark
|
||||
}
|
||||
|
||||
|
||||
36
pkg/vip/address_dad_linux_test.go
Normal file
36
pkg/vip/address_dad_linux_test.go
Normal file
@@ -0,0 +1,36 @@
|
||||
//go:build linux
|
||||
|
||||
package vip
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func TestAddIPPerCallDADSkipDoesNotPersist(t *testing.T) {
|
||||
address, err := netlink.ParseAddr("2001:db8::10/128")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
interfaceManager := networkinterface.NewManager()
|
||||
configurator := &network{
|
||||
address: address,
|
||||
link: interfaceManager.Get(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kube-vip-dad-test"}}),
|
||||
}
|
||||
|
||||
// The netlink operation may fail without CAP_NET_ADMIN, but the address
|
||||
// flags are set before that operation and are what this test exercises.
|
||||
_, _ = configurator.AddIP(false, true)
|
||||
if configurator.address.Flags&unix.IFA_F_NODAD == 0 {
|
||||
t.Fatal("skipDAD=true did not set IFA_F_NODAD")
|
||||
}
|
||||
|
||||
_, _ = configurator.AddIP(false, false)
|
||||
if configurator.address.Flags&unix.IFA_F_NODAD != 0 {
|
||||
t.Fatal("IFA_F_NODAD persisted into a normal AddIP call")
|
||||
}
|
||||
}
|
||||
220
pkg/vip/address_label_linux_test.go
Normal file
220
pkg/vip/address_label_linux_test.go
Normal file
@@ -0,0 +1,220 @@
|
||||
//go:build linux
|
||||
|
||||
package vip
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
"github.com/vishvananda/netns"
|
||||
)
|
||||
|
||||
const kubeVIPProtocol = 248
|
||||
|
||||
// requireNetworkNamespaces makes the privileged CI job fail instead of silently
|
||||
// skipping when it cannot enter a network namespace.
|
||||
var requireNetworkNamespaces = os.Getenv("KUBE_VIP_REQUIRE_NETNS") != ""
|
||||
|
||||
func TestAddressProtocolRoundTripsThroughNetlink(t *testing.T) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
originalNamespace, err := netns.Get()
|
||||
if err != nil {
|
||||
t.Fatalf("getting current network namespace: %v", err)
|
||||
}
|
||||
defer originalNamespace.Close()
|
||||
testNamespace, err := netns.New()
|
||||
if err != nil {
|
||||
if requireNetworkNamespaces {
|
||||
t.Fatalf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
t.Skipf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
defer testNamespace.Close()
|
||||
defer func() {
|
||||
if err := netns.Set(originalNamespace); err != nil {
|
||||
t.Errorf("restoring network namespace: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto0"}}
|
||||
if err := netlink.LinkAdd(link); err != nil {
|
||||
t.Fatalf("creating test interface: %v", err)
|
||||
}
|
||||
if err := netlink.LinkSetUp(link); err != nil {
|
||||
t.Fatalf("bringing test interface up: %v", err)
|
||||
}
|
||||
|
||||
parsed, err := netlink.ParseAddr("192.0.2.10/32")
|
||||
if err != nil {
|
||||
t.Fatalf("parsing IPv4 address: %v", err)
|
||||
}
|
||||
markKubeVIPAddress(parsed, kubeVIPProtocol)
|
||||
if err := netlink.AddrReplace(link, parsed); err != nil {
|
||||
t.Fatalf("adding IPv4 address with protocol: %v", err)
|
||||
}
|
||||
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
t.Fatalf("listing addresses: %v", err)
|
||||
}
|
||||
for _, configured := range addresses {
|
||||
if configured.IP.Equal(net.ParseIP("192.0.2.10")) {
|
||||
if !IsKubeVIPAddress(configured, kubeVIPProtocol) {
|
||||
t.Fatalf("configured address = %+v, want kube-vip protocol", configured)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("IPv4 address with kube-vip protocol was not configured")
|
||||
}
|
||||
|
||||
func TestKubeVIPAddressProtocolRoundTripsThroughIPv6Netlink(t *testing.T) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
originalNamespace, err := netns.Get()
|
||||
if err != nil {
|
||||
t.Fatalf("getting current network namespace: %v", err)
|
||||
}
|
||||
defer originalNamespace.Close()
|
||||
testNamespace, err := netns.New()
|
||||
if err != nil {
|
||||
if requireNetworkNamespaces {
|
||||
t.Fatalf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
t.Skipf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
defer testNamespace.Close()
|
||||
defer func() {
|
||||
if err := netns.Set(originalNamespace); err != nil {
|
||||
t.Errorf("restoring network namespace: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto1"}}
|
||||
if err := netlink.LinkAdd(link); err != nil {
|
||||
t.Fatalf("creating test interface: %v", err)
|
||||
}
|
||||
if err := netlink.LinkSetUp(link); err != nil {
|
||||
t.Fatalf("bringing test interface up: %v", err)
|
||||
}
|
||||
|
||||
address, err := netlink.ParseAddr("2001:db8::10/128")
|
||||
if err != nil {
|
||||
t.Fatalf("parsing IPv6 address: %v", err)
|
||||
}
|
||||
markKubeVIPAddress(address, kubeVIPProtocol)
|
||||
if err := netlink.AddrReplace(link, address); err != nil {
|
||||
t.Fatalf("adding IPv6 address with protocol: %v", err)
|
||||
}
|
||||
addresses, err := netlink.AddrList(link, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
t.Fatalf("listing IPv6 addresses: %v", err)
|
||||
}
|
||||
for _, configured := range addresses {
|
||||
if configured.IP.Equal(net.ParseIP("2001:db8::10")) {
|
||||
if !IsKubeVIPAddress(configured, kubeVIPProtocol) {
|
||||
t.Fatalf("configured IPv6 address = %+v, want kube-vip protocol", configured)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("configured IPv6 addresses = %+v, want 2001:db8::10", addresses)
|
||||
}
|
||||
|
||||
func TestCleanupKubeVIPAddressesRemovesOnlyUnretainedProtocolAddresses(t *testing.T) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
originalNamespace, err := netns.Get()
|
||||
if err != nil {
|
||||
t.Fatalf("getting current network namespace: %v", err)
|
||||
}
|
||||
defer originalNamespace.Close()
|
||||
testNamespace, err := netns.New()
|
||||
if err != nil {
|
||||
if requireNetworkNamespaces {
|
||||
t.Fatalf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
t.Skipf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
defer testNamespace.Close()
|
||||
defer func() {
|
||||
if err := netns.Set(originalNamespace); err != nil {
|
||||
t.Errorf("restoring network namespace: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvproto2"}}
|
||||
if err := netlink.LinkAdd(link); err != nil {
|
||||
t.Fatalf("creating test interface: %v", err)
|
||||
}
|
||||
if err := netlink.LinkSetUp(link); err != nil {
|
||||
t.Fatalf("bringing test interface up: %v", err)
|
||||
}
|
||||
for _, input := range []struct {
|
||||
cidr string
|
||||
protocol int
|
||||
}{
|
||||
{cidr: "192.0.2.10/32", protocol: kubeVIPProtocol},
|
||||
{cidr: "192.0.2.11/32", protocol: kubeVIPProtocol},
|
||||
{cidr: "192.0.2.12/32", protocol: 0},
|
||||
{cidr: "192.0.2.13/32", protocol: kubeVIPProtocol + 1},
|
||||
{cidr: "2001:db8::10/128", protocol: kubeVIPProtocol},
|
||||
{cidr: "2001:db8::11/128", protocol: kubeVIPProtocol},
|
||||
} {
|
||||
address, err := netlink.ParseAddr(input.cidr)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing address: %v", err)
|
||||
}
|
||||
markKubeVIPAddress(address, input.protocol)
|
||||
if err := netlink.AddrReplace(link, address); err != nil {
|
||||
t.Fatalf("adding address %s: %v", input.cidr, err)
|
||||
}
|
||||
}
|
||||
|
||||
retained, err := netlink.ParseAddr("192.0.2.10/32")
|
||||
if err != nil {
|
||||
t.Fatalf("parsing retained address: %v", err)
|
||||
}
|
||||
retained.LinkIndex = link.Attrs().Index
|
||||
retainedIPv6, err := netlink.ParseAddr("2001:db8::10/128")
|
||||
if err != nil {
|
||||
t.Fatalf("parsing retained IPv6 address: %v", err)
|
||||
}
|
||||
retainedIPv6.LinkIndex = link.Attrs().Index
|
||||
removed, err := CleanupKubeVIPAddresses(kubeVIPProtocol, map[string]struct{}{
|
||||
addressKey(*retained): {},
|
||||
addressKey(*retainedIPv6): {},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cleaning kube-vip addresses: %v", err)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("removed = %d, want 2", removed)
|
||||
}
|
||||
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
t.Fatalf("listing remaining addresses: %v", err)
|
||||
}
|
||||
wantAddresses := map[string]bool{
|
||||
"192.0.2.10": false,
|
||||
"192.0.2.12": false,
|
||||
"192.0.2.13": false,
|
||||
"2001:db8::10": false,
|
||||
}
|
||||
for _, address := range addresses {
|
||||
if _, wanted := wantAddresses[address.IP.String()]; wanted {
|
||||
wantAddresses[address.IP.String()] = true
|
||||
}
|
||||
}
|
||||
for address, found := range wantAddresses {
|
||||
if !found {
|
||||
t.Fatalf("remaining addresses = %+v, missing %s", addresses, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,32 @@ package vip
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
)
|
||||
|
||||
func TestNewConfigTagsStaticAndUpdatedAddresses(t *testing.T) {
|
||||
const protocol = 248
|
||||
networks, err := NewConfig("192.0.2.10", "lo", false, "32", false, "", false, false, 0, 0, protocol,
|
||||
"", "", "", false, 0, false, networkinterface.NewManager(), false, false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfig() error = %v", err)
|
||||
}
|
||||
configured, ok := networks[0].(*network)
|
||||
if !ok {
|
||||
t.Fatalf("network type = %T, want *network", networks[0])
|
||||
}
|
||||
if configured.address.Protocol != protocol {
|
||||
t.Fatalf("static address protocol = %d, want %d", configured.address.Protocol, protocol)
|
||||
}
|
||||
if err := configured.SetIP("192.0.2.11"); err != nil {
|
||||
t.Fatalf("SetIP() error = %v", err)
|
||||
}
|
||||
if configured.address.Protocol != protocol {
|
||||
t.Fatalf("updated address protocol = %d, want %d", configured.address.Protocol, protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldSkipDAD(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -27,12 +27,24 @@ type DHCPv4Client struct {
|
||||
initRebootFlag bool
|
||||
requestedIP net.IP
|
||||
broadcastFlag bool
|
||||
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
|
||||
releasedChan chan struct{} // indicate that the IP has been released
|
||||
stopChan chan struct{} // is used by external clients to stop DHCP
|
||||
errorChan chan error // indicates there was an error on the IP request
|
||||
ipChan chan string
|
||||
backoffAttempts uint
|
||||
stop sync.Once
|
||||
stopOnce sync.Once
|
||||
mtx sync.RWMutex
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) storeLease(lease *nclient4.Lease) {
|
||||
c.mtx.Lock()
|
||||
defer c.mtx.Unlock()
|
||||
c.lease = lease
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) loadLease() *nclient4.Lease {
|
||||
c.mtx.RLock()
|
||||
defer c.mtx.RUnlock()
|
||||
return c.lease
|
||||
}
|
||||
|
||||
// NewDHCPv4Client returns a new DHCP Client.
|
||||
@@ -40,7 +52,6 @@ func NewDHCPv4Client(iface *net.Interface, initRebootFlag bool, requestedIP stri
|
||||
return &DHCPv4Client{
|
||||
iface: iface,
|
||||
stopChan: make(chan struct{}),
|
||||
releasedChan: make(chan struct{}),
|
||||
errorChan: make(chan error),
|
||||
initRebootFlag: initRebootFlag,
|
||||
requestedIP: net.ParseIP(requestedIP),
|
||||
@@ -57,11 +68,13 @@ func (c *DHCPv4Client) WithHostName(hostname string) DHCPClient {
|
||||
|
||||
// Stop state-transition process and close dhcp client
|
||||
func (c *DHCPv4Client) Stop() {
|
||||
c.stop.Do(func() {
|
||||
close(c.ipChan)
|
||||
c.close()
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) close() {
|
||||
c.stopOnce.Do(func() {
|
||||
close(c.stopChan)
|
||||
})
|
||||
<-c.releasedChan
|
||||
}
|
||||
|
||||
// Gets the IPChannel for consumption
|
||||
@@ -129,72 +142,93 @@ func (c *DHCPv4Client) ErrorChannel() chan error {
|
||||
// ----------
|
||||
// Figure: State-transition diagram for DHCP clients
|
||||
func (c *DHCPv4Client) Start(ctx context.Context) error {
|
||||
dhcpCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
lease := c.requestWithBackoff(dhcpCtx)
|
||||
lease, err := c.requestWithBackoff(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DHCPv4 client failed: %w", err)
|
||||
}
|
||||
|
||||
c.initRebootFlag = false
|
||||
c.lease = lease
|
||||
|
||||
// Set up two ticker to renew/rebind regularly
|
||||
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
|
||||
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
|
||||
c.storeLease(lease)
|
||||
|
||||
// Set up two timers to renew/rebind regularly
|
||||
t1Timeout, t2Timeout := getLeaseTimeouts(lease)
|
||||
log.Debug("[DHCPv4] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
|
||||
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
|
||||
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c.stopChan:
|
||||
return c.killProcessing(t1, t2)
|
||||
case <-ctx.Done():
|
||||
c.close()
|
||||
return c.killProcessing(t1, t2)
|
||||
case <-t1.C:
|
||||
// renew is a unicast request of the IP renewal
|
||||
// A point on renew is: the library does not return the right message (NAK)
|
||||
// on renew error due to IP Change, but instead it returns a different error
|
||||
// This way there's not much to do other than log and continue, as the renew error
|
||||
// may be an offline server, or may be an incorrect package match
|
||||
lease, err := c.renew(dhcpCtx)
|
||||
lease, err := c.renew(ctx)
|
||||
if err == nil {
|
||||
c.lease = lease
|
||||
c.storeLease(lease)
|
||||
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
|
||||
log.Info("[DHCPv4] renew", "lease", lease)
|
||||
t2.Reset(t2Timeout)
|
||||
} else {
|
||||
log.Error("[DHCPv4] renew failed", "err", err)
|
||||
}
|
||||
t1.Reset(t1Timeout)
|
||||
case <-t2.C:
|
||||
// rebind is just like a request, but forcing to provide a new IP address
|
||||
lease, err := c.request(dhcpCtx, true)
|
||||
lease, err := c.request(ctx, true)
|
||||
if err == nil {
|
||||
c.lease = lease
|
||||
c.storeLease(lease)
|
||||
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
|
||||
log.Info("[DHCPv4] rebind", "lease", lease)
|
||||
} else {
|
||||
if _, ok := err.(*nclient4.ErrNak); !ok {
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
log.Error("[DHCPv4] rebind failed", "err", err)
|
||||
}
|
||||
log.Warn("[DHCPv4] ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
|
||||
lease = c.loadLease()
|
||||
log.Warn("[DHCPv4] ip may have changed", "ip", lease.ACK.YourIPAddr, "err", err)
|
||||
c.initRebootFlag = false
|
||||
c.lease = c.requestWithBackoff(dhcpCtx)
|
||||
lease, backoffErr := c.requestWithBackoff(ctx)
|
||||
if backoffErr != nil {
|
||||
log.Error("[DHCPv4] failed to reacquire lease", "err", backoffErr)
|
||||
continue
|
||||
}
|
||||
c.storeLease(lease)
|
||||
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
|
||||
}
|
||||
t1.Reset(t1Timeout)
|
||||
t2.Reset(t2Timeout)
|
||||
|
||||
case <-c.stopChan:
|
||||
// release is a unicast request of the IP release.
|
||||
var err error
|
||||
if err = c.release(); err != nil {
|
||||
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
|
||||
} else {
|
||||
log.Info("[DHCPv4] release", "lease", lease)
|
||||
}
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
|
||||
close(c.releasedChan)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getLeaseTimeouts(lease *nclient4.Lease) (time.Duration, time.Duration) {
|
||||
t1Timeout, t2Timeout := lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/2, (lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/8)*7
|
||||
log.Debug("[DHCPv4] timeouts", "address", lease.ACK.YourIPAddr.String(), "T1", t1Timeout, "T2", t2Timeout)
|
||||
return t1Timeout, t2Timeout
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) killProcessing(t1, t2 *time.Timer) error {
|
||||
// release is a unicast request of the IP release.
|
||||
var err error
|
||||
lease := c.loadLease()
|
||||
if lease != nil {
|
||||
if err = c.release(); err != nil {
|
||||
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
|
||||
} else {
|
||||
log.Info("[DHCPv4] release", "lease", lease)
|
||||
}
|
||||
}
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
return err
|
||||
}
|
||||
|
||||
// --------------------------------------------------------
|
||||
// | |INIT-REBOOT | RENEWING |REBINDING |
|
||||
// --------------------------------------------------------
|
||||
@@ -204,7 +238,7 @@ func (c *DHCPv4Client) Start(ctx context.Context) error {
|
||||
// |ciaddr |zero | IP address |IP address|
|
||||
// --------------------------------------------------------
|
||||
|
||||
func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) *nclient4.Lease {
|
||||
func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) (*nclient4.Lease, error) {
|
||||
backoff := backoff.Backoff{
|
||||
Factor: 2,
|
||||
Jitter: true,
|
||||
@@ -217,32 +251,47 @@ func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) *nclient4.Lease {
|
||||
|
||||
log.Debug("[DHCPv4]", "attempts", c.backoffAttempts)
|
||||
|
||||
RequestLoop:
|
||||
for {
|
||||
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
|
||||
lease, err = c.request(ctx, false)
|
||||
if err != nil {
|
||||
dur := backoff.Duration()
|
||||
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
|
||||
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
|
||||
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
|
||||
c.errorChan <- errMsg
|
||||
c.Stop()
|
||||
return nil
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("[DHCPv4] context error: %w", ctx.Err())
|
||||
default:
|
||||
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
|
||||
lease, err = c.request(ctx, false)
|
||||
if err != nil {
|
||||
dur := backoff.Duration()
|
||||
|
||||
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
|
||||
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
|
||||
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
|
||||
c.errorChan <- errMsg
|
||||
return nil, errMsg
|
||||
}
|
||||
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
|
||||
t := time.NewTimer(dur)
|
||||
select {
|
||||
case <-t.C:
|
||||
t.Stop()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
continue RequestLoop
|
||||
}
|
||||
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
|
||||
time.Sleep(dur)
|
||||
continue
|
||||
backoff.Reset()
|
||||
break RequestLoop
|
||||
}
|
||||
backoff.Reset()
|
||||
break
|
||||
}
|
||||
|
||||
if c.ipChan != nil {
|
||||
log.Debug("[DHCPv4] using channel")
|
||||
c.ipChan <- lease.ACK.YourIPAddr.String()
|
||||
// Nothing closes ipChan, so never block on a consumer that already stopped.
|
||||
select {
|
||||
case c.ipChan <- lease.ACK.YourIPAddr.String():
|
||||
case <-c.stopChan:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
|
||||
return lease
|
||||
return lease, nil
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) request(ctx context.Context, rebind bool) (*nclient4.Lease, error) {
|
||||
@@ -289,7 +338,12 @@ func (c *DHCPv4Client) release() error {
|
||||
defer dhclient.Close()
|
||||
|
||||
// TODO modify lease
|
||||
return dhclient.Release(c.lease)
|
||||
err = dhclient.Release(c.lease)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DHCPv4 release failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *DHCPv4Client) renew(ctx context.Context) (*nclient4.Lease, error) {
|
||||
|
||||
40
pkg/vip/dhcpv4_backoff_test.go
Normal file
40
pkg/vip/dhcpv4_backoff_test.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package vip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDHCPv4BackoffExhaustionDoesNotDeadlock(t *testing.T) {
|
||||
// DEFECT: requestWithBackoff calls Stop from the Start/request goroutine before Start can close releasedChan, so exhausted DHCPv4 retries deadlock (pkg/vip/dhcpv4.go:229).
|
||||
client := NewDHCPv4Client(
|
||||
&net.Interface{Name: "definitely-not-a-kube-vip-interface"},
|
||||
false,
|
||||
"",
|
||||
1,
|
||||
false,
|
||||
)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = client.requestWithBackoff(context.Background())
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-client.ErrorChannel():
|
||||
if err == nil {
|
||||
t.Fatal("expected DHCPv4 request error")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("DHCPv4 did not report exhausted backoff")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("DHCPv4 backoff exhaustion deadlocked")
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,9 @@ func init() {
|
||||
}
|
||||
|
||||
type DHCPv6ClientManager struct {
|
||||
// mu guards clients and the reference counts of its entries together, so a
|
||||
// concurrent Add cannot join a client that Delete is already retiring.
|
||||
mu sync.Mutex
|
||||
clients map[string]*DHCPv6InternalClient
|
||||
}
|
||||
|
||||
@@ -33,17 +36,17 @@ func NewDHCPv6ClientManager() *DHCPv6ClientManager {
|
||||
}
|
||||
|
||||
func (m *DHCPv6ClientManager) Get(iface string) *DHCPv6InternalClient {
|
||||
c, exists := m.clients[iface]
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
return c
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return m.clients[iface]
|
||||
}
|
||||
|
||||
func (m *DHCPv6ClientManager) Add(iface string) (*DHCPv6InternalClient, error) {
|
||||
c := m.Get(iface)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if c != nil {
|
||||
if c := m.clients[iface]; c != nil {
|
||||
c.references.Add(1)
|
||||
return c, nil
|
||||
}
|
||||
@@ -57,15 +60,16 @@ func (m *DHCPv6ClientManager) Add(iface string) (*DHCPv6InternalClient, error) {
|
||||
}
|
||||
|
||||
func (m *DHCPv6ClientManager) Delete(iface string) {
|
||||
c := m.Get(iface)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if c != nil {
|
||||
c.references.Add(-1)
|
||||
ref := c.references.Load()
|
||||
if ref < 1 {
|
||||
c.client.Close()
|
||||
delete(m.clients, iface)
|
||||
}
|
||||
c := m.clients[iface]
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
if c.references.Add(-1) < 1 {
|
||||
c.client.Close()
|
||||
delete(m.clients, iface)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,17 +94,30 @@ func NewDHCPv6InternalClient(iface string) (*DHCPv6InternalClient, error) {
|
||||
|
||||
type DHCPv6Client struct {
|
||||
iface *net.Interface
|
||||
managerKey string
|
||||
ddnsHostName string
|
||||
initRebootFlag bool
|
||||
requestedIP net.IP
|
||||
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
|
||||
releasedChan chan struct{} // indicate that the IP has been released
|
||||
errorChan chan error // indicates there was an error on the IP request
|
||||
ipChan chan string
|
||||
ic *DHCPv6InternalClient
|
||||
addr *dhcpv6.OptIAAddress
|
||||
backoffAttempts uint
|
||||
stop sync.Once
|
||||
mtx sync.RWMutex
|
||||
}
|
||||
|
||||
func (c *DHCPv6Client) storeAddr(addr *dhcpv6.OptIAAddress) {
|
||||
c.mtx.Lock()
|
||||
defer c.mtx.Unlock()
|
||||
c.addr = addr
|
||||
}
|
||||
|
||||
func (c *DHCPv6Client) loadAddr() *dhcpv6.OptIAAddress {
|
||||
c.mtx.RLock()
|
||||
defer c.mtx.RUnlock()
|
||||
return c.addr
|
||||
}
|
||||
|
||||
// NewDHCPv6Client returns a new DHCP6 Client.
|
||||
@@ -117,8 +134,8 @@ func NewDHCPv6Client(iface *net.Interface, parent netlink.Link, initRebootFlag b
|
||||
|
||||
return &DHCPv6Client{
|
||||
iface: iface,
|
||||
managerKey: name,
|
||||
stopChan: make(chan struct{}),
|
||||
releasedChan: make(chan struct{}),
|
||||
errorChan: make(chan error),
|
||||
initRebootFlag: initRebootFlag,
|
||||
requestedIP: net.ParseIP(requestedIP),
|
||||
@@ -135,12 +152,15 @@ func (c *DHCPv6Client) WithHostName(hostname string) DHCPClient {
|
||||
|
||||
// Stop state-transition process and close dhcp client
|
||||
func (c *DHCPv6Client) Stop() {
|
||||
c.close()
|
||||
}
|
||||
|
||||
// Close dhcp client channels
|
||||
func (c *DHCPv6Client) close() {
|
||||
c.stop.Do(func() {
|
||||
close(c.ipChan)
|
||||
close(c.stopChan)
|
||||
})
|
||||
<-c.releasedChan
|
||||
dhcpv6ClientManager.Delete(c.iface.Name)
|
||||
dhcpv6ClientManager.Delete(c.managerKey)
|
||||
}
|
||||
|
||||
// Gets the IPChannel for consumption
|
||||
@@ -154,27 +174,27 @@ func (c *DHCPv6Client) ErrorChannel() chan error {
|
||||
}
|
||||
|
||||
func (c *DHCPv6Client) Start(ctx context.Context) error {
|
||||
dhcpCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
addr, err := c.requestWithBackoff(dhcpCtx)
|
||||
addr, err := c.requestWithBackoff(ctx)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("DHCPv6 client failed: %w", err)
|
||||
}
|
||||
|
||||
c.addr = addr
|
||||
|
||||
c.initRebootFlag = false
|
||||
|
||||
c.storeAddr(addr)
|
||||
|
||||
// Set up two ticker to renew/rebind regularly
|
||||
t1Timeout := c.addr.PreferredLifetime / 2
|
||||
t2Timeout := (c.addr.ValidLifetime / 8) * 7
|
||||
log.Debug("[DHCPv6] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
|
||||
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
|
||||
t1Timeout, t2Timeout := getAddrTimeouts(addr)
|
||||
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c.stopChan:
|
||||
return c.killProcessing(t1, t2)
|
||||
case <-ctx.Done():
|
||||
c.close()
|
||||
return c.killProcessing(t1, t2)
|
||||
case <-t1.C:
|
||||
// renew is a unicast request of the IP renewal
|
||||
// A point on renew is: the library does not return the right message (NAK)
|
||||
@@ -182,49 +202,65 @@ func (c *DHCPv6Client) Start(ctx context.Context) error {
|
||||
// This way there's not much to do other than log and continue, as the renew error
|
||||
// may be an offline server, or may be an incorrect package match
|
||||
|
||||
addr, err := c.renew(dhcpCtx)
|
||||
addr, err := c.renew(ctx)
|
||||
if err == nil {
|
||||
c.addr = addr
|
||||
c.storeAddr(addr)
|
||||
log.Info("[DHCPv6] renew", "addr", addr.IPv6Addr.String())
|
||||
t1Timeout, t2Timeout = getAddrTimeouts(addr)
|
||||
t2.Reset(t2Timeout)
|
||||
} else {
|
||||
log.Error("[DHCPv6] renew failed", "err", err)
|
||||
}
|
||||
t1.Reset(t1Timeout)
|
||||
case <-t2.C:
|
||||
// rebind is just like a request, but forcing to provide a new IP address
|
||||
addr, err := c.request(dhcpCtx, true)
|
||||
addr, err := c.request(ctx, true)
|
||||
if err == nil {
|
||||
c.addr = addr
|
||||
c.storeAddr(addr)
|
||||
log.Info("[DHCPv6] rebind", "lease", addr)
|
||||
t1Timeout, t2Timeout = getAddrTimeouts(addr)
|
||||
} else {
|
||||
addr = c.loadAddr()
|
||||
log.Warn("[DHCPv6] ip may have changed", "ip", addr.IPv6Addr.String(), "err", err)
|
||||
c.initRebootFlag = false
|
||||
c.addr, err = c.requestWithBackoff(dhcpCtx)
|
||||
log.Error("[DHCPv6] rebind failed", "err", err)
|
||||
addr, backoffErr := c.requestWithBackoff(ctx)
|
||||
if backoffErr != nil {
|
||||
log.Error("[DHCPv6] failed to reacquire lease", "err", backoffErr)
|
||||
continue
|
||||
}
|
||||
c.storeAddr(addr)
|
||||
t1Timeout, t2Timeout = getAddrTimeouts(addr)
|
||||
}
|
||||
t1.Reset(t1Timeout)
|
||||
t2.Reset(t2Timeout)
|
||||
|
||||
case <-c.stopChan:
|
||||
// create new context for DHCP cleanup (independent)
|
||||
dhcpStopCtx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// IP address release.
|
||||
var err error
|
||||
if err = c.release(dhcpStopCtx); err != nil {
|
||||
log.Error("[DHCPv6] release failed", "err", err)
|
||||
} else {
|
||||
log.Info("[DHCPv6] released", "address", c.addr.String())
|
||||
}
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
|
||||
close(c.releasedChan)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getAddrTimeouts(addr *dhcpv6.OptIAAddress) (time.Duration, time.Duration) {
|
||||
t1Timeout, t2Timeout := addr.PreferredLifetime/2, (addr.ValidLifetime/8)*7
|
||||
log.Debug("[DHCPv6] timeouts", "address", addr.IPv6Addr.String(), "T1", t1Timeout, "T2", t2Timeout)
|
||||
return t1Timeout, t2Timeout
|
||||
}
|
||||
|
||||
func (c *DHCPv6Client) killProcessing(t1, t2 *time.Timer) error {
|
||||
// create new context for DHCP cleanup (independent)
|
||||
dhcpStopCtx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// IP address release.
|
||||
var err error
|
||||
if c.loadAddr() != nil {
|
||||
if err = c.release(dhcpStopCtx); err != nil {
|
||||
log.Error("[DHCPv6] release failed", "err", err)
|
||||
} else {
|
||||
log.Info("[DHCPv6] released", "address", c.addr.String())
|
||||
}
|
||||
}
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAddress, error) {
|
||||
backoff := backoff.Backoff{
|
||||
Factor: 2,
|
||||
@@ -233,34 +269,50 @@ func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAdd
|
||||
Max: 1 * time.Minute,
|
||||
}
|
||||
|
||||
var err error
|
||||
var addr *dhcpv6.OptIAAddress
|
||||
var err error
|
||||
|
||||
log.Debug("[DHCPv6]", "attempts", c.backoffAttempts)
|
||||
|
||||
RequestLoop:
|
||||
for {
|
||||
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("[DHCPv6] context error: %w", ctx.Err())
|
||||
default:
|
||||
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
|
||||
|
||||
addr, err = c.request(ctx, false)
|
||||
addr, err = c.request(ctx, false)
|
||||
|
||||
if err != nil {
|
||||
dur := backoff.Duration()
|
||||
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
|
||||
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
|
||||
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
|
||||
c.errorChan <- errMsg
|
||||
c.Stop()
|
||||
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
|
||||
if err != nil {
|
||||
dur := backoff.Duration()
|
||||
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
|
||||
errMsg := fmt.Errorf("failed to get an IPv6 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
|
||||
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
|
||||
c.errorChan <- errMsg
|
||||
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
|
||||
}
|
||||
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
|
||||
t := time.NewTimer(dur)
|
||||
select {
|
||||
case <-t.C:
|
||||
t.Stop()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
continue RequestLoop
|
||||
}
|
||||
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
|
||||
time.Sleep(dur)
|
||||
continue
|
||||
backoff.Reset()
|
||||
break RequestLoop
|
||||
}
|
||||
backoff.Reset()
|
||||
break
|
||||
}
|
||||
|
||||
if c.ipChan != nil {
|
||||
log.Debug("[DHCPv6] using channel")
|
||||
c.ipChan <- addr.IPv6Addr.String()
|
||||
// Nothing closes ipChan, so never block on a consumer that already stopped.
|
||||
select {
|
||||
case c.ipChan <- addr.IPv6Addr.String():
|
||||
case <-c.stopChan:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
|
||||
return addr, nil
|
||||
@@ -385,9 +437,10 @@ func getAddress(iana []*dhcpv6.OptIANA) (*dhcpv6.OptIAAddress, error) {
|
||||
return nil, fmt.Errorf("failed to get IANA")
|
||||
}
|
||||
|
||||
if len(iana) < 1 {
|
||||
return nil, fmt.Errorf("failed to get addresses data")
|
||||
addrs := iana[0].Options.Addresses()
|
||||
if len(addrs) == 0 {
|
||||
return nil, fmt.Errorf("IANA contained no addresses")
|
||||
}
|
||||
|
||||
return iana[0].Options.Addresses()[0], nil
|
||||
return addrs[0], nil
|
||||
}
|
||||
|
||||
83
pkg/vip/dhcpv6_robustness_test.go
Normal file
83
pkg/vip/dhcpv6_robustness_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package vip
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/insomniacslk/dhcp/dhcpv6"
|
||||
)
|
||||
|
||||
func TestDHCPv6StopReleasesManagerReferenceForParentInterface(t *testing.T) {
|
||||
// DEFECT: Stop deletes the manager entry using the VLAN child name even though NewDHCPv6Client keyed the shared client by its parent name (pkg/vip/dhcpv6.go:143).
|
||||
previousManager := dhcpv6ClientManager
|
||||
t.Cleanup(func() { dhcpv6ClientManager = previousManager })
|
||||
|
||||
references := &atomic.Int32{}
|
||||
references.Store(2)
|
||||
shared := &DHCPv6InternalClient{references: references}
|
||||
dhcpv6ClientManager = &DHCPv6ClientManager{
|
||||
clients: map[string]*DHCPv6InternalClient{"parent0": shared},
|
||||
}
|
||||
|
||||
client := &DHCPv6Client{
|
||||
iface: &net.Interface{Name: "vlan-child"},
|
||||
managerKey: "parent0",
|
||||
ipChan: make(chan string),
|
||||
stopChan: make(chan struct{}),
|
||||
ic: shared,
|
||||
addr: &dhcpv6.OptIAAddress{},
|
||||
}
|
||||
|
||||
client.Stop()
|
||||
|
||||
if got := references.Load(); got != 1 {
|
||||
t.Fatalf("manager reference count = %d, want 1 after stopping one VLAN client", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDHCPv6ClientManagerSharesOneClientPerParentInterface(t *testing.T) {
|
||||
references := &atomic.Int32{}
|
||||
references.Store(1)
|
||||
shared := &DHCPv6InternalClient{references: references}
|
||||
manager := &DHCPv6ClientManager{
|
||||
clients: map[string]*DHCPv6InternalClient{"parent0": shared},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 64 {
|
||||
wg.Go(func() {
|
||||
client, err := manager.Add("parent0")
|
||||
if err != nil {
|
||||
t.Errorf("Add() error = %v", err)
|
||||
return
|
||||
}
|
||||
if client != shared {
|
||||
t.Errorf("Add() client = %p, want the shared client %p", client, shared)
|
||||
}
|
||||
manager.Delete("parent0")
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := manager.Get("parent0"); got != shared {
|
||||
t.Fatalf("shared client = %v, want it retained while still referenced", got)
|
||||
}
|
||||
if got := references.Load(); got != 1 {
|
||||
t.Fatalf("manager reference count = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAddressRejectsIANAWithoutAddresses(t *testing.T) {
|
||||
// DEFECT: getAddress indexes the first IAADDR without checking whether the IANA contains one, so a malformed/expired reply panics (pkg/vip/dhcpv6.go:392).
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
t.Fatalf("getAddress panicked on an IANA without IAADDR: %v", recovered)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := getAddress([]*dhcpv6.OptIANA{{}}); err == nil {
|
||||
t.Fatal("getAddress accepted an IANA without an IAADDR")
|
||||
}
|
||||
}
|
||||
110
pkg/vip/util.go
110
pkg/vip/util.go
@@ -7,6 +7,7 @@ import (
|
||||
"net"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
@@ -14,6 +15,8 @@ import (
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
var errDefaultInterfaceSubscriptionClosed = errors.New("default interface subscription closed")
|
||||
|
||||
// getHostName return the hostname from the fqdn
|
||||
func getHostName(dnsName string) string {
|
||||
if dnsName == "" {
|
||||
@@ -75,26 +78,123 @@ func getDefaultRoute(family int) (*net.Interface, error) {
|
||||
return nil, errors.New("default route not found")
|
||||
}
|
||||
|
||||
// MonitorDefaultInterface monitor the default interface and catch the event of the default route
|
||||
// MonitorDefaultInterface monitors the default interface for route removal or link loss.
|
||||
func MonitorDefaultInterface(ctx context.Context, defaultIF *net.Interface) error {
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
return monitorDefaultInterfaceWithRetry(ctx, defaultIF, subscribeDefaultInterface, GetDefaultGatewayInterface, 100*time.Millisecond)
|
||||
}
|
||||
|
||||
func monitorDefaultInterfaceWithRetry(ctx context.Context, defaultIF *net.Interface,
|
||||
subscribe func(context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error),
|
||||
lookup func() (*net.Interface, error), retryDelay time.Duration) error {
|
||||
for {
|
||||
monitorCtx, cancel := context.WithCancel(ctx)
|
||||
routeCh, linkCh, err := subscribe(monitorCtx)
|
||||
if err == nil {
|
||||
err = monitorDefaultInterface(monitorCtx, defaultIF, routeCh, linkCh)
|
||||
}
|
||||
cancel()
|
||||
drainDefaultInterfaceSubscriptions(routeCh, linkCh)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, errDefaultInterfaceSubscriptionClosed) {
|
||||
log.Warn("default interface subscription failed, retrying", "err", err)
|
||||
} else if err == nil {
|
||||
return nil
|
||||
}
|
||||
if refreshed, lookupErr := lookup(); lookupErr == nil {
|
||||
defaultIF = refreshed
|
||||
} else {
|
||||
log.Warn("failed to refresh default interface while resubscribing", "err", lookupErr)
|
||||
}
|
||||
timer := time.NewTimer(retryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func subscribeDefaultInterface(ctx context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error) {
|
||||
const subscriptionBuffer = 64
|
||||
routeCh := make(chan netlink.RouteUpdate, subscriptionBuffer)
|
||||
if err := netlink.RouteSubscribe(routeCh, ctx.Done()); err != nil {
|
||||
return fmt.Errorf("subscribe route failed, error: %w", err)
|
||||
return nil, nil, fmt.Errorf("subscribe route failed, error: %w", err)
|
||||
}
|
||||
linkCh := make(chan netlink.LinkUpdate, subscriptionBuffer)
|
||||
if err := netlink.LinkSubscribe(linkCh, ctx.Done()); err != nil {
|
||||
return routeCh, nil, fmt.Errorf("subscribe link failed, error: %w", err)
|
||||
}
|
||||
|
||||
return routeCh, linkCh, nil
|
||||
}
|
||||
|
||||
func monitorDefaultInterface(ctx context.Context, defaultIF *net.Interface, routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) error {
|
||||
for {
|
||||
select {
|
||||
case r := <-routeCh:
|
||||
case r, ok := <-routeCh:
|
||||
if !ok {
|
||||
return subscriptionClosed(ctx, "route")
|
||||
}
|
||||
log.Debug(fmt.Sprintf("type: %d, route: %+v", r.Type, r.Route))
|
||||
if r.Type == syscall.RTM_DELROUTE && (r.Dst == nil || r.Dst.String() == "0.0.0.0/0") && r.LinkIndex == defaultIF.Index {
|
||||
if r.Type == syscall.RTM_DELROUTE && isDefaultRoute(r.Dst) && r.LinkIndex == defaultIF.Index {
|
||||
return fmt.Errorf("default route deleted and the default interface may be invalid")
|
||||
}
|
||||
case update, ok := <-linkCh:
|
||||
if !ok {
|
||||
return subscriptionClosed(ctx, "link")
|
||||
}
|
||||
if update.Link == nil {
|
||||
continue
|
||||
}
|
||||
attrs := update.Attrs()
|
||||
if attrs != nil && attrs.Index == defaultIF.Index && attrs.Flags&net.FlagUp == 0 {
|
||||
return fmt.Errorf("default interface %q is down", defaultIF.Name)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func subscriptionClosed(ctx context.Context, subscription string) error {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w: %s subscription closed", errDefaultInterfaceSubscriptionClosed, subscription)
|
||||
}
|
||||
|
||||
// isDefaultRoute accepts both families, matching the selection made by
|
||||
// GetDefaultGatewayInterface.
|
||||
func isDefaultRoute(dst *net.IPNet) bool {
|
||||
if dst == nil {
|
||||
return true
|
||||
}
|
||||
return dst.String() == "0.0.0.0/0" || dst.String() == "::/0"
|
||||
}
|
||||
|
||||
func drainDefaultInterfaceSubscriptions(routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) {
|
||||
timer := time.NewTimer(100 * time.Millisecond)
|
||||
defer timer.Stop()
|
||||
|
||||
for routeCh != nil || linkCh != nil {
|
||||
select {
|
||||
case _, ok := <-routeCh:
|
||||
if !ok {
|
||||
routeCh = nil
|
||||
}
|
||||
case _, ok := <-linkCh:
|
||||
if !ok {
|
||||
linkCh = nil
|
||||
}
|
||||
case <-timer.C:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GenerateMac() (mac string) {
|
||||
buf := make([]byte, 3)
|
||||
_, err := rand.Read(buf)
|
||||
|
||||
293
pkg/vip/util_linux_test.go
Normal file
293
pkg/vip/util_linux_test.go
Normal file
@@ -0,0 +1,293 @@
|
||||
//go:build linux
|
||||
|
||||
package vip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
"github.com/vishvananda/netns"
|
||||
)
|
||||
|
||||
func TestMonitorDefaultInterfaceDetectsDefaultRouteDeletionPerFamily(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
cidr string
|
||||
}{
|
||||
{name: "IPv4", cidr: "0.0.0.0/0"},
|
||||
{name: "IPv6", cidr: "::/0"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
defaultIF := &net.Interface{Index: 7, Name: "test0"}
|
||||
_, defaultRoute, err := net.ParseCIDR(test.cidr)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCIDR() error = %v", err)
|
||||
}
|
||||
routeCh := make(chan netlink.RouteUpdate, 1)
|
||||
routeCh <- netlink.RouteUpdate{
|
||||
Type: syscall.RTM_DELROUTE,
|
||||
Route: netlink.Route{Dst: defaultRoute, LinkIndex: defaultIF.Index},
|
||||
}
|
||||
linkCh := make(chan netlink.LinkUpdate)
|
||||
|
||||
err = monitorDefaultInterfaceForTest(t, context.Background(), defaultIF, routeCh, linkCh)
|
||||
if err == nil || !strings.Contains(err.Error(), "default route deleted") {
|
||||
t.Fatalf("monitor error = %v, want a default route deletion error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceReturnsErrorWhenLinkGoesDown(t *testing.T) {
|
||||
defaultIF := &net.Interface{Index: 7, Name: "test0"}
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
linkCh := make(chan netlink.LinkUpdate, 1)
|
||||
linkCh <- netlink.LinkUpdate{
|
||||
Link: &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Index: defaultIF.Index}},
|
||||
}
|
||||
|
||||
err := monitorDefaultInterfaceForTest(t, context.Background(), defaultIF, routeCh, linkCh)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when the default interface goes down")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "default interface \"test0\" is down") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceHandlesClosedSubscriptions(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
closeRoute bool
|
||||
cancel bool
|
||||
wantErr string
|
||||
}{
|
||||
{name: "closed route subscription", closeRoute: true, wantErr: "route subscription closed"},
|
||||
{name: "closed link subscription", wantErr: "link subscription closed"},
|
||||
{name: "context cancellation with closed route subscription", closeRoute: true, cancel: true},
|
||||
{name: "context cancellation with closed link subscription", cancel: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
linkCh := make(chan netlink.LinkUpdate)
|
||||
if test.closeRoute {
|
||||
close(routeCh)
|
||||
} else {
|
||||
close(linkCh)
|
||||
}
|
||||
if test.cancel {
|
||||
cancel()
|
||||
}
|
||||
|
||||
err := monitorDefaultInterfaceForTest(t, ctx, &net.Interface{}, routeCh, linkCh)
|
||||
if test.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("monitor error = %v, want nil", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("monitor error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func monitorDefaultInterfaceForTest(t *testing.T, ctx context.Context, defaultIF *net.Interface,
|
||||
routeCh <-chan netlink.RouteUpdate, linkCh <-chan netlink.LinkUpdate) error {
|
||||
t.Helper()
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- monitorDefaultInterface(ctx, defaultIF, routeCh, linkCh)
|
||||
}()
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return err
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("default interface monitor did not return")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceHandlesClosedSubscriptionsAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
linkCh := make(chan netlink.LinkUpdate)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- monitorDefaultInterface(ctx, &net.Interface{}, routeCh, linkCh)
|
||||
}()
|
||||
|
||||
cancel()
|
||||
close(routeCh)
|
||||
close(linkCh)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
t.Fatalf("monitor error = %v, want nil", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("monitor did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceRetriesClosedSubscription(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
secondSubscribed := make(chan struct{})
|
||||
attempts := 0
|
||||
subscribe := func(ctx context.Context) (chan netlink.RouteUpdate, chan netlink.LinkUpdate, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
attempts++
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
linkCh := make(chan netlink.LinkUpdate)
|
||||
if attempts == 1 {
|
||||
close(routeCh)
|
||||
close(linkCh)
|
||||
return routeCh, linkCh, nil
|
||||
}
|
||||
close(secondSubscribed)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
close(routeCh)
|
||||
close(linkCh)
|
||||
}()
|
||||
return routeCh, linkCh, nil
|
||||
}
|
||||
lookup := func() (*net.Interface, error) {
|
||||
return &net.Interface{Index: 2, Name: "refreshed"}, nil
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- monitorDefaultInterfaceWithRetry(ctx, &net.Interface{Index: 1, Name: "original"}, subscribe, lookup, time.Millisecond)
|
||||
}()
|
||||
select {
|
||||
case <-secondSubscribed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("monitor did not resubscribe after channel closure")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("monitor returned an error after cancellation: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("resubscribed monitor did not stop after cancellation")
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("subscription attempts = %d, want 2", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceIgnoresNilLink(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
routeCh := make(chan netlink.RouteUpdate)
|
||||
linkCh := make(chan netlink.LinkUpdate)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- monitorDefaultInterface(ctx, &net.Interface{}, routeCh, linkCh)
|
||||
}()
|
||||
|
||||
linkCh <- netlink.LinkUpdate{}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
t.Fatalf("monitor error = %v, want nil", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("monitor did not stop after nil link update")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDrainDefaultInterfaceSubscriptionsIsBounded(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
drainDefaultInterfaceSubscriptions(make(chan netlink.RouteUpdate), make(chan netlink.LinkUpdate))
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("subscription drain did not stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDefaultInterfaceReturnsErrorWhenTestLinkIsSetDown(t *testing.T) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
originalNS, err := netns.Get()
|
||||
if err != nil {
|
||||
t.Fatalf("getting current network namespace: %v", err)
|
||||
}
|
||||
defer originalNS.Close()
|
||||
|
||||
testNS, err := netns.New()
|
||||
if err != nil {
|
||||
if requireNetworkNamespaces {
|
||||
t.Fatalf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
t.Skipf("creating isolated network namespace: %v", err)
|
||||
}
|
||||
defer testNS.Close()
|
||||
defer func() {
|
||||
if err := netns.Set(originalNS); err != nil {
|
||||
t.Errorf("restoring network namespace: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kv-monitor0"}}
|
||||
if err := netlink.LinkAdd(link); err != nil {
|
||||
t.Fatalf("creating test interface: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := netlink.LinkDel(link); err != nil {
|
||||
t.Errorf("deleting test interface: %v", err)
|
||||
}
|
||||
}()
|
||||
if err := netlink.LinkSetUp(link); err != nil {
|
||||
t.Fatalf("bringing test interface up: %v", err)
|
||||
}
|
||||
|
||||
defaultIF, err := net.InterfaceByName(link.Attrs().Name)
|
||||
if err != nil {
|
||||
t.Fatalf("getting test interface: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
routeCh, linkCh, err := subscribeDefaultInterface(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("subscribing to link updates: %v", err)
|
||||
}
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- monitorDefaultInterface(ctx, defaultIF, routeCh, linkCh)
|
||||
}()
|
||||
|
||||
if err := netlink.LinkSetDown(link); err != nil {
|
||||
t.Fatalf("bringing test interface down: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err == nil || !strings.Contains(err.Error(), "default interface \"kv-monitor0\" is down") {
|
||||
t.Fatalf("monitor error = %v, want default-interface-down error", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("monitor did not report the interface going down")
|
||||
}
|
||||
}
|
||||
@@ -214,6 +214,10 @@ func ResolveVIP(ctx context.Context, c api.GoBgpServiceClient, vip string) []str
|
||||
// This counts Destination objects (used by the existing service BGP tests).
|
||||
func CheckPaths(ctx context.Context, c api.GoBgpServiceClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedDests int) []*api.Destination {
|
||||
var paths []*api.Destination
|
||||
// 120s: with ginkgo --procs=4, sibling processes create Kind clusters and load
|
||||
// images concurrently, which can delay kube-vip's endpoint-watch/advertise loop
|
||||
// well past 30s (observed >13s even in passing specs). Matches the timeout the
|
||||
// BGP health-check suite uses for route re-announcement.
|
||||
Eventually(func() error {
|
||||
var err error
|
||||
paths, err = ListPaths(ctx, c, family, prefixes)
|
||||
@@ -224,7 +228,7 @@ func CheckPaths(ctx context.Context, c api.GoBgpServiceClient, family *api.Famil
|
||||
return fmt.Errorf("expected %d destinations, found %d", expectedDests, len(paths))
|
||||
}
|
||||
return nil
|
||||
}, "30s", "1s").ShouldNot(HaveOccurred(), "should have %d destinations, found %d", expectedDests, len(paths))
|
||||
}, "120s", "1s").ShouldNot(HaveOccurred(), "should have %d destinations, found %d", expectedDests, len(paths))
|
||||
return paths
|
||||
}
|
||||
|
||||
|
||||
@@ -20,9 +20,11 @@ import (
|
||||
"github.com/kube-vip/kube-vip/testing/e2e"
|
||||
)
|
||||
|
||||
var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func() {
|
||||
var _ = Describe("kube-vip BGP when deployed as a regular pod", func() {
|
||||
if Mode == ModeBGP {
|
||||
var (
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
logger log.Logger
|
||||
imagePath string
|
||||
k8sImagePath string
|
||||
@@ -30,9 +32,8 @@ var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func()
|
||||
tempDirPathRoot string
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
|
||||
BeforeEach(func() {
|
||||
BeforeEach(OncePerOrdered, func() {
|
||||
ctx, cancel = context.WithCancel(context.TODO())
|
||||
klog.SetOutput(GinkgoWriter)
|
||||
logger = e2e.TestLogger{}
|
||||
|
||||
@@ -44,11 +45,11 @@ var _ = Describe("kube-vip BGP when deployed as a regular pod", Ordered, func()
|
||||
}
|
||||
})
|
||||
|
||||
BeforeAll(func() {
|
||||
BeforeEach(OncePerOrdered, func() {
|
||||
tempDirPathRoot = MustMkdirTemp("", fmt.Sprintf("%s-bgp-ds", testDirPrefix))
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
AfterEach(OncePerOrdered, func() {
|
||||
if os.Getenv("E2E_KEEP_LOGS") != "true" {
|
||||
Expect(os.RemoveAll(tempDirPathRoot)).To(Succeed())
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user