mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/kube-vip/kube-vip.git
synced 2026-09-20 16:13:49 +08:00
Compare commits
1 Commits
dupe_funct
...
fix/versio
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37e3446c5a |
2
.github/workflows/anchore-syft.yml
vendored
2
.github/workflows/anchore-syft.yml
vendored
@@ -26,6 +26,6 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
- name: Anchore SBOM Action
|
||||
uses: anchore/sbom-action@v0.20.4
|
||||
uses: anchore/sbom-action@v0.17.5
|
||||
with:
|
||||
format: cyclonedx-json
|
||||
|
||||
51
.github/workflows/ci-pull-request.yaml
vendored
51
.github/workflows/ci-pull-request.yaml
vendored
@@ -1,51 +0,0 @@
|
||||
name: For each PR
|
||||
on:
|
||||
pull_request:
|
||||
jobs:
|
||||
e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E tests
|
||||
strategy:
|
||||
matrix:
|
||||
mode: ["arp", "rt", "bgp"]
|
||||
fail-fast: true
|
||||
max-parallel: 1
|
||||
steps:
|
||||
- name: Ensure fs wont cause issues
|
||||
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Build image locally
|
||||
run: make dockerx86Local
|
||||
- name: Run Manifest generation tests
|
||||
run: make manifest-test
|
||||
- name: Run ARP mode tests v1.29.0 onwards
|
||||
run: make e2e-tests129-arp
|
||||
if: matrix.mode== 'arp'
|
||||
- name: Run RT mode tests v1.29.0 onwards
|
||||
run: make e2e-tests129-rt
|
||||
if: matrix.mode== 'rt'
|
||||
- name: Get GoBGP binaries
|
||||
run: make get-gobgp
|
||||
if: matrix.mode== 'bgp'
|
||||
- name: Run BGP mode tests v1.29.0 onwards
|
||||
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
|
||||
if: matrix.mode== 'bgp'
|
||||
service-e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E service tests
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Build image with iptables
|
||||
run: make dockerx86ActionIPTables
|
||||
- name: Run tests
|
||||
run: DOCKERTAG=action make service-tests
|
||||
55
.github/workflows/ci.yaml
vendored
55
.github/workflows/ci.yaml
vendored
@@ -1,16 +1,18 @@
|
||||
name: For each commit
|
||||
name: For each commit and PR
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
env:
|
||||
GO_VERSION: "1.21"
|
||||
jobs:
|
||||
validation:
|
||||
runs-on: ubuntu-latest
|
||||
name: Checks and linters
|
||||
steps:
|
||||
- name: Init
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential golint && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
- name: Install golangci-lint
|
||||
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
|
||||
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
@@ -35,14 +37,48 @@ jobs:
|
||||
name: Integration tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Run tests
|
||||
run: make integration-tests
|
||||
e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E ARP tests
|
||||
steps:
|
||||
- name: Ensure fs wont cause issues
|
||||
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Build image locally
|
||||
run: make dockerx86Local
|
||||
- name: Run Manifest generation tests
|
||||
run: make manifest-test
|
||||
- name: Run Control plane tests
|
||||
run: make e2e-tests
|
||||
- name: Run Control plane tests v1.29.0 onwards
|
||||
run: make e2e-tests129
|
||||
service-e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: E2E service tests
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Build image with iptables
|
||||
run: make dockerx86ActionIPTables
|
||||
- name: Run tests
|
||||
run: make integration-tests
|
||||
run: DOCKERTAG=action make service-tests
|
||||
image-vul-check:
|
||||
runs-on: ubuntu-latest
|
||||
name: Image vulnerability scan
|
||||
@@ -54,9 +90,10 @@ jobs:
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: "plndr/kube-vip:action"
|
||||
format: "table"
|
||||
exit-code: "1"
|
||||
image-ref: 'plndr/kube-vip:action'
|
||||
format: 'table'
|
||||
exit-code: '1'
|
||||
ignore-unfixed: true
|
||||
vuln-type: "os,library"
|
||||
severity: "CRITICAL,HIGH"
|
||||
vuln-type: 'os,library'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
|
||||
|
||||
8
.github/workflows/release.yaml
vendored
8
.github/workflows/release.yaml
vendored
@@ -27,12 +27,6 @@ jobs:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Generate Metadata
|
||||
uses: docker/metadata-action@v5.7.0
|
||||
id: metadata
|
||||
with:
|
||||
labels: |
|
||||
org.opencontainers.image.documentation=https://kube-vip.io/docs/
|
||||
- name: Build and push main branch
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -40,7 +34,6 @@ jobs:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
labels: ${{ steps.metadata.outputs.labels }}
|
||||
tags: >-
|
||||
plndr/kube-vip:${{ github.ref_name }},
|
||||
plndr/kube-vip:latest,
|
||||
@@ -54,7 +47,6 @@ jobs:
|
||||
file: Dockerfile_iptables
|
||||
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
labels: ${{ steps.metadata.outputs.labels }}
|
||||
tags: >-
|
||||
plndr/kube-vip-iptables:${{ github.ref_name }},
|
||||
plndr/kube-vip-iptables:latest,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:experimental
|
||||
|
||||
FROM golang:1.24.5-alpine3.22 as dev
|
||||
FROM golang:1.23.2-alpine3.20 as dev
|
||||
RUN apk add --no-cache git ca-certificates make
|
||||
RUN adduser -D appuser
|
||||
COPY . /src/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:experimental
|
||||
|
||||
FROM golang:1.24.5-alpine3.22 as dev
|
||||
FROM golang:1.23.2-alpine3.20 as dev
|
||||
RUN apk add --no-cache git make
|
||||
RUN adduser -D appuser
|
||||
COPY . /src/
|
||||
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
|
||||
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOOS=linux make build
|
||||
|
||||
FROM alpine:3.22.1
|
||||
FROM alpine:3.20.3
|
||||
# Update pkgs and add iptables
|
||||
RUN apk upgrade && \
|
||||
apk add --no-cache iptables iptables-legacy
|
||||
|
||||
38
Makefile
38
Makefile
@@ -5,7 +5,7 @@ TARGET := kube-vip
|
||||
.DEFAULT_GOAL := $(TARGET)
|
||||
|
||||
# These will be provided to the target
|
||||
VERSION := v1.0.0
|
||||
VERSION := v0.8.6
|
||||
|
||||
BUILD := `git rev-parse HEAD`
|
||||
|
||||
@@ -115,7 +115,7 @@ manifests:
|
||||
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
|
||||
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
|
||||
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
|
||||
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
|
||||
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
|
||||
@-rm ./kube-vip
|
||||
|
||||
manifest-test:
|
||||
@@ -133,21 +133,13 @@ integration-tests:
|
||||
go test -tags=integration,e2e -v ./pkg/etcd
|
||||
|
||||
e2e-tests:
|
||||
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
|
||||
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
|
||||
|
||||
e2e-tests129-arp:
|
||||
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
|
||||
e2e-tests129-rt:
|
||||
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
|
||||
e2e-tests129-bgp:
|
||||
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
|
||||
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
|
||||
e2e-tests129:
|
||||
V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
|
||||
|
||||
service-tests:
|
||||
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
|
||||
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/e2e/services -Services
|
||||
|
||||
trivy: dockerx86ActionIPTables
|
||||
docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
|
||||
@@ -159,21 +151,3 @@ trivy: dockerx86ActionIPTables
|
||||
--severity 'CRITICAL,HIGH' \
|
||||
$(REPOSITORY)/$(TARGET):action
|
||||
|
||||
kind-quick:
|
||||
echo "Standing up your cluster"
|
||||
kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
|
||||
kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
|
||||
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
|
||||
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
|
||||
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
|
||||
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
|
||||
|
||||
kind-reload:
|
||||
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
|
||||
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
|
||||
|
||||
get-gobgp:
|
||||
mkdir -p bin
|
||||
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
|
||||
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ High Availability and Load-Balancing
|
||||
|
||||

|
||||
|
||||
[](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [](https://insights.linuxfoundation.org/project/kube-vip) [&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
|
||||
[](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
|
||||
|
||||
## Overview
|
||||
Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
|
||||
@@ -32,7 +32,6 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
|
||||
- Service LoadBalancer address pools per namespace or global
|
||||
- Service LoadBalancer address via (existing network DHCP)
|
||||
- Service LoadBalancer address exposure to gateway via UPNP
|
||||
- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod.
|
||||
- ... manifest generation, vendor API integrations and many more...
|
||||
|
||||
## Why?
|
||||
|
||||
@@ -4,9 +4,8 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -23,7 +22,7 @@ func init() {
|
||||
var kubeKubeadm = &cobra.Command{
|
||||
Use: "kubeadm",
|
||||
Short: "Kubeadm functions",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
_ = cmd.Help()
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
},
|
||||
@@ -33,35 +32,32 @@ var kubeKubeadmInit = &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "kube-vip init",
|
||||
Long: "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalf("Error parsing environment from config: %v", err)
|
||||
}
|
||||
|
||||
// TODO - check for certain things VIP/interfaces
|
||||
if initConfig.Interface == "" {
|
||||
_ = cmd.Help()
|
||||
log.Error("No interface is specified for kube-vip to bind to")
|
||||
return
|
||||
log.Fatalln("No interface is specified for kube-vip to bind to")
|
||||
}
|
||||
|
||||
if initConfig.VIP == "" && initConfig.Address == "" {
|
||||
_ = cmd.Help()
|
||||
log.Error("No address is specified for kube-vip to expose services on")
|
||||
return
|
||||
log.Fatalln("No address is specified for kube-vip to expose services on")
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("generating VIPSubnet", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,40 +69,37 @@ var kubeKubeadmInit = &cobra.Command{
|
||||
var kubeKubeadmJoin = &cobra.Command{
|
||||
Use: "join",
|
||||
Short: "kube-vip join",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
|
||||
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalf("Error parsing environment from config: %v", err)
|
||||
}
|
||||
|
||||
// TODO - check for certain things VIP/interfaces
|
||||
if initConfig.Interface == "" {
|
||||
_ = cmd.Help()
|
||||
log.Error("No interface is specified for kube-vip to bind to")
|
||||
return
|
||||
log.Fatalln("No interface is specified for kube-vip to bind to")
|
||||
}
|
||||
|
||||
if initConfig.VIP == "" && initConfig.Address == "" {
|
||||
_ = cmd.Help()
|
||||
log.Error("No address is specified for kube-vip to expose services on")
|
||||
return
|
||||
log.Fatalln("No address is specified for kube-vip to expose services on")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
|
||||
log.Error("kubeConfig not found", "Path", kubeConfigPath)
|
||||
return
|
||||
log.Fatalf("Unable to find file [%s]", kubeConfigPath)
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("generating VIPSubnet", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,25 +3,23 @@ package cmd
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// manifests will eventually deprecate the kubeadm set of subcommands
|
||||
// manifests will be used to generate:
|
||||
// - Pod spec manifest, mainly used for a static pod (kubeadm)
|
||||
// - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
|
||||
// - RBAC manifest, used to generate the RBAC permissions for kube-vip
|
||||
|
||||
var taint, role, rolebinding bool
|
||||
// var inCluster bool
|
||||
var taint bool
|
||||
|
||||
func init() {
|
||||
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
|
||||
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
|
||||
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
|
||||
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
|
||||
|
||||
kubeManifest.AddCommand(kubeManifestPod)
|
||||
kubeManifest.AddCommand(kubeManifestDaemon)
|
||||
@@ -31,7 +29,7 @@ func init() {
|
||||
var kubeManifest = &cobra.Command{
|
||||
Use: "manifest",
|
||||
Short: "Manifest functions",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
_ = cmd.Help()
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
},
|
||||
@@ -40,29 +38,28 @@ var kubeManifest = &cobra.Command{
|
||||
var kubeManifestPod = &cobra.Command{
|
||||
Use: "pod",
|
||||
Short: "Generate a Pod Manifest",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalf("Error parsing environment from config: %v", err)
|
||||
}
|
||||
|
||||
// The control plane has a requirement for a VIP being specified
|
||||
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
|
||||
_ = cmd.Help()
|
||||
log.Error("No address is specified for kube-vip to expose services on")
|
||||
return
|
||||
log.Fatalln("No address is specified for kube-vip to expose services on")
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("config parse", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,28 +71,30 @@ var kubeManifestPod = &cobra.Command{
|
||||
var kubeManifestDaemon = &cobra.Command{
|
||||
Use: "daemonset",
|
||||
Short: "Generate a Daemonset Manifest",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalf("error parsing environment config: %v", err)
|
||||
}
|
||||
|
||||
// TODO - check for certain things VIP/interfaces
|
||||
|
||||
// The control plane has a requirement for a VIP being specified
|
||||
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
|
||||
_ = cmd.Help()
|
||||
log.Error("No address is specified for kube-vip to expose services on")
|
||||
return
|
||||
log.Fatalln("No address is specified for kube-vip to expose services on")
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("config parse", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,44 +106,33 @@ var kubeManifestDaemon = &cobra.Command{
|
||||
var kubeManifestRbac = &cobra.Command{
|
||||
Use: "rbac",
|
||||
Short: "Generate an RBAC Manifest",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
|
||||
// TODO - A load of text detailing what's actually happening
|
||||
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalf("Error parsing environment from config: %v", err)
|
||||
}
|
||||
|
||||
// The control plane has a requirement for a VIP being specified
|
||||
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
|
||||
_ = cmd.Help()
|
||||
log.Error("No address is specified for kube-vip to expose services on")
|
||||
return
|
||||
log.Fatalln("No address is specified for kube-vip to expose services on")
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("generating VIPSubnet", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
saCfg := kubevip.GenerateSA(&initConfig)
|
||||
roleCfg := kubevip.GenerateRole(&initConfig, role)
|
||||
if role {
|
||||
rolebinding = true
|
||||
}
|
||||
roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
|
||||
|
||||
// Output the YAML manifests to stdout
|
||||
fmt.Println("---") // Separator for YAML documents
|
||||
fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
|
||||
fmt.Println("---") // Separator for YAML documents
|
||||
fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
|
||||
fmt.Println("---") // Separator for YAML documents
|
||||
fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
|
||||
cfg := kubevip.GenerateSA()
|
||||
b, _ := yaml.Marshal(cfg)
|
||||
fmt.Println(string(b)) // output manifest to stdout
|
||||
},
|
||||
}
|
||||
|
||||
185
cmd/kube-vip.go
185
cmd/kube-vip.go
@@ -9,14 +9,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/manager"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
@@ -28,6 +27,12 @@ var inCluster bool
|
||||
// ConfigMap name within a Kubernetes cluster
|
||||
var configMap string
|
||||
|
||||
// Configure the level of logging
|
||||
var logLevel uint32
|
||||
|
||||
// Provider Config
|
||||
var providerConfig string
|
||||
|
||||
// Points to a kubernetes configuration file
|
||||
var kubeConfigPath string
|
||||
|
||||
@@ -53,19 +58,20 @@ func init() {
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc..")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
|
||||
|
||||
// VIP flags
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address. Default to 32 for IPv4 and 128 for IPv6") // todo: deprecate
|
||||
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
|
||||
kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "Port for the VIP")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
|
||||
|
||||
// LoadBalancer flags
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
|
||||
kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
|
||||
@@ -78,6 +84,13 @@ func init() {
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
|
||||
|
||||
// Equinix Metal flags
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
|
||||
|
||||
// BGP flags
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
|
||||
@@ -93,23 +106,18 @@ func init() {
|
||||
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
|
||||
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
|
||||
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
|
||||
|
||||
// Namespace for kube-vip
|
||||
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
|
||||
|
||||
// Manage logging
|
||||
kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
|
||||
kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
|
||||
|
||||
// Service flags
|
||||
kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
|
||||
|
||||
// Routing Table flags
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", 0, "The type of route that will be added to the routing table")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
|
||||
|
||||
@@ -128,9 +136,8 @@ func init() {
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
|
||||
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
|
||||
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
|
||||
|
||||
// Prometheus HTTP Server
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
|
||||
@@ -142,6 +149,7 @@ func init() {
|
||||
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
|
||||
|
||||
// Kubernetes client specific flags
|
||||
|
||||
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
|
||||
|
||||
kubeVipCmd.AddCommand(kubeKubeadm)
|
||||
@@ -150,9 +158,6 @@ func init() {
|
||||
kubeVipCmd.AddCommand(kubeVipSample)
|
||||
kubeVipCmd.AddCommand(kubeVipService)
|
||||
kubeVipCmd.AddCommand(kubeVipVersion)
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
|
||||
}
|
||||
|
||||
// Execute - starts the command parsing process
|
||||
@@ -166,7 +171,7 @@ func Execute() {
|
||||
var kubeVipVersion = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Version and Release information about the Kubernetes Virtual IP Server",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("Kube-VIP Release Information\n")
|
||||
fmt.Printf("Version: %s\n", Release.Version)
|
||||
fmt.Printf("Build: %s\n", Release.Build)
|
||||
@@ -176,7 +181,7 @@ var kubeVipVersion = &cobra.Command{
|
||||
var kubeVipSample = &cobra.Command{
|
||||
Use: "sample",
|
||||
Short: "Generate a Sample configuration",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
@@ -184,26 +189,18 @@ var kubeVipSample = &cobra.Command{
|
||||
var kubeVipService = &cobra.Command{
|
||||
Use: "service",
|
||||
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(logLevel))
|
||||
|
||||
// parse environment variables, these will overwrite anything loaded or flags
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing env", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// Change RTN_UNSPEC to default type
|
||||
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
|
||||
initConfig.RoutingProtocol = unix.RTN_UNICAST
|
||||
}
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
|
||||
|
||||
if err := initConfig.CheckInterface(); err != nil {
|
||||
log.Error("checking interface", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// User Environment variables as an option to make manifest clearer
|
||||
@@ -213,26 +210,23 @@ var kubeVipService = &cobra.Command{
|
||||
}
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("generating CIDR", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Define the new service manager
|
||||
mgr, err := manager.New(configMap, &initConfig)
|
||||
if err != nil {
|
||||
log.Error("new manager", "err", err)
|
||||
return
|
||||
log.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
// Start the service manager, this will watch the config Map and construct kube-vip services for it
|
||||
err = mgr.Start()
|
||||
if err != nil {
|
||||
log.Error("manager start", "err", err)
|
||||
return
|
||||
log.Fatalf("%v", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -240,33 +234,27 @@ var kubeVipService = &cobra.Command{
|
||||
var kubeVipManager = &cobra.Command{
|
||||
Use: "manager",
|
||||
Short: "Start the kube-vip manager",
|
||||
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// parse environment variables, these will overwrite anything loaded or flags
|
||||
err := kubevip.ParseEnvironment(&initConfig)
|
||||
if err != nil {
|
||||
log.Error("parsing environment", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// Change RTN_UNSPEC to default type
|
||||
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
|
||||
initConfig.RoutingProtocol = unix.RTN_UNICAST
|
||||
}
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
|
||||
|
||||
// Ensure there is an address to generate the CIDR from
|
||||
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
|
||||
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
|
||||
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
|
||||
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
|
||||
if err != nil {
|
||||
log.Error("No interface is specified for kube-vip to bind to")
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set the logging level for all subsequent functions
|
||||
log.SetLevel(log.Level(initConfig.Logging))
|
||||
|
||||
// Welome messages
|
||||
log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
|
||||
log.Infof("Starting kube-vip.io [%s]", Release.Version)
|
||||
log.Debugf("Build kube-vip.io [%s]", Release.Build)
|
||||
|
||||
// start prometheus server
|
||||
if initConfig.PrometheusHTTPServer != "" {
|
||||
@@ -294,22 +282,19 @@ var kubeVipManager = &cobra.Command{
|
||||
}
|
||||
|
||||
// Provide configuration to output/logging
|
||||
log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
|
||||
log.Infof("namespace [%s], Mode: [%s], Features(s): Control Plane:[%t], Services:[%t]", initConfig.Namespace, mode, initConfig.EnableControlPlane, initConfig.EnableServices)
|
||||
|
||||
// End if nothing is enabled
|
||||
if !initConfig.EnableServices && !initConfig.EnableControlPlane {
|
||||
log.Error("no features are enabled")
|
||||
return
|
||||
log.Fatalln("no features are enabled")
|
||||
}
|
||||
|
||||
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
|
||||
log.Error("auto subnet discovery cannot be used outside ARP mode")
|
||||
return
|
||||
if !initConfig.EnableARP && strings.Contains(initConfig.VIPCIDR, kubevip.Auto) {
|
||||
log.Fatalln("auto subnet discovery cannot be used outside ARP mode")
|
||||
}
|
||||
|
||||
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
|
||||
log.Error("auto subnet discovery cannot be used if VIP address was provided")
|
||||
return
|
||||
if strings.Contains(initConfig.VIPCIDR, kubevip.Auto) && initConfig.Address != "" {
|
||||
log.Fatalln("auto subnet discovery cannot be used if VIP address was provided")
|
||||
}
|
||||
|
||||
// If we're using wireguard then all traffic goes through the wg0 interface
|
||||
@@ -319,55 +304,48 @@ var kubeVipManager = &cobra.Command{
|
||||
initConfig.Interface = "wg0"
|
||||
}
|
||||
|
||||
log.Info("configuring Wireguard networking")
|
||||
log.Infof("configuring Wireguard networking")
|
||||
l, err := netlink.LinkByName(initConfig.Interface)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Link not found") {
|
||||
log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
|
||||
log.Warnf("interface \"%s\" doesn't exist, attempting to create wireguard interface", initConfig.Interface)
|
||||
err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
|
||||
if err != nil {
|
||||
log.Error("adding link", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
l, err = netlink.LinkByName(initConfig.Interface)
|
||||
if err != nil {
|
||||
log.Error("finding link", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
err = netlink.LinkSetUp(l)
|
||||
if err != nil {
|
||||
log.Error("setting link UP", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
} else { // if we're not using Wireguard then we'll need to use an actual interface
|
||||
// Check if the interface needs auto-detecting
|
||||
if initConfig.Interface == "" {
|
||||
log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
|
||||
log.Infof("No interface is specified for VIP in config, auto-detecting default Interface")
|
||||
defaultIF, err := vip.GetDefaultGatewayInterface()
|
||||
if err != nil {
|
||||
_ = cmd.Help()
|
||||
log.Error("detecting interface", "err", err)
|
||||
return
|
||||
log.Fatalf("unable to detect default interface -> [%v]", err)
|
||||
}
|
||||
initConfig.Interface = defaultIF.Name
|
||||
log.Info("kube-vip bind", "interface", initConfig.Interface)
|
||||
log.Infof("kube-vip will bind to interface [%s]", initConfig.Interface)
|
||||
|
||||
go func() {
|
||||
if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
|
||||
|
||||
log.Error("interface monitor", "err", err)
|
||||
return
|
||||
log.Fatalf("crash: %s", err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
// Perform a check on the state of the interface
|
||||
// Perform a check on th state of the interface
|
||||
if err := initConfig.CheckInterface(); err != nil {
|
||||
log.Error("checking interface", "err", err)
|
||||
return
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// User Environment variables as an option to make manifest clearer
|
||||
@@ -376,11 +354,22 @@ var kubeVipManager = &cobra.Command{
|
||||
configMap = envConfigMap
|
||||
}
|
||||
|
||||
// If Equinix Metal is enabled and there is a provider configuration passed
|
||||
if initConfig.EnableMetal {
|
||||
if providerConfig != "" {
|
||||
providerAPI, providerProject, err := equinixmetal.GetPacketConfig(providerConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("%v", err)
|
||||
}
|
||||
initConfig.MetalAPIKey = providerAPI
|
||||
initConfig.MetalProject = providerProject
|
||||
}
|
||||
}
|
||||
|
||||
// Define the new service manager
|
||||
mgr, err := manager.New(configMap, &initConfig)
|
||||
if err != nil {
|
||||
log.Error("new manager", "err", err)
|
||||
return
|
||||
log.Fatalf("configuring new Manager error -> %v", err)
|
||||
}
|
||||
|
||||
prometheus.MustRegister(mgr.PrometheusCollector()...)
|
||||
@@ -388,8 +377,7 @@ var kubeVipManager = &cobra.Command{
|
||||
// Start the service manager, this will watch the config Map and construct kube-vip services for it
|
||||
err = mgr.Start()
|
||||
if err != nil {
|
||||
log.Error("start manager", "err", err)
|
||||
return
|
||||
log.Fatalf("starting new Manager error -> %v", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -404,7 +392,7 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
|
||||
var err error
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`<html>
|
||||
<head><title>kube-vip</title></head>
|
||||
<body>
|
||||
@@ -422,16 +410,15 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
|
||||
|
||||
go func() {
|
||||
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Error("prometheus HTTP server", "err", err)
|
||||
return
|
||||
log.Fatalf("listen:%+s\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Info("prometheus HTTP server started")
|
||||
log.Printf("prometheus HTTP server started")
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
log.Info("prometheus HTTP server stopped")
|
||||
log.Printf("prometheus HTTP server stopped")
|
||||
|
||||
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer func() {
|
||||
@@ -439,8 +426,7 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
|
||||
}()
|
||||
|
||||
if err = srv.Shutdown(ctxShutDown); err != nil {
|
||||
log.Error("shutting down prometheus HTTP server", "err", err)
|
||||
return
|
||||
log.Fatalf("server Shutdown Failed:%+s", err)
|
||||
}
|
||||
|
||||
if err == http.ErrServerClosed {
|
||||
@@ -454,12 +440,9 @@ func GenerateCidrRange(address string) (string, error) {
|
||||
addresses := strings.Split(address, ",")
|
||||
for _, a := range addresses {
|
||||
ip := net.ParseIP(a)
|
||||
|
||||
if ip == nil {
|
||||
ips, err := net.LookupIP(a)
|
||||
if len(ips) == 0 || err != nil {
|
||||
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
|
||||
}
|
||||
ip = ips[0]
|
||||
return "", fmt.Errorf("invalid IP address: %s from [%s]", a, address)
|
||||
}
|
||||
|
||||
if ip.To4() != nil {
|
||||
|
||||
@@ -60,7 +60,7 @@ func main() {
|
||||
fmt.Println("error: ", err)
|
||||
}
|
||||
|
||||
ServerConn.WriteTo(buf[0:n], addr)
|
||||
ServerConn.WriteTo(buf[0:n])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
220
go.mod
220
go.mod
@@ -1,165 +1,133 @@
|
||||
module github.com/kube-vip/kube-vip
|
||||
|
||||
go 1.24.4
|
||||
go 1.21
|
||||
|
||||
toolchain go1.21.3
|
||||
|
||||
require (
|
||||
github.com/cloudflare/ipvs v0.11.0
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/docker/docker v28.3.3+incompatible
|
||||
github.com/cloudflare/ipvs v0.10.2
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/florianl/go-conntrack v0.4.0
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/google/nftables v0.3.0
|
||||
github.com/gookit/slog v0.5.8
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/huin/goupnp v1.3.0
|
||||
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
|
||||
github.com/insomniacslk/dhcp v0.0.0-20230731140434-0f9eb93a696c
|
||||
github.com/jpillora/backoff v1.0.0
|
||||
github.com/mdlayher/ndp v1.1.0
|
||||
github.com/onsi/ginkgo/v2 v2.23.4
|
||||
github.com/onsi/gomega v1.37.0
|
||||
github.com/osrg/gobgp/v3 v3.37.0
|
||||
github.com/mdlayher/ndp v1.0.1
|
||||
github.com/onsi/ginkgo/v2 v2.19.0
|
||||
github.com/onsi/gomega v1.33.1
|
||||
github.com/osrg/gobgp/v3 v3.27.0
|
||||
github.com/packethost/packngo v0.31.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.22.0
|
||||
github.com/prometheus/client_golang v1.19.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/vishvananda/netlink v1.3.1
|
||||
go.etcd.io/etcd/api/v3 v3.6.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.1
|
||||
go.etcd.io/etcd/client/v3 v3.6.1
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/vishvananda/netlink v1.2.1-beta.2
|
||||
go.etcd.io/etcd/api/v3 v3.5.13
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.13
|
||||
go.etcd.io/etcd/client/v3 v3.5.13
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
|
||||
golang.org/x/sync v0.15.0
|
||||
golang.org/x/sys v0.33.0
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
|
||||
google.golang.org/grpc v1.73.0
|
||||
google.golang.org/protobuf v1.36.6
|
||||
k8s.io/api v0.33.3
|
||||
k8s.io/apimachinery v0.33.3
|
||||
k8s.io/client-go v0.33.3
|
||||
golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8
|
||||
golang.org/x/sync v0.7.0
|
||||
golang.org/x/sys v0.20.0
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
k8s.io/api v0.29.1
|
||||
k8s.io/apimachinery v0.29.3
|
||||
k8s.io/client-go v0.29.1
|
||||
k8s.io/klog/v2 v2.130.1
|
||||
sigs.k8s.io/kind v0.29.0
|
||||
sigs.k8s.io/kind v0.22.0
|
||||
sigs.k8s.io/yaml v1.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
al.essio.dev/pkg/shellescape v1.5.1 // indirect
|
||||
github.com/BurntSushi/toml v1.4.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/BurntSushi/toml v1.3.2 // indirect
|
||||
github.com/alessio/shellescape v1.4.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.4 // indirect
|
||||
github.com/docker/go-connections v0.5.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/eapache/channels v1.1.0 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
|
||||
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.8.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.21.0 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
|
||||
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.6 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
github.com/go-openapi/swag v0.22.3 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/gnostic-models v0.6.9 // indirect
|
||||
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/gookit/goutil v0.6.18 // indirect
|
||||
github.com/gookit/gsr v0.1.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
|
||||
github.com/google/gnostic-models v0.6.8 // indirect
|
||||
github.com/google/gofuzz v1.2.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
|
||||
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/josharian/native v1.1.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/k-sone/critbitgo v1.4.0 // indirect
|
||||
github.com/magiconair/properties v1.8.9 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mdlayher/genetlink v1.3.2 // indirect
|
||||
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
|
||||
github.com/mdlayher/netlink v1.7.2 // indirect
|
||||
github.com/mdlayher/packet v1.1.2 // indirect
|
||||
github.com/mdlayher/socket v0.5.1 // indirect
|
||||
github.com/mdlayher/socket v0.4.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/sys/atomicwriter v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.62.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/sagikazarmark/locafero v0.6.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/spf13/viper v1.19.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/pelletier/go-toml v1.9.4 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.5.0 // indirect
|
||||
github.com/prometheus/common v0.48.0 // indirect
|
||||
github.com/prometheus/procfs v0.12.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/spf13/afero v1.9.5 // indirect
|
||||
github.com/spf13/cast v1.5.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.16.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/tj/go-spin v1.1.0 // indirect
|
||||
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/vishvananda/netns v0.0.5 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xlab/c-for-go v1.3.0 // indirect
|
||||
github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect
|
||||
github.com/vishvananda/netns v0.0.4 // indirect
|
||||
github.com/xlab/c-for-go v0.0.0-20230906092656-a1822f0a09c1 // indirect
|
||||
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
|
||||
go.opentelemetry.io/otel v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.37.0 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/crypto v0.39.0 // indirect
|
||||
golang.org/x/mod v0.25.0 // indirect
|
||||
golang.org/x/net v0.41.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/term v0.32.0 // indirect
|
||||
golang.org/x/text v0.26.0 // indirect
|
||||
golang.org/x/time v0.9.0 // indirect
|
||||
golang.org/x/tools v0.33.0 // indirect
|
||||
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
|
||||
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/crypto v0.23.0 // indirect
|
||||
golang.org/x/mod v0.17.0 // indirect
|
||||
golang.org/x/net v0.25.0 // indirect
|
||||
golang.org/x/oauth2 v0.16.0 // indirect
|
||||
golang.org/x/term v0.20.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.21.0 // indirect
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
|
||||
google.golang.org/grpc v1.59.0 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
|
||||
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
|
||||
modernc.org/cc/v4 v4.24.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/opt v0.1.4 // indirect
|
||||
modernc.org/sortutil v1.2.1 // indirect
|
||||
modernc.org/strutil v1.2.1 // indirect
|
||||
modernc.org/token v1.1.0 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
|
||||
sigs.k8s.io/randfill v1.0.0 // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
|
||||
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
|
||||
modernc.org/cc/v4 v4.1.0 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/strutil v1.1.3 // indirect
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
|
||||
)
|
||||
|
||||
149
pkg/arp/arp.go
149
pkg/arp/arp.go
@@ -1,149 +0,0 @@
|
||||
package arp
|
||||
|
||||
import (
|
||||
"context"
|
||||
log "log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
instances map[string]*Instance
|
||||
config *kubevip.Config
|
||||
}
|
||||
|
||||
type Instance struct {
|
||||
network vip.Network
|
||||
ndp *vip.NdpResponder
|
||||
mu sync.Mutex
|
||||
counter int
|
||||
}
|
||||
|
||||
func NewManager(config *kubevip.Config) *Manager {
|
||||
return &Manager{
|
||||
instances: make(map[string]*Instance),
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
|
||||
return &Instance{
|
||||
ndp: ndp,
|
||||
network: network,
|
||||
counter: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Instance) Name() string {
|
||||
return i.network.ARPName()
|
||||
}
|
||||
|
||||
func (m *Manager) Insert(instance *Instance) {
|
||||
i, ok := m.instances[instance.Name()]
|
||||
if !ok {
|
||||
log.Info("inserting ARP/NDP instance", "name", instance.Name())
|
||||
m.instances[instance.Name()] = instance
|
||||
} else {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
i.counter++
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Remove(instance *Instance) {
|
||||
if i, ok := m.instances[instance.Name()]; ok {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
if i.counter > 1 {
|
||||
i.counter--
|
||||
} else {
|
||||
log.Info("removing ARP/NDP instance", "name", instance.Name())
|
||||
delete(m.instances, instance.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Count(name string) int {
|
||||
if i, ok := m.instances[name]; ok {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
return i.counter
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *Manager) StartAdvertisement(ctx context.Context) {
|
||||
log.Info("Starting ARP/NDP advertisement")
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done(): // if cancel() execute
|
||||
return
|
||||
default:
|
||||
for _, instance := range m.instances {
|
||||
if instance.counter > 0 {
|
||||
ensureIPAndSendGratuitous(instance)
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.config.ArpBroadcastRate < 500 {
|
||||
log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
|
||||
m.config.ArpBroadcastRate = 3000
|
||||
}
|
||||
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
|
||||
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
|
||||
// and in a dadfailed state.
|
||||
func ensureIPAndSendGratuitous(instance *Instance) {
|
||||
iface := instance.network.Interface()
|
||||
ipString := instance.network.IP()
|
||||
|
||||
// Check if IP is dadfailed
|
||||
if instance.network.IsDADFAIL() {
|
||||
log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
|
||||
deleted, err := instance.network.DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
|
||||
// if _, err := instance.network.AddIP(false); err != nil {
|
||||
// log.Error("failed to recreate address", "IP", ipString, "interface", iface)
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the address exists on the interface before attempting to ARP
|
||||
// if instance.network.HasEndpoints() {
|
||||
if added, err := instance.network.AddIP(true); err != nil {
|
||||
log.Warn(err.Error())
|
||||
} else if added {
|
||||
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
|
||||
}
|
||||
// }
|
||||
|
||||
if vip.IsIPv6(ipString) {
|
||||
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
|
||||
if instance.ndp == nil {
|
||||
log.Error("NDP responder was not created")
|
||||
} else {
|
||||
err := instance.ndp.SendGratuitous(ipString)
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
|
||||
err := vip.ARPSendGratuitous(ipString, iface)
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -4,19 +4,17 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/k8s"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
type Entry struct {
|
||||
Addr string
|
||||
Port uint16
|
||||
IsLocal bool
|
||||
Addr string
|
||||
Port int
|
||||
}
|
||||
|
||||
type Map map[Entry]bool
|
||||
@@ -40,27 +38,28 @@ func (e *Entry) Check() bool {
|
||||
switch {
|
||||
case utils.FileExists(adminConfigPath):
|
||||
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
|
||||
// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
|
||||
if err != nil {
|
||||
log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
|
||||
log.Errorf("could not create k8s REST config for external file: %q: %v", adminConfigPath, err)
|
||||
return false
|
||||
}
|
||||
default:
|
||||
config, err = k8s.NewRestConfig("", true, k8sAddr)
|
||||
if err != nil {
|
||||
log.Error("create k8s REST config", "err", err)
|
||||
log.Errorf("could not create k8s REST config %v", err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
client, err = k8s.NewClientset(config)
|
||||
if err != nil {
|
||||
log.Error("create k8s client", "err", err)
|
||||
log.Errorf("failed to create k8s client: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = client.DiscoveryClient.ServerVersion()
|
||||
if err != nil {
|
||||
log.Error("discover k8s version", "err", err)
|
||||
log.Errorf("failed check k8s server version: %s", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
||||
210
pkg/bgp/peers.go
210
pkg/bgp/peers.go
@@ -7,11 +7,9 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
//nolint
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/golang/protobuf/ptypes" //nolint
|
||||
"github.com/golang/protobuf/ptypes/any"
|
||||
api "github.com/osrg/gobgp/v3/api"
|
||||
"github.com/osrg/gobgp/v3/pkg/server"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
// AddPeer will add peers to the BGP configuration
|
||||
@@ -44,94 +42,36 @@ func (b *Server) AddPeer(peer Peer) (err error) {
|
||||
},
|
||||
}
|
||||
|
||||
if b.c.MpbgpNexthop != "" {
|
||||
p.AfiSafis = []*api.AfiSafi{
|
||||
{
|
||||
Config: &api.AfiSafiConfig{
|
||||
Family: &api.Family{
|
||||
Afi: api.Family_AFI_IP,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Config: &api.AfiSafiConfig{
|
||||
Family: &api.Family{
|
||||
Afi: api.Family_AFI_IP6,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
peer.setMpbgpOptions(b.c)
|
||||
|
||||
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
|
||||
}
|
||||
|
||||
mask := "128"
|
||||
address := ipv4Address
|
||||
family := api.Family_AFI_IP
|
||||
if vip.IsIPv4(p.Conf.NeighborAddress) {
|
||||
mask = "32"
|
||||
address = ipv6Address
|
||||
family = api.Family_AFI_IP6
|
||||
}
|
||||
|
||||
err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
|
||||
DefinedSet: &api.DefinedSet{
|
||||
DefinedType: api.DefinedType_NEIGHBOR,
|
||||
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
|
||||
List: []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add defined set: %v", err)
|
||||
}
|
||||
|
||||
if address != "" {
|
||||
if err := insertPolicy(b.s, address, p, family); err != nil {
|
||||
return fmt.Errorf("failed to add policy: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if b.c.SourceIP != "" {
|
||||
p.Transport.LocalAddress = b.c.SourceIP
|
||||
}
|
||||
|
||||
if b.c.SourceIF != "" {
|
||||
p.Transport.BindInterface = b.c.SourceIF
|
||||
}
|
||||
if b.c.SourceIP != "" {
|
||||
p.Transport.LocalAddress = b.c.SourceIP
|
||||
}
|
||||
|
||||
if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
|
||||
return fmt.Errorf("failed to add peer: %v", err)
|
||||
if b.c.SourceIF != "" {
|
||||
p.Transport.BindInterface = b.c.SourceIF
|
||||
}
|
||||
|
||||
return nil
|
||||
return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
|
||||
Peer: p,
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Server) getPath(ip net.IP) (path *api.Path) {
|
||||
isV6 := ip.To4() == nil
|
||||
|
||||
//nolint
|
||||
originAttr, _ := anypb.New(&api.OriginAttribute{
|
||||
originAttr, _ := ptypes.MarshalAny(&api.OriginAttribute{
|
||||
Origin: 0,
|
||||
})
|
||||
|
||||
if !isV6 {
|
||||
//nolint
|
||||
nlri, _ := anypb.New(&api.IPAddressPrefix{
|
||||
nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
|
||||
Prefix: ip.String(),
|
||||
PrefixLen: 32,
|
||||
})
|
||||
|
||||
//nolint
|
||||
nhAttr, _ := anypb.New(&api.NextHopAttribute{
|
||||
nhAttr, _ := ptypes.MarshalAny(&api.NextHopAttribute{
|
||||
NextHop: "0.0.0.0", // gobgp will fill this
|
||||
})
|
||||
|
||||
@@ -141,11 +81,11 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
Nlri: nlri,
|
||||
Pattrs: []*anypb.Any{originAttr, nhAttr},
|
||||
Pattrs: []*any.Any{originAttr, nhAttr},
|
||||
}
|
||||
} else {
|
||||
//nolint
|
||||
nlri, _ := anypb.New(&api.IPAddressPrefix{
|
||||
nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
|
||||
Prefix: ip.String(),
|
||||
PrefixLen: 128,
|
||||
})
|
||||
@@ -156,16 +96,16 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
|
||||
}
|
||||
|
||||
//nolint
|
||||
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
|
||||
mpAttr, _ := ptypes.MarshalAny(&api.MpReachNLRIAttribute{
|
||||
Family: v6Family,
|
||||
NextHops: []string{"::"}, // gobgp will fill this
|
||||
Nlris: []*anypb.Any{nlri},
|
||||
Nlris: []*any.Any{nlri},
|
||||
})
|
||||
|
||||
path = &api.Path{
|
||||
Family: v6Family,
|
||||
Nlri: nlri,
|
||||
Pattrs: []*anypb.Any{originAttr, mpAttr},
|
||||
Pattrs: []*any.Any{originAttr, mpAttr},
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -175,13 +115,11 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
|
||||
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
|
||||
peers := strings.Split(config, ",")
|
||||
if len(peers) == 0 {
|
||||
return nil, fmt.Errorf("no BGP Peer configurations found")
|
||||
return nil, fmt.Errorf("No BGP Peer configurations found")
|
||||
}
|
||||
|
||||
for x := range peers {
|
||||
peerStr := peers[x]
|
||||
config := strings.Split(peerStr, "/")
|
||||
peerStr = config[0]
|
||||
if peerStr == "" {
|
||||
continue
|
||||
}
|
||||
@@ -224,118 +162,14 @@ func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
var port uint64
|
||||
if len(peer) >= 5 {
|
||||
port, err = strconv.ParseUint(peer[4], 10, 16)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
|
||||
}
|
||||
} else {
|
||||
port = 179
|
||||
}
|
||||
|
||||
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
|
||||
|
||||
if len(config) > 1 {
|
||||
configData := strings.Split(config[1], ";")
|
||||
for _, cfg := range configData {
|
||||
c := strings.Split(cfg, "=")
|
||||
switch c[0] {
|
||||
case "mpbgp_nexthop":
|
||||
mpbgpNexthop = c[1]
|
||||
case "mpbgp_ipv4":
|
||||
mpbgpIPv4 = c[1]
|
||||
case "mpbgp_ipv6":
|
||||
mpbgpIPv6 = c[1]
|
||||
default:
|
||||
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
peerConfig := Peer{
|
||||
Address: address,
|
||||
AS: uint32(ASNumber),
|
||||
Port: uint16(port),
|
||||
Password: password,
|
||||
MultiHop: multiHop,
|
||||
MpbgpNexthop: mpbgpNexthop,
|
||||
MpbgpIPv4: mpbgpIPv4,
|
||||
MpbgpIPv6: mpbgpIPv6,
|
||||
Address: address,
|
||||
AS: uint32(ASNumber),
|
||||
Password: password,
|
||||
MultiHop: multiHop,
|
||||
}
|
||||
|
||||
bgpPeers = append(bgpPeers, peerConfig)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
|
||||
familyType := "v4"
|
||||
if family == api.Family_AFI_IP6 {
|
||||
familyType = "v6"
|
||||
}
|
||||
|
||||
setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
|
||||
policyName := fmt.Sprintf("%s-%s", setName, familyType)
|
||||
|
||||
policy := &api.Policy{
|
||||
Name: policyName,
|
||||
Statements: []*api.Statement{
|
||||
{
|
||||
Conditions: &api.Conditions{
|
||||
AfiSafiIn: []*api.Family{
|
||||
{
|
||||
Afi: family,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
},
|
||||
NeighborSet: &api.MatchSet{
|
||||
Type: api.MatchSet_ANY,
|
||||
Name: setName,
|
||||
},
|
||||
},
|
||||
Actions: &api.Actions{
|
||||
RouteAction: api.RouteAction_ACCEPT,
|
||||
Nexthop: &api.NexthopAction{
|
||||
Address: address,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Conditions: &api.Conditions{
|
||||
NeighborSet: &api.MatchSet{
|
||||
Type: api.MatchSet_ANY,
|
||||
Name: setName,
|
||||
},
|
||||
},
|
||||
Actions: &api.Actions{
|
||||
RouteAction: api.RouteAction_ACCEPT,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
|
||||
Policy: policy,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add policy: %w", err)
|
||||
}
|
||||
|
||||
err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
|
||||
Assignment: &api.PolicyAssignment{
|
||||
Name: "global",
|
||||
Direction: api.PolicyDirection_EXPORT,
|
||||
Policies: []*api.Policy{
|
||||
{
|
||||
Name: policy.Name,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add policy assignment: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
package bgp
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBGPPeerConfig(t *testing.T) {
|
||||
type args struct {
|
||||
config string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantBgpPeers []Peer
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "IPv4, default port",
|
||||
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
|
||||
wantBgpPeers: []Peer{
|
||||
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
|
||||
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IPv4, different port",
|
||||
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
|
||||
wantBgpPeers: []Peer{
|
||||
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
|
||||
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IPv6, multi-protocol",
|
||||
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
|
||||
wantBgpPeers: []Peer{
|
||||
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
|
||||
t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,49 +5,35 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
api "github.com/osrg/gobgp/v3/api"
|
||||
gobgp "github.com/osrg/gobgp/v3/pkg/server"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// NewBGPServer takes a configuration and returns a running BGP server instance
|
||||
func NewBGPServer(c *Config) (b *Server, err error) {
|
||||
func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
|
||||
if c.AS == 0 {
|
||||
return nil, fmt.Errorf("you need to provide AS")
|
||||
return nil, fmt.Errorf("You need to provide AS")
|
||||
}
|
||||
|
||||
if c.SourceIP != "" && c.SourceIF != "" {
|
||||
return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
|
||||
return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
|
||||
}
|
||||
|
||||
if len(c.Peers) == 0 {
|
||||
return nil, fmt.Errorf("you need to provide at least one peer")
|
||||
return nil, fmt.Errorf("You need to provide at least one peer")
|
||||
}
|
||||
|
||||
b = &Server{
|
||||
s: gobgp.NewBgpServer(),
|
||||
c: c,
|
||||
|
||||
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: "kube_vip",
|
||||
Subsystem: "manager",
|
||||
Name: "bgp_session_info",
|
||||
Help: "Display state of session by setting metric for label value with current state to 1",
|
||||
}, []string{"state", "peer"}),
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Start starts the BGP server
|
||||
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
|
||||
go b.s.Serve()
|
||||
|
||||
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
|
||||
Global: &api.Global{
|
||||
Asn: b.c.AS,
|
||||
RouterId: b.c.RouterID,
|
||||
Asn: c.AS,
|
||||
RouterId: c.RouterID,
|
||||
ListenPort: -1,
|
||||
},
|
||||
}); err != nil {
|
||||
@@ -56,7 +42,7 @@ func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_Peer
|
||||
|
||||
if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
|
||||
if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
|
||||
log.Info("[BGP]", "peer", p.String())
|
||||
log.Infof("[BGP] %s", p.String())
|
||||
if peerStateChangeCallback != nil {
|
||||
peerStateChangeCallback(p)
|
||||
}
|
||||
@@ -65,23 +51,12 @@ func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_Peer
|
||||
return
|
||||
}
|
||||
|
||||
for _, p := range b.c.Peers {
|
||||
for _, p := range c.Peers {
|
||||
if err = b.AddPeer(p); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if b.c.Zebra.Enabled {
|
||||
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
|
||||
Url: b.c.Zebra.URL,
|
||||
Version: b.c.Zebra.Version,
|
||||
SoftwareName: b.c.Zebra.SoftwareName,
|
||||
}); err != nil {
|
||||
log.Error(err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
133
pkg/bgp/types.go
133
pkg/bgp/types.go
@@ -1,145 +1,30 @@
|
||||
package bgp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
api "github.com/osrg/gobgp/v3/api"
|
||||
gobgp "github.com/osrg/gobgp/v3/pkg/server"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
import gobgp "github.com/osrg/gobgp/v3/pkg/server"
|
||||
|
||||
// Peer defines a BGP Peer
|
||||
type Peer struct {
|
||||
Address string
|
||||
Port uint16
|
||||
AS uint32
|
||||
Password string
|
||||
MultiHop bool
|
||||
MpbgpNexthop string
|
||||
MpbgpIPv4 string
|
||||
MpbgpIPv6 string
|
||||
}
|
||||
|
||||
func (p *Peer) setMpbgpOptions(server *Config) {
|
||||
if p.MpbgpNexthop == "" {
|
||||
p.MpbgpNexthop = server.MpbgpNexthop
|
||||
}
|
||||
|
||||
if p.MpbgpIPv4 == "" {
|
||||
p.MpbgpIPv4 = server.MpbgpIPv4
|
||||
}
|
||||
|
||||
if p.MpbgpIPv6 == "" {
|
||||
p.MpbgpIPv6 = server.MpbgpIPv6
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
|
||||
var ipv4Address, ipv6Address string
|
||||
switch p.MpbgpNexthop {
|
||||
case "fixed":
|
||||
ap.Transport.LocalAddress = server.SourceIP
|
||||
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
|
||||
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
|
||||
p.MpbgpIPv4, p.MpbgpIPv6)
|
||||
}
|
||||
|
||||
if p.MpbgpIPv4 != "" {
|
||||
if net.ParseIP(p.MpbgpIPv4) == nil {
|
||||
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
|
||||
}
|
||||
}
|
||||
if p.MpbgpIPv6 != "" {
|
||||
if net.ParseIP(p.MpbgpIPv6) == nil {
|
||||
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
|
||||
}
|
||||
}
|
||||
|
||||
ipv4Address = p.MpbgpIPv4
|
||||
ipv6Address = p.MpbgpIPv6
|
||||
case "auto_sourceip":
|
||||
ap.Transport.LocalAddress = server.SourceIP
|
||||
|
||||
// Resolve the local interface by SourceIP
|
||||
iface, err := vip.GetInterfaceByIP(server.SourceIP)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
|
||||
}
|
||||
|
||||
if vip.IsIPv4(server.SourceIP) {
|
||||
// Get the non link-local IPv6 address on that interface
|
||||
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
|
||||
}
|
||||
} else {
|
||||
// Get the non link-local IPv4 address on that interface
|
||||
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
|
||||
}
|
||||
}
|
||||
case "auto_sourceif":
|
||||
ap.Transport.BindInterface = server.SourceIF
|
||||
|
||||
iface, err := netlink.LinkByName(server.SourceIF)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
|
||||
}
|
||||
|
||||
// Get the non link-local IPv4 address on that interface
|
||||
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
|
||||
}
|
||||
|
||||
// Get the non link-local IPv6 address on that interface
|
||||
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
|
||||
}
|
||||
default:
|
||||
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
|
||||
}
|
||||
|
||||
return ipv4Address, ipv6Address, nil
|
||||
Address string
|
||||
AS uint32
|
||||
Password string
|
||||
MultiHop bool
|
||||
}
|
||||
|
||||
// Config defines the BGP server configuration
|
||||
type Config struct {
|
||||
AS uint32
|
||||
RouterID string
|
||||
SourceIP string
|
||||
SourceIF string
|
||||
MpbgpNexthop string
|
||||
MpbgpIPv4 string
|
||||
MpbgpIPv6 string
|
||||
AS uint32
|
||||
RouterID string
|
||||
SourceIP string
|
||||
SourceIF string
|
||||
|
||||
HoldTime uint64
|
||||
KeepaliveInterval uint64
|
||||
|
||||
Peers []Peer
|
||||
|
||||
Zebra ZebraConfig
|
||||
}
|
||||
|
||||
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
|
||||
type ZebraConfig struct {
|
||||
Enabled bool
|
||||
URL string
|
||||
Version uint32
|
||||
SoftwareName string
|
||||
}
|
||||
|
||||
// Server manages a server object
|
||||
type Server struct {
|
||||
s *gobgp.BgpServer
|
||||
c *Config
|
||||
|
||||
// This is a prometheus gauge indicating the state of the sessions.
|
||||
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
|
||||
BGPSessionInfoGauge *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
@@ -3,11 +3,9 @@ package cluster
|
||||
import (
|
||||
"sync"
|
||||
|
||||
log "log/slog"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
@@ -17,17 +15,16 @@ type Cluster struct {
|
||||
completed chan bool
|
||||
once sync.Once
|
||||
Network []vip.Network
|
||||
arpMgr *arp.Manager
|
||||
}
|
||||
|
||||
// InitCluster - Will attempt to initialise all of the required settings for the cluster
|
||||
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
|
||||
func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
|
||||
var networks []vip.Network
|
||||
var err error
|
||||
|
||||
if !disableVIP {
|
||||
// Start the Virtual IP Networking configuration
|
||||
networks, err = startNetworking(c, intfMgr)
|
||||
networks, err = startNetworking(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -35,15 +32,14 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
|
||||
// Initialise the Cluster structure
|
||||
newCluster := &Cluster{
|
||||
Network: networks,
|
||||
arpMgr: arpMgr,
|
||||
}
|
||||
|
||||
log.Debug("service security", "enabled", c.EnableServiceSecurity)
|
||||
log.Debugf("init enable service security: %t", c.EnableServiceSecurity)
|
||||
|
||||
return newCluster, nil
|
||||
}
|
||||
|
||||
func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
|
||||
func startNetworking(c *kubevip.Config) ([]vip.Network, error) {
|
||||
address := c.VIP
|
||||
|
||||
if c.Address != "" {
|
||||
@@ -54,9 +50,7 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
|
||||
|
||||
networks := []vip.Network{}
|
||||
for _, addr := range addresses {
|
||||
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
|
||||
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
|
||||
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
|
||||
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -11,13 +11,15 @@ import (
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
|
||||
"github.com/kube-vip/kube-vip/pkg/etcd"
|
||||
"github.com/kube-vip/kube-vip/pkg/k8s"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
|
||||
|
||||
log "log/slog"
|
||||
"github.com/packethost/packngo"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -94,7 +96,7 @@ func NewManager(path string, inCluster bool, port int) (*Manager, error) {
|
||||
func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
|
||||
var err error
|
||||
|
||||
log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
|
||||
log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, c.LeaseName, c.NodeName)
|
||||
|
||||
// use a Go context so we can tell the leaderelection code when we
|
||||
// want to step down
|
||||
@@ -132,12 +134,9 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
|
||||
// (attempt to) Remove the virtual IP, in case it already exists
|
||||
|
||||
for i := range cluster.Network {
|
||||
deleted, err := cluster.Network[i].DeleteIP()
|
||||
err = cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Error("could not delete virtualIP", "err", err)
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
|
||||
log.Errorf("could not delete virtualIP: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,15 +147,41 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
|
||||
}
|
||||
}()
|
||||
|
||||
// If Equinix Metal is enabled then we can begin our preparation work
|
||||
var packetClient *packngo.Client
|
||||
if c.EnableMetal {
|
||||
if c.ProviderConfig != "" {
|
||||
key, project, err := equinixmetal.GetPacketConfig(c.ProviderConfig)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
} else {
|
||||
// Set the environment variable with the key for the project
|
||||
os.Setenv("PACKET_AUTH_TOKEN", key)
|
||||
// Update the configuration with the project key
|
||||
c.MetalProjectID = project
|
||||
}
|
||||
}
|
||||
packetClient, err = packngo.NewClient()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
// We're using Equinix Metal with BGP, populate the Peer information from the API
|
||||
if c.EnableBGP {
|
||||
log.Infoln("Looking up the BGP configuration from Equinix Metal")
|
||||
err = equinixmetal.BGPLookup(packetClient, c)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c.EnableBGP && bgpServer == nil {
|
||||
// Lets start BGP
|
||||
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
|
||||
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
|
||||
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
|
||||
if err != nil {
|
||||
log.Error("new BGP server", "err", err)
|
||||
}
|
||||
if err := bgpServer.Start(nil); err != nil {
|
||||
log.Error("starting BGP server", "err", err)
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,11 +189,11 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
|
||||
config: c,
|
||||
leaseID: c.NodeName,
|
||||
sm: sm,
|
||||
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
|
||||
onStartedLeading: func(ctx context.Context) {
|
||||
// As we're leading lets start the vip service
|
||||
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
|
||||
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient)
|
||||
if err != nil {
|
||||
log.Error("starting VIP service on leader", "err", err)
|
||||
log.Errorf("Error starting the VIP service on the leader [%s]", err)
|
||||
}
|
||||
},
|
||||
onStoppedLeading: func() {
|
||||
@@ -184,26 +209,22 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
|
||||
if bgpServer != nil {
|
||||
err := bgpServer.Close()
|
||||
if err != nil {
|
||||
log.Warn("close BGP server", "err", err)
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for i := range cluster.Network {
|
||||
deleted, err := cluster.Network[i].DeleteIP()
|
||||
err := cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn("delete VIP", "err", err)
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("lost leadership, restarting kube-vip")
|
||||
panic("") // TODO - we could also return here
|
||||
log.Fatal("lost leadership, restarting kube-vip")
|
||||
},
|
||||
onNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
log.Info("New leader", "leader", identity)
|
||||
log.Infof("Node [%s] is assuming leadership of the cluster", identity)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -284,16 +305,16 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
|
||||
})
|
||||
}
|
||||
|
||||
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
|
||||
func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) error {
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
log.Info("Kube-Vip is watching nodes for control-plane labels")
|
||||
log.Infof("Kube-Vip is watching nodes for control-plane labels")
|
||||
|
||||
listOptions := metav1.ListOptions{
|
||||
LabelSelector: "node-role.kubernetes.io/control-plane",
|
||||
}
|
||||
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
|
||||
},
|
||||
})
|
||||
@@ -322,16 +343,9 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
|
||||
// Find the node IP address (this isn't foolproof)
|
||||
for x := range node.Status.Addresses {
|
||||
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
|
||||
if checkIfNodeIsReady(node) {
|
||||
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
|
||||
if err != nil {
|
||||
log.Error("add IPVS backend", "err", err)
|
||||
}
|
||||
} else {
|
||||
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
|
||||
if err != nil {
|
||||
log.Error("remove IPVS backend", "err", err)
|
||||
}
|
||||
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
|
||||
if err != nil {
|
||||
log.Errorf("add IPVS backend [%v]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -346,12 +360,12 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
|
||||
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
|
||||
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
|
||||
if err != nil {
|
||||
log.Error("Del IPVS backend", "err", err)
|
||||
log.Errorf("Del IPVS backend [%v]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Node deleted", "name", node.Name)
|
||||
log.Infof("Node [%s] has been deleted", node.Name)
|
||||
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
@@ -362,29 +376,15 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error("watcher", "status", status)
|
||||
log.Errorf("%v", status)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Exiting Node watcher")
|
||||
log.Infoln("Exiting Node watcher")
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkIfNodeIsReady(node *v1.Node) bool {
|
||||
if node == nil {
|
||||
return false
|
||||
}
|
||||
for _, condition := range node.Status.Conditions {
|
||||
if condition.Type == v1.NodeReady {
|
||||
if condition.Status == v1.ConditionTrue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -8,25 +8,24 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/backend"
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/vishvananda/netlink"
|
||||
"github.com/packethost/packngo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
|
||||
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, packetClient *packngo.Client) error {
|
||||
var err error
|
||||
|
||||
// listen for interrupts or the Linux SIGTERM signal and cancel
|
||||
@@ -39,74 +38,117 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
// Add Notification for SIGTERM (sent from Kubernetes)
|
||||
signal.Notify(signalChan, syscall.SIGTERM)
|
||||
|
||||
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
|
||||
|
||||
var arpWG sync.WaitGroup
|
||||
|
||||
for i := range cluster.Network {
|
||||
network := cluster.Network[i]
|
||||
if err := network.SetMask(c.VIPSubnet); err != nil {
|
||||
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
|
||||
panic("")
|
||||
}
|
||||
if network.IsDDNS() {
|
||||
if cluster.Network[i].IsDDNS() {
|
||||
if err := cluster.StartDDNS(ctxDNS); err != nil {
|
||||
log.Error(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// start the dns updater if address is dns
|
||||
if network.IsDNS() {
|
||||
log.Info("starting the DNS updater", "address", network.DNSName())
|
||||
ipUpdater := vip.NewIPUpdater(network)
|
||||
if cluster.Network[i].IsDNS() {
|
||||
log.Infof("starting the DNS updater for the address %s", cluster.Network[i].DNSName())
|
||||
ipUpdater := vip.NewIPUpdater(cluster.Network[i])
|
||||
ipUpdater.Run(ctxDNS)
|
||||
}
|
||||
|
||||
if !c.EnableRoutingTable {
|
||||
if _, err = network.AddIP(false); err != nil {
|
||||
log.Error(err.Error())
|
||||
if c.EnableARP {
|
||||
subnets := vip.Split(c.VIPCIDR)
|
||||
subnet := ""
|
||||
if len(subnets) > 0 {
|
||||
subnet = subnets[0]
|
||||
}
|
||||
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
|
||||
subnet = subnets[1]
|
||||
}
|
||||
if subnet == "" {
|
||||
log.Fatalf("no subnet provided for IP %s", cluster.Network[i].IP())
|
||||
}
|
||||
if err = cluster.Network[i].SetMask(subnet); err != nil {
|
||||
log.Fatalf("failed to set mask %s: %s", subnet, err.Error())
|
||||
}
|
||||
}
|
||||
if err = cluster.Network[i].AddIP(false); err != nil {
|
||||
log.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if c.EnableMetal {
|
||||
// We're not using Equinix Metal with BGP
|
||||
if !c.EnableBGP {
|
||||
// Attempt to attach the EIP in the standard manner
|
||||
log.Debugf("Attaching the Equinix Metal EIP through the API to this host")
|
||||
err = equinixmetal.AttachEIP(packetClient, c, c.NodeName)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c.EnableBGP {
|
||||
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
|
||||
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
|
||||
err = bgpServer.AddHost(network.CIDR())
|
||||
cidrVip := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), c.VIPCIDR)
|
||||
log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
|
||||
|
||||
err = bgpServer.AddHost(cidrVip)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if c.EnableLoadBalancer {
|
||||
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
|
||||
log.Infof("Starting IPVS LoadBalancer")
|
||||
|
||||
lb, err := loadbalancer.NewIPVSLB(cluster.Network[i].IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval)
|
||||
if err != nil {
|
||||
log.Error("Error creating IPVS LoadBalancer", "err", err)
|
||||
log.Errorf("Error creating IPVS LoadBalancer [%s]", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
|
||||
err = sm.NodeWatcher(lb, c.Port)
|
||||
if err != nil {
|
||||
log.Error("Error watching node labels", "err", err)
|
||||
log.Errorf("Error watching node labels [%s]", err)
|
||||
}
|
||||
}()
|
||||
|
||||
loadbalancers = append(loadbalancers, lb)
|
||||
// Shutdown function that will wait on this signal, unless we call it ourselves
|
||||
go func() {
|
||||
<-signalChan
|
||||
err = lb.RemoveIPVSLB()
|
||||
if err != nil {
|
||||
log.Errorf("Error stopping IPVS LoadBalancer [%s]", err)
|
||||
}
|
||||
log.Info("Stopping IPVS LoadBalancer")
|
||||
}()
|
||||
}
|
||||
|
||||
if c.EnableARP {
|
||||
arpWG.Add(1)
|
||||
go cluster.layer2Update(ctxArp, network, c, &arpWG)
|
||||
}
|
||||
}
|
||||
go func(ctx context.Context, i int) {
|
||||
ipString := cluster.Network[i].IP()
|
||||
isIPv6 := vip.IsIPv6(ipString)
|
||||
|
||||
if c.EnableLoadBalancer {
|
||||
// Shutdown function that will wait on this signal, unless we call it ourselves
|
||||
<-signalChan
|
||||
for _, lb := range loadbalancers {
|
||||
err = lb.RemoveIPVSLB()
|
||||
if err != nil {
|
||||
log.Error("Error stopping IPVS LoadBalancer", "err", err)
|
||||
}
|
||||
var ndp *vip.NdpResponder
|
||||
if isIPv6 {
|
||||
ndp, err = vip.NewNDPResponder(cluster.Network[i].Interface())
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create new NDP Responder")
|
||||
}
|
||||
}
|
||||
|
||||
if ndp != nil {
|
||||
defer ndp.Close()
|
||||
}
|
||||
log.Infof("Gratuitous Arp broadcast will repeat every 3 seconds for [%s/%s]", ipString, cluster.Network[i].Interface())
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done(): // if cancel() execute
|
||||
return
|
||||
default:
|
||||
cluster.ensureIPAndSendGratuitous(i, ndp)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
}(ctxArp, i)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +167,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
ips := []string{}
|
||||
if nodename != "" {
|
||||
if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
|
||||
log.Error("failed to get IP of control-plane nod", "err", err)
|
||||
log.Error("failed to get IP of control-plane node: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,7 +182,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
ips = append(ips, "::1")
|
||||
}
|
||||
|
||||
log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
|
||||
log.Infof("no IP address found for node - will fallback to use localhost address: %v", ips)
|
||||
}
|
||||
|
||||
for _, ip := range ips {
|
||||
@@ -166,8 +208,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
|
||||
backend.Watch(func() {
|
||||
for i := range cluster.Network {
|
||||
network := cluster.Network[i]
|
||||
networkIP := network.IP()
|
||||
networkIP := cluster.Network[i].IP()
|
||||
isNetworkV6, err := isV6(networkIP)
|
||||
if err != nil {
|
||||
log.Error("failed to check IP type", "IP", networkIP, "error", err)
|
||||
@@ -181,19 +222,19 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
|
||||
for entry := range *backendMap {
|
||||
if entry.Check() {
|
||||
_, err = network.AddIP(true)
|
||||
err = cluster.Network[i].AddIP(true)
|
||||
if err != nil {
|
||||
log.Error("error adding address", "err", err)
|
||||
log.Fatalf("error adding IP: %v", err)
|
||||
}
|
||||
if !(*backendMap)[entry] {
|
||||
log.Info("added backend", "ip", network.IP())
|
||||
log.Infof("added IP: %s", cluster.Network[i].IP())
|
||||
}
|
||||
|
||||
err = network.AddRoute(true)
|
||||
err = cluster.Network[i].AddRoute(true)
|
||||
if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
|
||||
log.Warn(err.Error())
|
||||
log.Warnf("%v", err)
|
||||
} else if err == nil && !(*backendMap)[entry] {
|
||||
log.Info("added route", "route", network.PrepareRoute().String())
|
||||
log.Infof("added route: %s", cluster.Network[i].PrepareRoute().String())
|
||||
}
|
||||
|
||||
(*backendMap)[entry] = true
|
||||
@@ -211,20 +252,23 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
|
||||
}
|
||||
|
||||
if deleteAddress {
|
||||
err = network.DeleteRoute()
|
||||
err = cluster.Network[i].DeleteRoute()
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
|
||||
log.Warn("deleting route", "err", err)
|
||||
log.Warnf("error while deleting route: %v", err)
|
||||
} else if err == nil {
|
||||
log.Info("deleted route", "route", network.PrepareRoute().String())
|
||||
log.Infof("deleted route: %s", cluster.Network[i].PrepareRoute().String())
|
||||
}
|
||||
|
||||
deleted, err := network.DeleteIP()
|
||||
isSet, err := cluster.Network[i].IsSet()
|
||||
if err != nil {
|
||||
log.Error("error deleting IP", "err", err)
|
||||
panic("")
|
||||
log.Error("failed to check IP address", "error", err)
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
|
||||
if isSet {
|
||||
err = cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Fatalf("error deleting IP: %v", err)
|
||||
}
|
||||
log.Infof("deleted IP: %s", cluster.Network[i].IP())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -257,50 +301,88 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
|
||||
}
|
||||
|
||||
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
|
||||
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
|
||||
func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) {
|
||||
// use a Go context so we can tell the arp loop code when we
|
||||
// want to step down
|
||||
//nolint
|
||||
ctxArp, cancelArp := context.WithCancel(ctx)
|
||||
ctxArp, cancelArp := context.WithCancel(context.Background())
|
||||
|
||||
cluster.stop = make(chan bool, 1)
|
||||
cluster.completed = make(chan bool, 1)
|
||||
|
||||
var arpWG sync.WaitGroup
|
||||
|
||||
for i := range cluster.Network {
|
||||
network := cluster.Network[i]
|
||||
if err := network.SetMask(c.VIPSubnet); err != nil {
|
||||
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
|
||||
panic("")
|
||||
}
|
||||
_, err := network.DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn("attempted to clean existing VIP", "err", err)
|
||||
}
|
||||
|
||||
err := network.DeleteIP()
|
||||
if err != nil {
|
||||
log.Warnf("Attempted to clean existing VIP => %v", err)
|
||||
}
|
||||
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
|
||||
err = network.AddRoute(false)
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
} else if !c.EnableRoutingTable {
|
||||
if _, err = network.AddIP(false); err != nil {
|
||||
log.Warn(err.Error())
|
||||
if c.EnableARP {
|
||||
subnets := vip.Split(c.VIPCIDR)
|
||||
subnet := ""
|
||||
if len(subnets) > 0 {
|
||||
subnet = subnets[0]
|
||||
}
|
||||
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
|
||||
subnet = subnets[1]
|
||||
}
|
||||
if subnet == "" {
|
||||
log.Fatalf("no subnet provided for IP %s", cluster.Network[i].IP())
|
||||
}
|
||||
if err = network.SetMask(subnet); err != nil {
|
||||
log.Fatalf("failed to set mask %s: %s", subnet, err.Error())
|
||||
}
|
||||
}
|
||||
if err = network.AddIP(false); err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if c.EnableARP {
|
||||
arpWG.Add(1)
|
||||
go cluster.layer2Update(ctxArp, network, c, &arpWG)
|
||||
ipString := network.IP()
|
||||
var ndp *vip.NdpResponder
|
||||
if vip.IsIPv6(ipString) {
|
||||
ndp, err = vip.NewNDPResponder(network.Interface())
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create new NDP Responder")
|
||||
}
|
||||
}
|
||||
go func(ctx context.Context) {
|
||||
if ndp != nil {
|
||||
defer ndp.Close()
|
||||
}
|
||||
log.Debugf("(svcs) broadcasting ARP update for %s via %s, every %dms", ipString, network.Interface(), c.ArpBroadcastRate)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done(): // if cancel() execute
|
||||
log.Debugf("(svcs) ending ARP update for %s via %s, every %dms", ipString, network.Interface(), c.ArpBroadcastRate)
|
||||
return
|
||||
default:
|
||||
cluster.ensureIPAndSendGratuitous(i, ndp)
|
||||
}
|
||||
if c.ArpBroadcastRate < 500 {
|
||||
log.Errorf("arp broadcast rate is [%d], this shouldn't be lower that 300ms (defaulting to 3000)", c.ArpBroadcastRate)
|
||||
c.ArpBroadcastRate = 3000
|
||||
}
|
||||
time.Sleep(time.Duration(c.ArpBroadcastRate) * time.Millisecond)
|
||||
}
|
||||
}(ctxArp)
|
||||
}
|
||||
|
||||
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
|
||||
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
|
||||
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
|
||||
err = bgp.AddHost(network.CIDR())
|
||||
cidrVip := fmt.Sprintf("%s/%s", network.IP(), c.VIPCIDR)
|
||||
log.Debugf("(svcs) attempting to advertise the address [%s] over BGP", cidrVip)
|
||||
err = bgp.AddHost(cidrVip)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,19 +392,13 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
|
||||
// Stop the Arp context if it is running
|
||||
cancelArp()
|
||||
|
||||
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
|
||||
|
||||
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
|
||||
log.Info("[LOADBALANCER] Stopping load balancers")
|
||||
|
||||
if c.EnableRoutingTable {
|
||||
for i := range cluster.Network {
|
||||
// chek if route is not referenced by another service
|
||||
r := cluster.Network[i].PrepareRoute()
|
||||
if CountRouteReferences(r) < 1 {
|
||||
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
|
||||
if err := cluster.Network[i].DeleteRoute(); err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
log.Infof("[VIP] Deleting Route for Virtual IP [%s]", cluster.Network[i].IP())
|
||||
if err := cluster.Network[i].DeleteRoute(); err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,16 +406,9 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
|
||||
return
|
||||
}
|
||||
for i := range cluster.Network {
|
||||
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
|
||||
continue
|
||||
}
|
||||
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
|
||||
deleted, err := cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
|
||||
log.Infof("[VIP] Releasing the Virtual IP [%s]", cluster.Network[i].IP())
|
||||
if err := cluster.Network[i].DeleteIP(); err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,35 +416,52 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
|
||||
}()
|
||||
}
|
||||
|
||||
// Layer2Update, handles the creation of the
|
||||
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
|
||||
defer arpWG.Done()
|
||||
log.Info("layer 2 broadcaster starting")
|
||||
var ndp *vip.NdpResponder
|
||||
var err error
|
||||
ipString := network.IP()
|
||||
if vip.IsIPv6(ipString) {
|
||||
if network.IPisLinkLocal() {
|
||||
log.Error("layer2 is link-local can't use NDP", "address", ipString)
|
||||
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
|
||||
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
|
||||
// and in a dadfailed state.
|
||||
func (cluster *Cluster) ensureIPAndSendGratuitous(index int, ndp *vip.NdpResponder) {
|
||||
iface := cluster.Network[index].Interface()
|
||||
ipString := cluster.Network[index].IP()
|
||||
|
||||
} else {
|
||||
ndp, err = vip.NewNDPResponder(network.Interface())
|
||||
if err != nil {
|
||||
log.Error("failed to create new NDP Responder", "error", err)
|
||||
} else {
|
||||
if ndp != nil {
|
||||
defer ndp.Close()
|
||||
}
|
||||
}
|
||||
// Check if IP is dadfailed
|
||||
if cluster.Network[index].IsDADFAIL() {
|
||||
log.Warnf("IP address is in dadfailed state, removing [%s] from interface [%s]", ipString, iface)
|
||||
err := cluster.Network[index].DeleteIP()
|
||||
if err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
|
||||
// Ensure the address exists on the interface before attempting to ARP
|
||||
set, err := cluster.Network[index].IsSet()
|
||||
if err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
if !set {
|
||||
log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, iface)
|
||||
err = cluster.Network[index].AddIP(false)
|
||||
if err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
arpInstance := arp.NewInstance(network, ndp)
|
||||
cluster.arpMgr.Insert(arpInstance)
|
||||
if vip.IsIPv6(ipString) {
|
||||
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
|
||||
if ndp == nil {
|
||||
log.Error("NDP responder was not created")
|
||||
} else {
|
||||
err := ndp.SendGratuitous(ipString)
|
||||
if err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
|
||||
err := vip.ARPSendGratuitous(ipString, iface)
|
||||
if err != nil {
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
<-ctx.Done() // if cancel() execute
|
||||
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
|
||||
cluster.arpMgr.Remove(arpInstance)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@ package cluster
|
||||
import (
|
||||
"context"
|
||||
|
||||
log "log/slog"
|
||||
"github.com/packethost/packngo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
@@ -15,7 +16,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
|
||||
// Start kube-vip as a single node server
|
||||
|
||||
// TODO - Split all this code out as a separate function
|
||||
log.Info("Starting kube-vip as a single node cluster")
|
||||
log.Infoln("Starting kube-vip as a single node cluster")
|
||||
|
||||
log.Info("This node is assuming leadership of the cluster")
|
||||
|
||||
@@ -24,17 +25,14 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
|
||||
|
||||
for i := range cluster.Network {
|
||||
if !disableVIP {
|
||||
deleted, err := cluster.Network[i].DeleteIP()
|
||||
err := cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn("Attempted to clean existing VIP", "err", err)
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
|
||||
log.Warnf("Attempted to clean existing VIP => %v", err)
|
||||
}
|
||||
|
||||
_, err = cluster.Network[i].AddIP(false)
|
||||
err = cluster.Network[i].AddIP(false)
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -43,7 +41,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
|
||||
// Gratuitous ARP, will broadcast to new MAC <-> IP
|
||||
err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,23 +51,20 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
|
||||
|
||||
if !disableVIP {
|
||||
for i := range cluster.Network {
|
||||
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
|
||||
deleted, err := cluster.Network[i].DeleteIP()
|
||||
log.Infof("[VIP] Releasing the Virtual IP [%s]", cluster.Network[i].IP())
|
||||
err := cluster.Network[i].DeleteIP()
|
||||
if err != nil {
|
||||
log.Warn(err.Error())
|
||||
}
|
||||
if deleted {
|
||||
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
|
||||
log.Warnf("%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
close(cluster.completed)
|
||||
}()
|
||||
log.Info("Started Load Balancer and Virtual IP")
|
||||
log.Infoln("Started Load Balancer and Virtual IP")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
|
||||
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server, packetClient *packngo.Client) error {
|
||||
// use a Go context so we can tell the arp loop code when we
|
||||
// want to step down
|
||||
ctxArp, cancelArp := context.WithCancel(context.Background())
|
||||
@@ -80,5 +75,5 @@ func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp
|
||||
ctxDNS, cancelDNS := context.WithCancel(context.Background())
|
||||
defer cancelDNS()
|
||||
|
||||
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
|
||||
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, packetClient)
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package egress
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/nftables"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
|
||||
// Look up the destination ports from the annotations on the service
|
||||
destinationPorts := annotations[kubevip.EgressDestinationPorts]
|
||||
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
|
||||
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
|
||||
internalEgress := annotations[kubevip.EgressInternal]
|
||||
|
||||
protocol := iptables.ProtocolIPv4
|
||||
IPv6 := false
|
||||
if vip.IsIPv6(podIP) {
|
||||
protocol = iptables.ProtocolIPv6
|
||||
IPv6 = true
|
||||
}
|
||||
|
||||
// Use the internal egress implementation
|
||||
if internalEgress != "" {
|
||||
return nftables.DeleteSNAT(IPv6, serviceUUID)
|
||||
}
|
||||
|
||||
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error Creating iptables client [%s]", err)
|
||||
}
|
||||
|
||||
if deniedNetworks != "" {
|
||||
networks := strings.Split(deniedNetworks, ",")
|
||||
for x := range networks {
|
||||
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if allowedNetworks != "" {
|
||||
networks := strings.Split(allowedNetworks, ",")
|
||||
for x := range networks {
|
||||
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Remove the marking of egress packets
|
||||
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Clear up SNAT rules
|
||||
if destinationPorts != "" {
|
||||
fixedPorts := strings.Split(destinationPorts, ",")
|
||||
|
||||
for _, fixedPort := range fixedPorts {
|
||||
var proto, port string
|
||||
|
||||
data := strings.Split(fixedPort, ":")
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
} else if len(data) == 1 {
|
||||
proto = "tcp"
|
||||
port = data[0]
|
||||
} else {
|
||||
proto = data[0]
|
||||
port = data[1]
|
||||
}
|
||||
|
||||
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
err = i.DeleteSourceNat(podIP, vipIP)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
type Processor struct {
|
||||
config *kubevip.Config
|
||||
provider providers.Provider
|
||||
bgpServer *bgp.Server
|
||||
worker endpointWorker
|
||||
instances *[]*instance.Instance
|
||||
}
|
||||
|
||||
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
|
||||
instances *[]*instance.Instance) *Processor {
|
||||
return &Processor{
|
||||
config: config,
|
||||
provider: provider,
|
||||
bgpServer: bgpServer,
|
||||
instances: instances,
|
||||
worker: newEndpointWorker(config, provider, bgpServer, instances),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
|
||||
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
|
||||
serviceFunc func(context.Context, *v1.Service) error,
|
||||
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
|
||||
|
||||
var err error
|
||||
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
|
||||
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
endpoints, err := p.worker.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
|
||||
log.Error("updating instance", "err", err)
|
||||
}
|
||||
|
||||
// Find out if we have any local endpoints
|
||||
// if out endpoint is empty then populate it
|
||||
// if not, go through the endpoints and see if ours still exists
|
||||
// If we have a local endpoint then begin the leader Election, unless it's already running
|
||||
//
|
||||
|
||||
// Check that we have local endpoints
|
||||
if len(endpoints) != 0 {
|
||||
// Ignore IPv4
|
||||
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
|
||||
// start leader election if it's enabled and not already started
|
||||
if !*leaderElectionActive && p.config.EnableServicesElection {
|
||||
go func() {
|
||||
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
|
||||
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
|
||||
}()
|
||||
}
|
||||
|
||||
// There are local endpoints available on the node
|
||||
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
|
||||
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
|
||||
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// There are no local endpoints
|
||||
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
|
||||
}
|
||||
|
||||
// Set the service accordingly
|
||||
p.updateAnnotations(service, lastKnownGoodEndpoint)
|
||||
|
||||
log.Debug("watcher", "provider",
|
||||
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Delete(service *v1.Service, id string) error {
|
||||
if err := p.worker.delete(service, id); err != nil {
|
||||
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
|
||||
// if we haven't populated one, then do so
|
||||
if *lastKnownGoodEndpoint == "" {
|
||||
*lastKnownGoodEndpoint = endpoints[0]
|
||||
return
|
||||
}
|
||||
|
||||
// check out previous endpoint exists
|
||||
stillExists := false
|
||||
|
||||
for x := range endpoints {
|
||||
if endpoints[x] == *lastKnownGoodEndpoint {
|
||||
stillExists = true
|
||||
}
|
||||
}
|
||||
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
|
||||
if !stillExists {
|
||||
p.worker.removeEgress(service, lastKnownGoodEndpoint)
|
||||
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
|
||||
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
|
||||
// Stop the existing leaderElection
|
||||
cancel()
|
||||
// disable last leaderElection flag
|
||||
*leaderElectionActive = false
|
||||
}
|
||||
// Set our active endpoint to an existing one
|
||||
*lastKnownGoodEndpoint = endpoints[0]
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
|
||||
// Set the service accordingly
|
||||
if service.Annotations[kubevip.Egress] == "true" {
|
||||
activeEndpointAnnotation := kubevip.ActiveEndpoint
|
||||
|
||||
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
|
||||
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
|
||||
}
|
||||
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
|
||||
}
|
||||
}
|
||||
|
||||
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
|
||||
// This is a blocking function, that will restart (in the event of failure)
|
||||
for {
|
||||
// if the context isn't cancelled restart
|
||||
if ctx.Err() != context.Canceled {
|
||||
*leaderElectionActive = true
|
||||
err := serviceFunc(ctx, service)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
*leaderElectionActive = false
|
||||
} else {
|
||||
*leaderElectionActive = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
type BGP struct {
|
||||
generic
|
||||
bgpServer *bgp.Server
|
||||
}
|
||||
|
||||
func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
|
||||
return &BGP{
|
||||
generic: generic,
|
||||
bgpServer: bgpServer,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
|
||||
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
|
||||
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
|
||||
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
|
||||
if err != nil {
|
||||
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
|
||||
} else {
|
||||
log.Info("added BGP host", "provider",
|
||||
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
|
||||
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
|
||||
*leaderElectionActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
|
||||
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
|
||||
// If BGP mode is enabled - routes should be deleted
|
||||
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
|
||||
if err != nil {
|
||||
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
|
||||
} else {
|
||||
log.Info("deleted BGP host", "provider",
|
||||
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
|
||||
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
|
||||
*leaderElectionActive = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
|
||||
}
|
||||
|
||||
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
return b.getAllEndpoints(service, id)
|
||||
}
|
||||
|
||||
func (b *BGP) delete(service *v1.Service, id string) error {
|
||||
// When no-leader-elecition mode
|
||||
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
|
||||
// find all existing local endpoints
|
||||
endpoints, err := b.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
// If there were local endpoints deleted
|
||||
if len(endpoints) > 0 {
|
||||
b.deleteAction(service)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BGP) deleteAction(service *v1.Service) {
|
||||
b.clearBGPHosts(service)
|
||||
}
|
||||
|
||||
func (b *BGP) clearBGPHosts(service *v1.Service) {
|
||||
ClearBGPHosts(service, b.instances, b.bgpServer)
|
||||
}
|
||||
|
||||
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
|
||||
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
|
||||
ClearBGPHostsByInstance(instance, bgpServer)
|
||||
}
|
||||
}
|
||||
|
||||
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
network := cluster.Network[i]
|
||||
err := bgpServer.DelHost(network.CIDR())
|
||||
if err != nil {
|
||||
log.Error("[endpoint] error deleting BGP host", "err", err)
|
||||
} else {
|
||||
log.Debug("[endpoint] deleted BGP host", "ip",
|
||||
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/egress"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
type endpointWorker interface {
|
||||
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
|
||||
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
|
||||
getEndpoints(service *v1.Service, id string) ([]string, error)
|
||||
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
|
||||
delete(service *v1.Service, id string) error
|
||||
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
|
||||
}
|
||||
|
||||
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
|
||||
generic := newGeneric(config, provider, instances)
|
||||
|
||||
if config.EnableRoutingTable {
|
||||
return newRoutingTable(generic)
|
||||
}
|
||||
if config.EnableBGP {
|
||||
return newBGP(generic, bgpServer)
|
||||
}
|
||||
|
||||
return &generic
|
||||
}
|
||||
|
||||
type generic struct {
|
||||
config *kubevip.Config
|
||||
provider providers.Provider
|
||||
instances *[]*instance.Instance
|
||||
}
|
||||
|
||||
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
|
||||
return generic{
|
||||
config: config,
|
||||
provider: provider,
|
||||
instances: instances,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
|
||||
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
|
||||
}
|
||||
|
||||
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
|
||||
if *lastKnownGoodEndpoint != "" {
|
||||
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
|
||||
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
|
||||
log.Error("error removing redundant egress rules", "err", err)
|
||||
}
|
||||
|
||||
*lastKnownGoodEndpoint = "" // reset endpoint
|
||||
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
|
||||
cancel() // stop services watcher
|
||||
}
|
||||
*leaderElectionActive = false
|
||||
}
|
||||
}
|
||||
|
||||
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
|
||||
return g.getLocalEndpoints(id)
|
||||
}
|
||||
|
||||
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
|
||||
// Build endpoints
|
||||
var endpoints []string
|
||||
var err error
|
||||
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
|
||||
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
// Build endpoints
|
||||
var err error
|
||||
var endpoints []string
|
||||
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
|
||||
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
|
||||
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
|
||||
}
|
||||
} else {
|
||||
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
|
||||
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
|
||||
}
|
||||
|
||||
func (g *generic) delete(_ *v1.Service, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/egress"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
type RoutingTable struct {
|
||||
generic
|
||||
}
|
||||
|
||||
func newRoutingTable(generic generic) endpointWorker {
|
||||
return &RoutingTable{
|
||||
generic: generic,
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
|
||||
instance := instance.FindServiceInstance(service, *rt.instances)
|
||||
if instance != nil {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
|
||||
err := cluster.Network[i].AddRoute(false)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EEXIST) {
|
||||
// If route exists, but protocol is not set (e.g. the route was created by the older version
|
||||
// of kube-vip) try to update it if necessary
|
||||
isUpdated, err := cluster.Network[i].UpdateRoutes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
|
||||
}
|
||||
if isUpdated {
|
||||
log.Info("updated route", "provider",
|
||||
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
|
||||
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
|
||||
} else {
|
||||
log.Info("route already present", "provider",
|
||||
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
|
||||
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
|
||||
}
|
||||
} else {
|
||||
// If other error occurs, return error
|
||||
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Info("added route", "provider",
|
||||
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
|
||||
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
|
||||
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
|
||||
*leaderElectionActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
|
||||
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
|
||||
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
|
||||
svcCtx.ConfiguredNetworks.Clear()
|
||||
} else {
|
||||
for _, err := range errs {
|
||||
log.Error("error while clearing routes", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
return rt.getAllEndpoints(service, id)
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
|
||||
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
|
||||
service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
|
||||
log.Warn("removing redundant egress rules", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
|
||||
// When no-leader-elecition mode
|
||||
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
|
||||
// find all existing local endpoints
|
||||
endpoints, err := rt.getEndpoints(service, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
// If there were local endpoints deleted
|
||||
if len(endpoints) > 0 {
|
||||
rt.deleteAction(service)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) deleteAction(service *v1.Service) {
|
||||
ClearRoutes(service, rt.instances)
|
||||
}
|
||||
|
||||
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
|
||||
instance := instance.FindServiceInstance(service, *rt.instances)
|
||||
if instance == nil {
|
||||
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
|
||||
} else {
|
||||
for _, c := range instance.Clusters {
|
||||
for n := range c.Network {
|
||||
// if there are no endpoints set HasEndpoints false just in case
|
||||
if len(endpoints) < 1 {
|
||||
c.Network[n].SetHasEndpoints(false)
|
||||
}
|
||||
// check if endpoint are available and are of same IP family as service
|
||||
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
|
||||
c.Network[n].SetHasEndpoints(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
|
||||
errs := []error{}
|
||||
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
route := cluster.Network[i].PrepareRoute()
|
||||
// check if route we are about to delete is not referenced by more than one service
|
||||
if CountRouteReferences(route, instances) <= 1 {
|
||||
err := cluster.Network[i].DeleteRoute()
|
||||
if err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
log.Debug("deleted route", "ip",
|
||||
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
|
||||
cnt := 0
|
||||
for _, instance := range *instances {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for n := range cluster.Network {
|
||||
if cluster.Network[n].HasEndpoints() {
|
||||
r := cluster.Network[n].PrepareRoute()
|
||||
if r.Dst.String() == route.Dst.String() {
|
||||
cnt++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
"k8s.io/client-go/util/retry"
|
||||
|
||||
log "log/slog"
|
||||
)
|
||||
|
||||
type Endpoints struct {
|
||||
label string
|
||||
//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
|
||||
endpoints *v1.Endpoints
|
||||
}
|
||||
|
||||
func NewEndpoints() Provider {
|
||||
return &Endpoints{
|
||||
label: "endpoints",
|
||||
}
|
||||
}
|
||||
|
||||
func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
|
||||
service *v1.Service) (*watchtools.RetryWatcher, error) {
|
||||
opts := metav1.ListOptions{
|
||||
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
|
||||
}
|
||||
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
|
||||
}
|
||||
|
||||
return rw, nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
|
||||
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
|
||||
eps, ok := endpoints.(*v1.Endpoints)
|
||||
if !ok {
|
||||
cancel()
|
||||
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
|
||||
}
|
||||
ep.endpoints = eps
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
|
||||
result := []string{}
|
||||
for subset := range ep.endpoints.Subsets {
|
||||
for address := range ep.endpoints.Subsets[subset].Addresses {
|
||||
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
|
||||
result = append(result, addr[0])
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
|
||||
var localEndpoints []string
|
||||
|
||||
for _, subset := range ep.endpoints.Subsets {
|
||||
for _, address := range subset.Addresses {
|
||||
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
|
||||
|
||||
// 1. Compare the Nodename
|
||||
if address.NodeName != nil && id == *address.NodeName {
|
||||
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
|
||||
localEndpoints = append(localEndpoints, address.IP)
|
||||
continue
|
||||
}
|
||||
// 2. Compare the Hostname (only useful if address.NodeName is not available)
|
||||
if id == address.Hostname {
|
||||
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
|
||||
localEndpoints = append(localEndpoints, address.IP)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return localEndpoints, nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
|
||||
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
// Retrieve the latest version of Deployment before attempting update
|
||||
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
|
||||
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentServiceCopy := currentService.DeepCopy()
|
||||
if currentServiceCopy.Annotations == nil {
|
||||
currentServiceCopy.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
|
||||
|
||||
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if retryErr != nil {
|
||||
log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
|
||||
return retryErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpoints) GetLabel() string {
|
||||
return ep.label
|
||||
}
|
||||
|
||||
func (ep *Endpoints) GetProtocol() string {
|
||||
return ""
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
CreateRetryWatcher(context.Context, *kubernetes.Clientset,
|
||||
*v1.Service) (*watchtools.RetryWatcher, error)
|
||||
GetAllEndpoints() ([]string, error)
|
||||
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
|
||||
GetLabel() string
|
||||
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
|
||||
LoadObject(runtime.Object, context.CancelFunc) error
|
||||
GetProtocol() string
|
||||
}
|
||||
63
pkg/equinixmetal/bgp.go
Normal file
63
pkg/equinixmetal/bgp.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package equinixmetal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/packethost/packngo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// BGPLookup will use the Equinix Metal API functions to populate the BGP information
|
||||
func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
|
||||
var thisDevice *packngo.Device
|
||||
if k.MetalProjectID == "" {
|
||||
proj := findProject(k.MetalProject, c)
|
||||
if proj == nil {
|
||||
return fmt.Errorf("Unable to find Project [%s]", k.MetalProject)
|
||||
}
|
||||
thisDevice = findSelf(c, proj.ID)
|
||||
} else {
|
||||
thisDevice = findSelf(c, k.MetalProjectID)
|
||||
}
|
||||
if thisDevice == nil {
|
||||
return fmt.Errorf("Unable to find local/this device in Equinix Metal API")
|
||||
}
|
||||
|
||||
log.Infof("Querying BGP settings for [%s]", thisDevice.Hostname)
|
||||
neighbours, _, err := c.Devices.ListBGPNeighbors(thisDevice.ID, &packngo.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Ensure neighbours exist (and it's enabled)
|
||||
if len(neighbours) == 0 {
|
||||
return fmt.Errorf("The server [%s]/[%s] has no BGP neighbours, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
|
||||
}
|
||||
|
||||
// Add a warning (TODO)
|
||||
if len(neighbours) > 1 {
|
||||
log.Warnf("There are [%d] neighbours, only designed to manage one", len(neighbours))
|
||||
}
|
||||
|
||||
// Ensure a peer exists
|
||||
if len(neighbours[0].PeerIps) == 0 {
|
||||
return fmt.Errorf("The server [%s]/[%s] has no BGP peers, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
|
||||
}
|
||||
|
||||
k.BGPConfig.RouterID = neighbours[0].CustomerIP
|
||||
k.BGPConfig.AS = uint32(neighbours[0].CustomerAs)
|
||||
|
||||
// Add the peer(s)
|
||||
for x := range neighbours[0].PeerIps {
|
||||
peer := bgp.Peer{
|
||||
Address: neighbours[0].PeerIps[x],
|
||||
AS: uint32(neighbours[0].PeerAs),
|
||||
MultiHop: neighbours[0].Multihop,
|
||||
Password: neighbours[0].Md5Password,
|
||||
}
|
||||
k.BGPConfig.Peers = append(k.BGPConfig.Peers, peer)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
65
pkg/equinixmetal/eip.go
Normal file
65
pkg/equinixmetal/eip.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package equinixmetal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/packethost/packngo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// AttachEIP will use the Equinix Metal APIs to move an EIP and attach to a host
|
||||
func AttachEIP(c *packngo.Client, k *kubevip.Config, _ string) error {
|
||||
// Use MetalProjectID if it is defined
|
||||
projID := k.MetalProjectID
|
||||
|
||||
if projID == "" {
|
||||
// Fallback to attempting to find the project by name
|
||||
proj := findProject(k.MetalProject, c)
|
||||
if proj == nil {
|
||||
return fmt.Errorf("unable to find Project [%s]", k.MetalProject)
|
||||
}
|
||||
|
||||
projID = proj.ID
|
||||
}
|
||||
|
||||
// Prefer Address over VIP
|
||||
vip := k.Address
|
||||
if vip == "" {
|
||||
vip = k.VIP
|
||||
}
|
||||
|
||||
ips, _, _ := c.ProjectIPs.List(projID, &packngo.ListOptions{})
|
||||
for _, ip := range ips {
|
||||
// Find the device id for our EIP
|
||||
if ip.Address == vip {
|
||||
log.Infof("Found EIP ->%s ID -> %s\n", ip.Address, ip.ID)
|
||||
// If attachments already exist then remove them
|
||||
if len(ip.Assignments) != 0 {
|
||||
hrefID := path.Base(ip.Assignments[0].Href)
|
||||
_, err := c.DeviceIPs.Unassign(hrefID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to unassign deviceIP %q: %v", hrefID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup this server through the Equinix Metal API
|
||||
thisDevice := findSelf(c, projID)
|
||||
if thisDevice == nil {
|
||||
return fmt.Errorf("unable to find local/this device in Equinix Metal API")
|
||||
}
|
||||
|
||||
// Assign the EIP to this device
|
||||
log.Infof("Assigning EIP to -> %s\n", thisDevice.Hostname)
|
||||
_, _, err := c.DeviceIPs.Assign(thisDevice.ID, &packngo.AddressStruct{
|
||||
Address: vip,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
59
pkg/equinixmetal/utils.go
Normal file
59
pkg/equinixmetal/utils.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package equinixmetal
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/packethost/packngo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func findProject(project string, c *packngo.Client) *packngo.Project {
|
||||
l := &packngo.ListOptions{Includes: []string{project}}
|
||||
ps, _, err := c.Projects.List(l)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
for _, p := range ps {
|
||||
|
||||
// Find our project
|
||||
if p.Name == project {
|
||||
return &p
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findSelf(c *packngo.Client, projectID string) *packngo.Device {
|
||||
// Go through devices
|
||||
dev, _, _ := c.Devices.List(projectID, &packngo.ListOptions{})
|
||||
for _, d := range dev {
|
||||
// TODO do we need to replace os.Hostname with config.NodeName here?
|
||||
me, _ := os.Hostname()
|
||||
if me == d.Hostname {
|
||||
return &d
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetPacketConfig will lookup the configuration from a file path
|
||||
func GetPacketConfig(providerConfig string) (string, string, error) {
|
||||
var config struct {
|
||||
AuthToken string `json:"apiKey"`
|
||||
ProjectID string `json:"projectId"`
|
||||
}
|
||||
// get our token and project
|
||||
if providerConfig != "" {
|
||||
configBytes, err := os.ReadFile(providerConfig)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get read configuration file at path %s: %v", providerConfig, err)
|
||||
}
|
||||
err = json.Unmarshal(configBytes, &config)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to process json of configuration file at path %s: %v", providerConfig, err)
|
||||
}
|
||||
}
|
||||
return config.AuthToken, config.ProjectID, nil
|
||||
}
|
||||
@@ -5,9 +5,8 @@ import (
|
||||
"hash/fnv"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
pb "go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"go.etcd.io/etcd/client/v3/concurrency"
|
||||
@@ -32,7 +31,7 @@ type LeaderElectionConfig struct {
|
||||
|
||||
// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
|
||||
// from string to int using hashing.
|
||||
MemberUniqueID *uint64
|
||||
MemberUniqueID *int64
|
||||
|
||||
// LeaseDurationSeconds is the duration that non-leader candidates will
|
||||
// wait to force acquire leadership.
|
||||
@@ -74,7 +73,7 @@ func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
|
||||
// RunElection blocks until leader election loop is
|
||||
// stopped by ctx or it has stopped holding the leader lease.
|
||||
func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
|
||||
var memberID uint64
|
||||
var memberID int64
|
||||
if config.MemberUniqueID != nil {
|
||||
memberID = *config.MemberUniqueID
|
||||
} else {
|
||||
@@ -82,11 +81,11 @@ func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
|
||||
if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
|
||||
return err
|
||||
}
|
||||
memberID = h.Sum64()
|
||||
memberID = int64(h.Sum64())
|
||||
}
|
||||
|
||||
ttl := config.LeaseDurationSeconds
|
||||
r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
|
||||
r := &pb.LeaseGrantRequest{TTL: ttl, ID: memberID}
|
||||
lease, err := clientv3.RetryLeaseClient(
|
||||
config.EtcdConfig.Client,
|
||||
).LeaseGrant(ctx, r)
|
||||
@@ -148,9 +147,9 @@ watcher:
|
||||
|
||||
m.isLeader = true
|
||||
m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
|
||||
log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
|
||||
log.Debugf("[%s] Marking self as leader with key %s\n", m.memberID, m.key)
|
||||
case response := <-changes:
|
||||
log.Debug("Leader Changes", "id", m.memberID, "response", response)
|
||||
log.Debugf("[%s] Leader Changes: %+v\n", m.memberID, response)
|
||||
if len(response.Kvs) == 0 {
|
||||
// There is a race condition where just after we stop being the leader
|
||||
// if there are no more leaders, we might get a response with no key-values
|
||||
@@ -182,15 +181,15 @@ watcher:
|
||||
m.callbacks.OnStoppedLeading()
|
||||
}
|
||||
|
||||
log.Debug("Exiting watcher", "id", m.memberID)
|
||||
log.Debugf("[%s] Exiting watcher\n", m.memberID)
|
||||
}
|
||||
|
||||
func (m *member) tryToBeLeader(ctx context.Context) {
|
||||
if err := m.election.Campaign(ctx, m.memberID); err != nil {
|
||||
log.Error("Failed trying to become the leader", "err", err)
|
||||
log.Errorf("Failed trying to become the leader: %s", err)
|
||||
// Resign just in case we acquired leadership just before failing
|
||||
if err := m.election.Resign(m.client.Ctx()); err != nil {
|
||||
log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
|
||||
log.Warnf("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader: %s", err)
|
||||
}
|
||||
return
|
||||
// TODO: what to do here?
|
||||
@@ -210,7 +209,7 @@ func (m *member) tryToBeLeader(ctx context.Context) {
|
||||
// the previous leader to detect the new leadership (if there was one) and
|
||||
// stop its processes
|
||||
// TODO: is this too cautious?
|
||||
log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
|
||||
log.Debugf("[%s] Waiting %d seconds before running OnStartedLeading", m.memberID, m.leaseTTL)
|
||||
time.Sleep(time.Second * time.Duration(m.leaseTTL))
|
||||
|
||||
// We are the leader, execute our code
|
||||
@@ -222,6 +221,6 @@ func (m *member) tryToBeLeader(ctx context.Context) {
|
||||
func (m *member) resignOnCancel(ctx context.Context) {
|
||||
<-ctx.Done()
|
||||
if err := m.election.Resign(m.client.Ctx()); err != nil {
|
||||
log.Error("Failed to resign after the context was canceled", "err", err)
|
||||
log.Errorf("Failed to resign after the context was canceled: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,12 +91,12 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
|
||||
config1 := configBase
|
||||
config1.EtcdConfig.Client = cliMember1
|
||||
config1.MemberID = "my-host"
|
||||
uniqueID := rand.Uint64()
|
||||
uniqueID := rand.Int63()
|
||||
config1.MemberUniqueID = &uniqueID
|
||||
config1.Callbacks = baseCallbacksForName(config1.MemberID)
|
||||
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
|
||||
log.Println("I'm my-host, the new leader!!!!")
|
||||
log.Println("Losing the leadership on purpose by stopping renewing the lease")
|
||||
log.Println("Loosing the leadership on purpose by stopping renewing the lease")
|
||||
g.Expect(cliMember1.Lease.Close()).To(Succeed())
|
||||
log.Println("Member1 leases closed")
|
||||
}
|
||||
|
||||
@@ -6,8 +6,7 @@ import (
|
||||
"net"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
@@ -100,21 +99,21 @@ func FindWorkingKubernetesAddress(configPath string, inCluster bool) (*rest.Conf
|
||||
return nil, err
|
||||
}
|
||||
for x := range ips {
|
||||
log.Debug("[k8s client] testing", "address", ips[x].String())
|
||||
log.Debugf("[k8s client] checking with IP address [%s]", ips[x].String())
|
||||
c, err := NewRestConfig(configPath, inCluster, net.JoinHostPort(ips[x].String(), "6443"))
|
||||
if err != nil {
|
||||
log.Error("failed to create k8s REST config", "err", err)
|
||||
log.Errorf("failed to create k8s REST config: %v", err)
|
||||
}
|
||||
|
||||
c.Timeout = 2 * time.Second
|
||||
k, err := NewClientset(c)
|
||||
if err != nil {
|
||||
log.Error("failed to create k8s clientset", "err", err)
|
||||
log.Errorf("failed to create k8s clientset: %v", err)
|
||||
}
|
||||
|
||||
_, err = k.DiscoveryClient.ServerVersion()
|
||||
if err == nil {
|
||||
log.Info("[k8s client] working", "address", ips[x].String())
|
||||
log.Infof("[k8s client] working with IP address [%s]", ips[x].String())
|
||||
c.Timeout = defaultTimeout
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package kubevip
|
||||
|
||||
const (
|
||||
// Hardware address of the host that has the VIP
|
||||
HwAddrKey = "kube-vip.io/hwaddr"
|
||||
|
||||
// The IP address that is requested
|
||||
RequestedIP = "kube-vip.io/requestedIP"
|
||||
|
||||
// The host that has the VIP
|
||||
VipHost = "kube-vip.io/vipHost"
|
||||
|
||||
// Enable Egress on a service
|
||||
Egress = "kube-vip.io/egress"
|
||||
|
||||
// Enable internal Egress
|
||||
EgressInternal = "kube-vip.io/egress-internal"
|
||||
|
||||
// Egress should be IPv6
|
||||
EgressIPv6 = "kube-vip.io/egress-ipv6"
|
||||
|
||||
// Ports that traffic is allowed to access from the egress VIP
|
||||
EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
|
||||
|
||||
// Allowed incoming ports to the VIP
|
||||
EgressSourcePorts = "kube-vip.io/egress-source-ports"
|
||||
|
||||
// Allowed networks for the Egress to be enabled for
|
||||
EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
|
||||
|
||||
// Networks that we wont Egress for
|
||||
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
|
||||
|
||||
// The current active endpoint(pod) for the Egress VIP
|
||||
ActiveEndpoint = "kube-vip.io/active-endpoint"
|
||||
|
||||
// The current active endpoint(pod) for the Egress VIP (v6)
|
||||
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
|
||||
|
||||
// Flush the conntrack rules (remove existing sessions) once Egress is configured
|
||||
FlushContrack = "kube-vip.io/flush-conntrack"
|
||||
|
||||
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
|
||||
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
|
||||
ServiceInterface = "kube-vip.io/serviceInterface"
|
||||
UpnpEnabled = "kube-vip.io/forwardUPNP"
|
||||
|
||||
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
|
||||
)
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"math/bits"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/detector"
|
||||
@@ -20,13 +19,17 @@ func ParseEnvironment(c *Config) error {
|
||||
}
|
||||
// Ensure that logging is set through the environment variables
|
||||
env := os.Getenv(vipLogLevel)
|
||||
// Set default value
|
||||
if env == "" {
|
||||
env = "4"
|
||||
}
|
||||
|
||||
if env != "" {
|
||||
logLevel, err := strconv.ParseInt(env, 10, 32)
|
||||
logLevel, err := strconv.ParseUint(env, 10, 32)
|
||||
if err != nil {
|
||||
panic("Unable to parse environment variable [vip_loglevel], should be int")
|
||||
}
|
||||
c.Logging = int32(logLevel)
|
||||
c.Logging = int(logLevel)
|
||||
}
|
||||
|
||||
// Find interface
|
||||
@@ -50,6 +53,12 @@ func ParseEnvironment(c *Config) error {
|
||||
c.ServicesInterface = env
|
||||
}
|
||||
|
||||
// Find provider configuration
|
||||
env = os.Getenv(providerConfig)
|
||||
if env != "" {
|
||||
c.ProviderConfig = env
|
||||
}
|
||||
|
||||
// Find Kubernetes Leader Election configuration
|
||||
env = os.Getenv(vipLeaderElection)
|
||||
if env != "" {
|
||||
@@ -127,11 +136,11 @@ func ParseEnvironment(c *Config) error {
|
||||
// Find vip port
|
||||
env = os.Getenv(port)
|
||||
if env != "" {
|
||||
i, err := strconv.ParseUint(env, 10, 16)
|
||||
i, err := strconv.ParseInt(env, 10, 32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Port = uint16(i)
|
||||
c.Port = int(i)
|
||||
}
|
||||
|
||||
// Find vipDdns
|
||||
@@ -233,6 +242,12 @@ func ParseEnvironment(c *Config) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Find vip address cidr range
|
||||
env = os.Getenv(vipCidr)
|
||||
if env != "" {
|
||||
c.VIPCIDR = env
|
||||
}
|
||||
|
||||
// Find vip address subnet
|
||||
env = os.Getenv(vipSubnet)
|
||||
if env != "" {
|
||||
@@ -429,24 +444,6 @@ func ParseEnvironment(c *Config) error {
|
||||
c.BGPConfig.Peers = peers
|
||||
}
|
||||
|
||||
// MPBGP mode
|
||||
env = os.Getenv(mpbgpNexthop)
|
||||
if env != "" {
|
||||
c.BGPConfig.MpbgpNexthop = env
|
||||
}
|
||||
|
||||
// MPBGP fixed IPv4
|
||||
env = os.Getenv(mpbgpIPv4)
|
||||
if env != "" {
|
||||
c.BGPConfig.MpbgpIPv4 = env
|
||||
}
|
||||
|
||||
// MPBGP fixed IPv6
|
||||
env = os.Getenv(mpbgpIPv6)
|
||||
if env != "" {
|
||||
c.BGPConfig.MpbgpIPv6 = env
|
||||
}
|
||||
|
||||
// BGP Peer mutlihop
|
||||
env = os.Getenv(bgpMultiHop)
|
||||
if env != "" {
|
||||
@@ -501,32 +498,28 @@ func ParseEnvironment(c *Config) error {
|
||||
c.BGPConfig.KeepaliveInterval = u64
|
||||
}
|
||||
|
||||
env = os.Getenv(zebraEnable)
|
||||
// Enable the Equinix Metal API calls
|
||||
env = os.Getenv(vipPacket)
|
||||
if env != "" {
|
||||
result, err := strconv.ParseBool(env)
|
||||
b, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.BGPConfig.Zebra.Enabled = result
|
||||
c.EnableMetal = b
|
||||
}
|
||||
|
||||
env = os.Getenv(zebraURL)
|
||||
// Find the Equinix Metal project name
|
||||
env = os.Getenv(vipPacketProject)
|
||||
if env != "" {
|
||||
c.BGPConfig.Zebra.URL = env
|
||||
// TODO - parse address net.Host()
|
||||
c.MetalProject = env
|
||||
}
|
||||
|
||||
env = os.Getenv(zebraVersion)
|
||||
// Find the Equinix Metal project ID
|
||||
env = os.Getenv(vipPacketProjectID)
|
||||
if env != "" {
|
||||
u64, err := strconv.ParseUint(env, 10, 32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.BGPConfig.Zebra.Version = uint32(u64)
|
||||
}
|
||||
|
||||
env = os.Getenv(zebraSoftwareName)
|
||||
if env != "" {
|
||||
c.BGPConfig.Zebra.SoftwareName = env
|
||||
// TODO - parse address net.Host()
|
||||
c.MetalProjectID = env
|
||||
}
|
||||
|
||||
// Enable the load-balancer
|
||||
@@ -542,11 +535,11 @@ func ParseEnvironment(c *Config) error {
|
||||
// Find loadbalancer port
|
||||
env = os.Getenv(lbPort)
|
||||
if env != "" {
|
||||
i, err := strconv.ParseUint(env, 10, 16)
|
||||
i, err := strconv.ParseInt(env, 10, 32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.LoadBalancerPort = uint16(i)
|
||||
c.LoadBalancerPort = int(i)
|
||||
}
|
||||
|
||||
// Find loadbalancer forwarding method
|
||||
@@ -607,13 +600,13 @@ func ParseEnvironment(c *Config) error {
|
||||
c.K8sConfigFile = env
|
||||
}
|
||||
|
||||
env = os.Getenv(enableEndpoints)
|
||||
env = os.Getenv(enableEndpointSlices)
|
||||
if env != "" {
|
||||
b, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.EnableEndpoints = b
|
||||
c.EnableEndpointSlices = b
|
||||
}
|
||||
|
||||
env = os.Getenv(mirrorDestInterface)
|
||||
@@ -635,37 +628,5 @@ func ParseEnvironment(c *Config) error {
|
||||
c.BackendHealthCheckInterval = int(i)
|
||||
}
|
||||
|
||||
env = os.Getenv(healthCheckPort)
|
||||
if env != "" {
|
||||
i, err := strconv.ParseInt(env, 10, 32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if i < 1024 {
|
||||
return fmt.Errorf("health check port should be > 1024")
|
||||
}
|
||||
c.HealthCheckPort = int(i)
|
||||
}
|
||||
|
||||
env = os.Getenv(enableUPNP)
|
||||
if env != "" {
|
||||
b, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.EnableUPNP = b
|
||||
}
|
||||
|
||||
if env = os.Getenv(egressClean); env == "" {
|
||||
env = os.Getenv(strings.ToUpper(egressClean))
|
||||
}
|
||||
if env != "" {
|
||||
b, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.EgressClean = b
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -39,6 +39,9 @@ const (
|
||||
// vipServicesInterface - defines the interface that the service vips should bind too
|
||||
vipServicesInterface = "vip_servicesinterface"
|
||||
|
||||
// vipCidr - defines the cidr that the vip will use (for BGP)
|
||||
vipCidr = "vip_cidr"
|
||||
|
||||
// vipSubnet - defines the subnet that the vip will use
|
||||
vipSubnet = "vip_subnet"
|
||||
|
||||
@@ -83,6 +86,18 @@ const (
|
||||
// vipStartLeader - will start this instance as the leader of the cluster
|
||||
vipStartLeader = "vip_startleader"
|
||||
|
||||
// vipPacket defines that the packet API will be used for EIP
|
||||
vipPacket = "vip_packet"
|
||||
|
||||
// vipPacketProject defines which project within Packet to use
|
||||
vipPacketProject = "vip_packetproject"
|
||||
|
||||
// vipPacketProjectID defines which projectID within Packet to use
|
||||
vipPacketProjectID = "vip_packetprojectid"
|
||||
|
||||
// providerConfig defines a path to a configuration that should be parsed
|
||||
providerConfig = "provider_config"
|
||||
|
||||
// bgpEnable defines if BGP should be enabled
|
||||
bgpEnable = "bgp_enable"
|
||||
// bgpRouterID defines the routerID for the BGP server
|
||||
@@ -110,22 +125,6 @@ const (
|
||||
// bgpKeepaliveInterval defines bgp timers keepalive interval
|
||||
bgpKeepaliveInterval = "bgp_keepalive_interval"
|
||||
|
||||
// zebraEnable defines if Zebra integraton should be enabled
|
||||
zebraEnable = "zebra_enable"
|
||||
// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
|
||||
zebraURL = "zebra_url"
|
||||
// zebraVersion specifies Zebra API Version
|
||||
zebraVersion = "zebra_version"
|
||||
// zebraSoftwareName specifies Software Name for Zebra
|
||||
zebraSoftwareName = "zebra_software_name"
|
||||
|
||||
// mpbgpNexthop defines MPBGP mode
|
||||
mpbgpNexthop = "mpbgp_nexthop"
|
||||
// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
|
||||
mpbgpIPv4 = "mpbgp_ipv4"
|
||||
// mpbgpIPv6 defines fixed IPv6 to be used with MPBGP
|
||||
mpbgpIPv6 = "mpbgp_ipv6"
|
||||
|
||||
// vipWireguard - defines if wireguard will be used for vips
|
||||
vipWireguard = "vip_wireguard" //nolint
|
||||
|
||||
@@ -212,8 +211,8 @@ const (
|
||||
// disableServiceUpdates disables service updating
|
||||
disableServiceUpdates = "disable_service_updates"
|
||||
|
||||
// enableEndpoints enables use of Endpoints instead of EndpointSlices
|
||||
enableEndpoints = "enable_endpoints"
|
||||
// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
|
||||
enableEndpointSlices = "enable_endpointslices"
|
||||
|
||||
// mirrorDestInterface is the network interface where all traffics that go through service interface
|
||||
// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
|
||||
@@ -225,13 +224,4 @@ const (
|
||||
|
||||
// backendHealthCheckInterval Interval in seconds for checking backend health.
|
||||
backendHealthCheckInterval = "backend_health_check_interval"
|
||||
|
||||
// healthCheckPort, if set to non-zero will be the port the health check will listen on
|
||||
healthCheckPort = "health_check_port"
|
||||
|
||||
// enableUPNP enables UPNP functions
|
||||
enableUPNP = "enable_upnp"
|
||||
|
||||
// egressClean enables egress cleaning on kube-vip's start
|
||||
egressClean = "egress_clean"
|
||||
)
|
||||
|
||||
@@ -2,14 +2,11 @@ package kubevip
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
|
||||
appv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
|
||||
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
|
||||
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
|
||||
@@ -17,43 +14,11 @@ import (
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
// TransformApplyObjectToManifest transforms an apply object into a normal Kubernetes manifest
|
||||
func TransformApplyObjectToManifest(applyObject interface{}) string {
|
||||
// Convert the apply object to an unstructured object
|
||||
unstructuredObj := &unstructured.Unstructured{}
|
||||
err := runtime.DefaultUnstructuredConverter.FromUnstructured(applyConfigToMap(applyObject), unstructuredObj)
|
||||
if err != nil {
|
||||
log.Fatalf("Error converting apply object to unstructured: %v", err)
|
||||
}
|
||||
|
||||
// Marshal the unstructured object into YAML
|
||||
yamlData, err := yaml.Marshal(unstructuredObj.Object)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshaling unstructured object to YAML: %v", err)
|
||||
}
|
||||
|
||||
return string(yamlData)
|
||||
}
|
||||
|
||||
// Helper function to convert apply configuration to a map
|
||||
func applyConfigToMap(applyConfig interface{}) map[string]interface{} {
|
||||
data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(applyConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Error converting apply configuration to map: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// GenerateSA will create the service account for kube-vip
|
||||
func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
|
||||
func GenerateSA() *applyCoreV1.ServiceAccountApplyConfiguration {
|
||||
kind := "ServiceAccount"
|
||||
name := "kube-vip"
|
||||
var namespace string
|
||||
if c.ServiceNamespace != "" {
|
||||
namespace = c.ServiceNamespace
|
||||
} else {
|
||||
namespace = metav1.NamespaceSystem
|
||||
}
|
||||
namespace := "kube-system"
|
||||
newManifest := &applyCoreV1.ServiceAccountApplyConfiguration{
|
||||
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &corev1.SchemeGroupVersion.Version, Kind: &kind},
|
||||
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
|
||||
@@ -65,31 +30,15 @@ func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
|
||||
}
|
||||
|
||||
// GenerateCR will generate the Cluster role for kube-vip
|
||||
func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
|
||||
var kind, name string
|
||||
var namespace *string
|
||||
if role {
|
||||
kind = "Role"
|
||||
name = "kube-vip"
|
||||
if c.ServiceNamespace != "" {
|
||||
namespace = &c.ServiceNamespace
|
||||
} else {
|
||||
// If the namespace is empty then we need to set it to the system namespace
|
||||
copiedNamespace := metav1.NamespaceSystem
|
||||
namespace = &copiedNamespace
|
||||
}
|
||||
|
||||
} else {
|
||||
kind = "ClusterRole"
|
||||
name = "system:kube-vip-role"
|
||||
|
||||
}
|
||||
func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
|
||||
name := "system:kube-vip-role"
|
||||
roleRefKind := "ClusterRole"
|
||||
apiVersion := "rbac.authorization.k8s.io/v1"
|
||||
newManifest := &applyRbacV1.RoleApplyConfiguration{
|
||||
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
|
||||
|
||||
newManifest := &applyRbacV1.ClusterRoleApplyConfiguration{
|
||||
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &roleRefKind},
|
||||
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
|
||||
Name: &name,
|
||||
Namespace: namespace,
|
||||
Name: &name,
|
||||
},
|
||||
Rules: []applyRbacV1.PolicyRuleApplyConfiguration{
|
||||
{
|
||||
@@ -100,7 +49,7 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"services", "endpoints"},
|
||||
Verbs: []string{"list", "get", "watch", "update"},
|
||||
Verbs: []string{"list", "get", "watch", "endoints"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
@@ -112,54 +61,38 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
|
||||
Resources: []string{"leases"},
|
||||
Verbs: []string{"list", "get", "watch", "update", "create"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"discovery.k8s.io"},
|
||||
Resources: []string{"endpointslices"},
|
||||
Verbs: []string{"list", "get", "watch", "update"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"pods"},
|
||||
Verbs: []string{"list"},
|
||||
},
|
||||
},
|
||||
}
|
||||
return newManifest
|
||||
}
|
||||
|
||||
// GenerateCRB will generate the clusterRoleBinding or rolebinding
|
||||
func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountApplyConfiguration, crCfg *applyRbacV1.RoleApplyConfiguration) *applyRbacV1.RoleBindingApplyConfiguration {
|
||||
// GenerateCRB will generate the clusterRoleBinding
|
||||
func GenerateCRB() *applyRbacV1.ClusterRoleBindingApplyConfiguration {
|
||||
kind := "ClusterRoleBinding"
|
||||
apiVersion := "rbac.authorization.k8s.io/v1"
|
||||
subjectKind := "ServiceAccount"
|
||||
apiGroup := "rbac.authorization.k8s.io"
|
||||
var kind, bindName string
|
||||
var namespace, objectNamespace *string
|
||||
if rolebinding {
|
||||
kind = "RoleBinding"
|
||||
bindName = "kube-vip"
|
||||
namespace = nil
|
||||
objectNamespace = saCfg.Namespace
|
||||
} else {
|
||||
kind = "ClusterRoleBinding"
|
||||
bindName = "system:kube-vip-binding"
|
||||
namespace = saCfg.Namespace
|
||||
objectNamespace = nil
|
||||
}
|
||||
newManifest := &applyRbacV1.RoleBindingApplyConfiguration{
|
||||
roleRefKind := "ClusterRole"
|
||||
roleRefName := "system:kube-vip-role"
|
||||
name := "kube-vip"
|
||||
bindName := "system:kube-vip-role-binding"
|
||||
namespace := "kube-system"
|
||||
|
||||
newManifest := &applyRbacV1.ClusterRoleBindingApplyConfiguration{
|
||||
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
|
||||
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
|
||||
Name: &bindName,
|
||||
Namespace: objectNamespace,
|
||||
Name: &bindName,
|
||||
},
|
||||
RoleRef: &applyRbacV1.RoleRefApplyConfiguration{
|
||||
APIGroup: &apiGroup,
|
||||
Kind: crCfg.Kind,
|
||||
Name: crCfg.Name,
|
||||
Kind: &roleRefKind,
|
||||
Name: &roleRefName,
|
||||
},
|
||||
Subjects: []applyRbacV1.SubjectApplyConfiguration{
|
||||
{
|
||||
Kind: saCfg.Kind,
|
||||
Name: saCfg.Name,
|
||||
Namespace: namespace,
|
||||
Kind: &subjectKind,
|
||||
Name: &name,
|
||||
Namespace: &namespace,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -228,6 +161,18 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
newEnvironment = append(newEnvironment, svcInterface...)
|
||||
}
|
||||
|
||||
// If a CIDR is used add it to the manifest
|
||||
if c.VIPCIDR != "" {
|
||||
// build environment variables
|
||||
cidr := []corev1.EnvVar{
|
||||
{
|
||||
Name: vipCidr,
|
||||
Value: c.VIPCIDR,
|
||||
},
|
||||
}
|
||||
newEnvironment = append(newEnvironment, cidr...)
|
||||
}
|
||||
|
||||
// If a subnet is required for the VIP
|
||||
if c.VIPSubnet != "" {
|
||||
// build environment variables
|
||||
@@ -386,6 +331,40 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
|
||||
}
|
||||
|
||||
// If we're specifying a configuration
|
||||
if c.ProviderConfig != "" {
|
||||
provider := []corev1.EnvVar{
|
||||
{
|
||||
Name: providerConfig,
|
||||
Value: c.ProviderConfig,
|
||||
},
|
||||
}
|
||||
newEnvironment = append(newEnvironment, provider...)
|
||||
}
|
||||
|
||||
// If Equinix Metal is enabled then add it to the manifest
|
||||
if c.EnableMetal {
|
||||
packet := []corev1.EnvVar{
|
||||
{
|
||||
Name: vipPacket,
|
||||
Value: strconv.FormatBool(c.EnableMetal),
|
||||
},
|
||||
{
|
||||
Name: vipPacketProject,
|
||||
Value: c.MetalProject,
|
||||
},
|
||||
{
|
||||
Name: vipPacketProjectID,
|
||||
Value: c.MetalProjectID,
|
||||
},
|
||||
{
|
||||
Name: "PACKET_AUTH_TOKEN",
|
||||
Value: c.MetalAPIKey,
|
||||
},
|
||||
}
|
||||
newEnvironment = append(newEnvironment, packet...)
|
||||
}
|
||||
|
||||
// Detect and enable wireguard mode
|
||||
if c.EnableWireguard {
|
||||
wireguard := []corev1.EnvVar{
|
||||
@@ -407,7 +386,8 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
}
|
||||
newEnvironment = append(newEnvironment, routingtable...)
|
||||
}
|
||||
// If BGP
|
||||
|
||||
// If BGP, but we're not using Equinix Metal
|
||||
if c.EnableBGP {
|
||||
bgp := []corev1.EnvVar{
|
||||
{
|
||||
@@ -417,9 +397,8 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
}
|
||||
newEnvironment = append(newEnvironment, bgp...)
|
||||
}
|
||||
|
||||
// If BGP
|
||||
if c.EnableBGP {
|
||||
// If BGP, but we're not using Equinix Metal
|
||||
if c.EnableBGP && !c.EnableMetal {
|
||||
bgpConfig := []corev1.EnvVar{
|
||||
{
|
||||
Name: bgpRouterID,
|
||||
@@ -521,10 +500,10 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
}
|
||||
newEnvironment = append(newEnvironment, prometheus...)
|
||||
|
||||
if c.EnableEndpoints {
|
||||
if c.EnableEndpointSlices {
|
||||
newEnvironment = append(newEnvironment, corev1.EnvVar{
|
||||
Name: enableEndpoints,
|
||||
Value: strconv.FormatBool(c.EnableEndpoints),
|
||||
Name: enableEndpointSlices,
|
||||
Value: strconv.FormatBool(c.EnableEndpointSlices),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -549,16 +528,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
newEnvironment = append(newEnvironment, mdif...)
|
||||
}
|
||||
|
||||
if c.HealthCheckPort != 0 {
|
||||
healthPort := []corev1.EnvVar{
|
||||
{
|
||||
Name: healthCheckPort,
|
||||
Value: fmt.Sprintf("%d", c.HealthCheckPort),
|
||||
},
|
||||
}
|
||||
newEnvironment = append(newEnvironment, healthPort...)
|
||||
}
|
||||
|
||||
var securityContext *corev1.SecurityContext
|
||||
if c.LoadBalancerForwardingMethod == "masquerade" {
|
||||
var privileged = true
|
||||
@@ -572,9 +541,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
"NET_ADMIN",
|
||||
"NET_RAW",
|
||||
},
|
||||
Drop: []corev1.Capability{
|
||||
"ALL",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -633,6 +599,26 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
|
||||
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
|
||||
}
|
||||
|
||||
if c.ProviderConfig != "" {
|
||||
providerConfigMount := corev1.VolumeMount{
|
||||
Name: "cloud-sa-volume",
|
||||
MountPath: "/etc/cloud-sa",
|
||||
ReadOnly: true,
|
||||
}
|
||||
newManifest.Spec.Containers[0].VolumeMounts = append(newManifest.Spec.Containers[0].VolumeMounts, providerConfigMount)
|
||||
|
||||
providerConfigVolume := corev1.Volume{
|
||||
Name: "cloud-sa-volume",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: "metal-cloud-config",
|
||||
},
|
||||
},
|
||||
}
|
||||
newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, providerConfigVolume)
|
||||
|
||||
}
|
||||
|
||||
return newManifest
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,7 @@ package kubevip
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
@@ -45,8 +44,8 @@ func isValidInterface(iface string) error {
|
||||
// userspace has set operational state. Interface must be considered for user
|
||||
// data as setting operational state has not been implemented in every driver."
|
||||
if attrs.OperState == netlink.OperUnknown {
|
||||
log.Warn(
|
||||
"the status of the interface is unknown. Ensure your interface is ready to accept traffic, if so you can safely ignore this message", "interface",
|
||||
log.Warningf(
|
||||
"the status of the interface %s is unknown. Ensure your interface is ready to accept traffic, if so you can safely ignore this message",
|
||||
iface,
|
||||
)
|
||||
} else if attrs.OperState != netlink.OperUp {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
// Config defines all of the settings for the Kube-Vip Pod
|
||||
type Config struct {
|
||||
// Logging, settings
|
||||
Logging int32 `yaml:"logging"`
|
||||
Logging int `yaml:"logging"`
|
||||
|
||||
// EnableARP, will use ARP to advertise the VIP address
|
||||
EnableARP bool `yaml:"enableARP"`
|
||||
@@ -76,11 +76,14 @@ type Config struct {
|
||||
// VipSubnet is the Subnet that is applied to the VIP
|
||||
VIPSubnet string `yaml:"vipSubnet"`
|
||||
|
||||
// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
|
||||
VIPCIDR string `yaml:"vipCidr"`
|
||||
|
||||
// Address is the IP or DNS Name to use as a VirtualIP
|
||||
Address string `yaml:"address"`
|
||||
|
||||
// Listen port for the VirtualIP
|
||||
Port uint16 `yaml:"port"`
|
||||
Port int `yaml:"port"`
|
||||
|
||||
// Namespace will define which namespace the control plane pods will run in
|
||||
Namespace string `yaml:"namespace"`
|
||||
@@ -110,7 +113,7 @@ type Config struct {
|
||||
EnableLoadBalancer bool `yaml:"enableLoadBalancer"`
|
||||
|
||||
// Listen port for the IPVS Service
|
||||
LoadBalancerPort uint16 `yaml:"lbPort"`
|
||||
LoadBalancerPort int `yaml:"lbPort"`
|
||||
|
||||
// Forwarding method for the IPVS Service
|
||||
LoadBalancerForwardingMethod string `yaml:"lbForwardingMethod"`
|
||||
@@ -132,6 +135,21 @@ type Config struct {
|
||||
BGPPeerConfig bgp.Peer
|
||||
BGPPeers []string
|
||||
|
||||
// EnableMetal, will use the metal API to update the EIP <-> VIP (if BGP is enabled then BGP will be used)
|
||||
EnableMetal bool `yaml:"enableMetal"`
|
||||
|
||||
// MetalAPIKey, is the API token used to authenticate to the API
|
||||
MetalAPIKey string
|
||||
|
||||
// MetalProject, is the name of a particular defined project
|
||||
MetalProject string
|
||||
|
||||
// MetalProjectID, is the name of a particular defined project
|
||||
MetalProjectID string
|
||||
|
||||
// ProviderConfig, is the path to a provider configuration file
|
||||
ProviderConfig string
|
||||
|
||||
// LoadBalancers are the various services we can load balance over
|
||||
LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
|
||||
|
||||
@@ -161,8 +179,8 @@ type Config struct {
|
||||
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
|
||||
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
|
||||
|
||||
// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
|
||||
EnableEndpoints bool `yaml:"enableEndpoints"`
|
||||
// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
|
||||
EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
|
||||
|
||||
// MirrorDestInterface is the network interface where all traffics that go through service interface
|
||||
// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
|
||||
@@ -177,15 +195,6 @@ type Config struct {
|
||||
|
||||
// LoInterfaceGlobalScope, if true will set global scope when using the lo interface, otherwise a host scope will be used
|
||||
LoInterfaceGlobalScope bool `yaml:"loInterfaceGlobalScope"`
|
||||
|
||||
// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
|
||||
HealthCheckPort int `yaml:"healthCheckPort"`
|
||||
|
||||
// EnableUPNP, enables UPNP functions
|
||||
EnableUPNP bool `yaml:"enableUPNP"`
|
||||
|
||||
// EgressClean, enables egress cleaning on Kube-vip's start
|
||||
EgressClean bool `yaml:"egressClean"`
|
||||
}
|
||||
|
||||
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection
|
||||
|
||||
@@ -1,23 +1,17 @@
|
||||
package loadbalancer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/cloudflare/ipvs"
|
||||
"github.com/cloudflare/ipvs/netmask"
|
||||
"github.com/kube-vip/kube-vip/pkg/backend"
|
||||
"github.com/kube-vip/kube-vip/pkg/sysctl"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/vishvananda/netlink"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -45,51 +39,44 @@ const (
|
||||
type IPVSLoadBalancer struct {
|
||||
client ipvs.Client
|
||||
loadBalancerService ipvs.Service
|
||||
Port uint16
|
||||
Port int
|
||||
forwardingMethod ipvs.ForwardType
|
||||
backendMap backend.Map
|
||||
interval int
|
||||
lock sync.Mutex
|
||||
stop chan struct{}
|
||||
networkInterface string
|
||||
leaderCancel context.CancelFunc
|
||||
signal chan os.Signal
|
||||
address string
|
||||
family ipvs.AddressFamily
|
||||
}
|
||||
|
||||
func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
|
||||
log.Info("Starting IPVS LoadBalancer", "address", address)
|
||||
|
||||
func NewIPVSLB(address string, port int, forwardingMethod string, backendHealthCheckInterval int) (*IPVSLoadBalancer, error) {
|
||||
// Create IPVS client
|
||||
c, err := ipvs.New()
|
||||
if err != nil {
|
||||
log.Error("ensure IPVS kernel modules are loaded")
|
||||
log.Error("Error starting IPVS", "err", err)
|
||||
panic("")
|
||||
log.Errorf("ensure IPVS kernel modules are loaded")
|
||||
log.Fatalf("Error starting IPVS [%v]", err)
|
||||
}
|
||||
i, err := c.Info()
|
||||
if err != nil {
|
||||
log.Error("ensure IPVS kernel modules are loaded")
|
||||
log.Error("Error retrieving IPVS info", "err", err)
|
||||
if errors.Is(err, os.ErrPermission) {
|
||||
log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
|
||||
}
|
||||
panic("")
|
||||
log.Errorf("ensure IPVS kernel modules are loaded")
|
||||
log.Fatalf("Error getting IPVS version [%v]", err)
|
||||
}
|
||||
log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
|
||||
|
||||
ip, family := ipAndFamily(address)
|
||||
log.Infof("IPVS Loadbalancer enabled for %d.%d.%d", i.Version[0], i.Version[1], i.Version[2])
|
||||
|
||||
if strings.ToLower(forwardingMethod) == "masquerade" {
|
||||
enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
|
||||
if family == ipvs.INET6 {
|
||||
enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
|
||||
} else {
|
||||
enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
|
||||
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/vs/conntrack", "1")
|
||||
if err != nil {
|
||||
log.Fatalf("Error ensuring net.ipv4.vs.conntrack enabled [%v]", err)
|
||||
}
|
||||
log.Infof("sysctl set net.ipv4.vs.conntrack to 1")
|
||||
|
||||
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/ip_forward", "1")
|
||||
if err != nil {
|
||||
log.Fatalf("Error ensuring net.ipv4.ip_forward enabled [%v]", err)
|
||||
}
|
||||
log.Infof("sysctl set net.ipv4.ip_forward to 1")
|
||||
}
|
||||
|
||||
ip, family := ipAndFamily(address)
|
||||
|
||||
netMask := netmask.MaskFrom(31, 32) // For ipv4
|
||||
if family == ipvs.INET6 {
|
||||
netMask = netmask.MaskFrom(128, 128) // For ipv6
|
||||
@@ -100,7 +87,7 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
|
||||
Netmask: netMask,
|
||||
Family: family,
|
||||
Protocol: ipvs.TCP,
|
||||
Port: port,
|
||||
Port: uint16(port),
|
||||
Address: ip,
|
||||
Scheduler: ROUNDROBIN,
|
||||
}
|
||||
@@ -119,7 +106,7 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
|
||||
m = ipvs.Bypass
|
||||
default:
|
||||
m = ipvs.Local
|
||||
log.Warn("unknown forwarding method. Defaulting to Local")
|
||||
log.Warnf("unknown forwarding method. Defaulting to Local")
|
||||
}
|
||||
|
||||
if backendHealthCheckInterval <= 0 {
|
||||
@@ -133,33 +120,17 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
|
||||
forwardingMethod: m,
|
||||
interval: backendHealthCheckInterval,
|
||||
backendMap: make(backend.Map),
|
||||
stop: make(chan struct{}),
|
||||
networkInterface: networkInterface,
|
||||
leaderCancel: leaderCancel,
|
||||
signal: signal,
|
||||
address: address,
|
||||
family: family,
|
||||
}
|
||||
|
||||
go lb.healthCheck()
|
||||
if strings.ToLower(forwardingMethod) == "masquerade" {
|
||||
go lb.healthCheck()
|
||||
}
|
||||
|
||||
// Return our created load-balancer
|
||||
return lb, nil
|
||||
}
|
||||
|
||||
func enableProcSys(path, name string) {
|
||||
isSet, err := sysctl.EnableProcSys(path)
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
|
||||
panic("")
|
||||
}
|
||||
if isSet {
|
||||
log.Info(fmt.Sprintf("sysctl set %s to 1", name))
|
||||
}
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
|
||||
log.Info("Stopping IPVS LoadBalancer", "address", lb.address)
|
||||
close(lb.stop)
|
||||
err := lb.client.RemoveService(lb.loadBalancerService)
|
||||
if err != nil {
|
||||
@@ -168,31 +139,15 @@ func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
|
||||
isLocal := false
|
||||
var err error
|
||||
|
||||
// Discard backend if it is of different IP family than LB address.
|
||||
if _, family := ipAndFamily(address); family != lb.family {
|
||||
return nil
|
||||
}
|
||||
|
||||
if lb.forwardingMethod == ipvs.Local {
|
||||
log.Info("checking if backend is local", "addr", address)
|
||||
isLocal, err = lb.isLocal(address)
|
||||
if err != nil {
|
||||
log.Error("checking if backend is local", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
backend := backend.Entry{Addr: address, Port: port, IsLocal: isLocal}
|
||||
func (lb *IPVSLoadBalancer) AddBackend(address string, port int) error {
|
||||
backend := backend.Entry{Addr: address, Port: port}
|
||||
|
||||
lb.lock.Lock()
|
||||
defer lb.lock.Unlock()
|
||||
if _, ok := lb.backendMap[backend]; !ok {
|
||||
isHealth := backend.Check()
|
||||
if isHealth {
|
||||
err := lb.addBackend(address, port)
|
||||
err := lb.addBackend(backend)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -202,19 +157,18 @@ func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
|
||||
backend := backend.Entry{Addr: address, Port: port}
|
||||
func (lb *IPVSLoadBalancer) addBackend(backend backend.Entry) error {
|
||||
// Check if this is the first backend
|
||||
backends, err := lb.client.Destinations(lb.loadBalancerService)
|
||||
if err != nil && strings.Contains(err.Error(), "file does not exist") {
|
||||
log.Error("querying backends", "err", err)
|
||||
log.Errorf("Error querying backends %s", err)
|
||||
}
|
||||
// If this is our first backend, then we can create the load-balancer service and add a backend
|
||||
if len(backends) == 0 {
|
||||
err = lb.client.CreateService(lb.loadBalancerService)
|
||||
// If we've an error it could be that the IPVS lb instance has been left from a previous leadership
|
||||
if err != nil && strings.Contains(err.Error(), "file exists") {
|
||||
log.Warn("load balancer for API server already exists, attempting to remove and re-create")
|
||||
log.Warnf("load balancer for API server already exists, attempting to remove and re-create")
|
||||
err = lb.client.RemoveService(lb.loadBalancerService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error re-creating IPVS service: %v", err)
|
||||
@@ -225,12 +179,10 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
|
||||
}
|
||||
} else if err != nil {
|
||||
// Fatal error at this point as IPVS is probably not working
|
||||
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
|
||||
log.Error("IPVS service", "err", err)
|
||||
panic("")
|
||||
|
||||
log.Errorf("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
|
||||
log.Fatalf("IPVS service error: %v", err)
|
||||
}
|
||||
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
|
||||
log.Infof("Created Load-Balancer services on [%s:%d]", lb.addrString(), lb.Port)
|
||||
}
|
||||
|
||||
ip, family := ipAndFamily(backend.Addr)
|
||||
@@ -243,7 +195,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
|
||||
|
||||
dst := ipvs.Destination{
|
||||
Address: ip,
|
||||
Port: backend.Port,
|
||||
Port: uint16(backend.Port),
|
||||
Family: family,
|
||||
Weight: 1,
|
||||
FwdMethod: lb.forwardingMethod,
|
||||
@@ -259,12 +211,12 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
|
||||
// file exists is fine, we will just return at this point
|
||||
return nil
|
||||
}
|
||||
log.Info("backend added", "src addr", lb.addrString(), "src port", lb.Port, "dst addr", backend.Addr, "dst port", backend.Port)
|
||||
log.Infof("Added backend for [%s:%d] on [%s:%d]", lb.addrString(), lb.Port, backend.Addr, backend.Port)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) RemoveBackend(address string, port uint16) error {
|
||||
func (lb *IPVSLoadBalancer) RemoveBackend(address string, port int) error {
|
||||
backend := backend.Entry{Addr: address, Port: port}
|
||||
|
||||
lb.lock.Lock()
|
||||
@@ -281,7 +233,7 @@ func (lb *IPVSLoadBalancer) RemoveBackend(address string, port uint16) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) removeBackend(address string, port uint16) error {
|
||||
func (lb *IPVSLoadBalancer) removeBackend(address string, port int) error {
|
||||
ip, family := ipAndFamily(address)
|
||||
if family != lb.loadBalancerService.Family {
|
||||
return nil
|
||||
@@ -289,12 +241,12 @@ func (lb *IPVSLoadBalancer) removeBackend(address string, port uint16) error {
|
||||
|
||||
dst := ipvs.Destination{
|
||||
Address: ip,
|
||||
Port: port,
|
||||
Port: uint16(port),
|
||||
Family: family,
|
||||
Weight: 1,
|
||||
}
|
||||
err := lb.client.RemoveDestination(lb.loadBalancerService, dst)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error removing backend: %v", err)
|
||||
}
|
||||
return nil
|
||||
@@ -315,77 +267,29 @@ func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
|
||||
func (lb *IPVSLoadBalancer) healthCheck() {
|
||||
backend.Watch(func() {
|
||||
lb.lock.Lock()
|
||||
defer lb.lock.Unlock()
|
||||
for backend, oldStatus := range lb.backendMap {
|
||||
newStatus := backend.Check()
|
||||
if newStatus {
|
||||
// old status -> health
|
||||
if !oldStatus {
|
||||
err := lb.addBackend(backend.Addr, backend.Port)
|
||||
err := lb.AddBackend(backend.Addr, backend.Port)
|
||||
if err != nil {
|
||||
log.Error("add backend", "err", err)
|
||||
log.Errorf("failed to add backend: %s", err)
|
||||
}
|
||||
lb.backendMap[backend] = newStatus
|
||||
}
|
||||
} else {
|
||||
// old status -> not health
|
||||
if oldStatus {
|
||||
log.Info("healthCheck failed - removing backend", "address", backend.Addr, "port", backend.Port)
|
||||
log.Infof("healthCheck failed for backend %s:%d, attempting to remove from load balancer", backend.Addr, backend.Port)
|
||||
err := lb.removeBackend(backend.Addr, backend.Port)
|
||||
if err != nil {
|
||||
log.Error("failed to remove backend", "address", backend.Addr, "port", backend.Port, "err", err)
|
||||
log.Errorf("failed to remove backend %s:%d: %s", backend.Addr, backend.Port, err)
|
||||
}
|
||||
lb.backendMap[backend] = newStatus
|
||||
}
|
||||
if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
|
||||
if lb.signal != nil {
|
||||
close(lb.signal)
|
||||
}
|
||||
|
||||
if lb.leaderCancel != nil {
|
||||
lb.leaderCancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lb.lock.Unlock()
|
||||
}, lb.interval, lb.stop)
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
|
||||
link, err := netlink.LinkByName(lb.networkInterface)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("getting link '%s': %w", lb.networkInterface, err)
|
||||
}
|
||||
|
||||
family := netlink.FAMILY_V6
|
||||
if vip.IsIPv4(address) {
|
||||
family = netlink.FAMILY_V4
|
||||
}
|
||||
|
||||
target := net.ParseIP(address)
|
||||
if target == nil {
|
||||
return false, fmt.Errorf("address '%s' is not a valid IP address", address)
|
||||
}
|
||||
|
||||
addrs, err := netlink.AddrList(link, family)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("listing addresses for link '%s': %w", lb.networkInterface, err)
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
if addr.IP.Equal(target) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (lb *IPVSLoadBalancer) localBackendExists() bool {
|
||||
for backend, isHealthy := range lb.backendMap {
|
||||
if backend.IsLocal && isHealthy {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,56 +1,54 @@
|
||||
package instance
|
||||
package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"log/slog"
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/cluster"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/sysctl"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
// Instance defines an instance of everything needed to manage vips
|
||||
type Instance struct {
|
||||
// Virtual IP / Load Balancer configuration
|
||||
VIPConfigs []*kubevip.Config
|
||||
vipConfigs []*kubevip.Config
|
||||
|
||||
// cluster instances
|
||||
Clusters []*cluster.Cluster
|
||||
clusters []*cluster.Cluster
|
||||
|
||||
// Service uses DHCP
|
||||
IsDHCP bool
|
||||
DHCPInterface string
|
||||
DHCPInterfaceHwaddr string
|
||||
DHCPInterfaceIP string
|
||||
DHCPHostname string
|
||||
DHCPClient *vip.DHCPClient
|
||||
isDHCP bool
|
||||
dhcpInterface string
|
||||
dhcpInterfaceHwaddr string
|
||||
dhcpInterfaceIP string
|
||||
dhcpHostname string
|
||||
dhcpClient *vip.DHCPClient
|
||||
|
||||
// External Gateway IP the service is forwarded from
|
||||
UPNPGatewayIPs []string
|
||||
upnpGatewayIPs []string
|
||||
|
||||
// Kubernetes service mapping
|
||||
ServiceSnapshot *v1.Service
|
||||
VIPs []string
|
||||
UID string
|
||||
ExternalPorts []Port
|
||||
|
||||
serviceSnapshot *v1.Service
|
||||
}
|
||||
|
||||
type Port struct {
|
||||
Port uint16
|
||||
Port int32
|
||||
Type string
|
||||
}
|
||||
|
||||
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
|
||||
instanceAddresses := FetchServiceAddresses(svc)
|
||||
//instanceUID := string(svc.UID)
|
||||
func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
|
||||
instanceAddresses := fetchServiceAddresses(svc)
|
||||
instanceUID := string(svc.UID)
|
||||
|
||||
var newVips []*kubevip.Config
|
||||
var link netlink.Link
|
||||
@@ -59,14 +57,14 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
for _, address := range instanceAddresses {
|
||||
// Detect if we're using a specific interface for services
|
||||
var svcInterface string
|
||||
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
|
||||
svcInterface = svc.Annotations[serviceInterface] // If the service has a specific interface defined, then use it
|
||||
if svcInterface == kubevip.Auto {
|
||||
link, err = autoFindInterface(address)
|
||||
if err != nil {
|
||||
log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
|
||||
log.Errorf("failed to automatically discover network interface for annotated IP address [%s] with error: %s", address, err.Error())
|
||||
} else {
|
||||
if link == nil {
|
||||
log.Error("automatically discover network interface for annotated IP address", "address", address)
|
||||
log.Errorf("failed to automatically discover network interface for annotated IP address [%s]", address)
|
||||
}
|
||||
}
|
||||
if link == nil {
|
||||
@@ -81,9 +79,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
case kubevip.Auto:
|
||||
link, err = autoFindInterface(address)
|
||||
if err != nil {
|
||||
log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
|
||||
log.Errorf("failed to automatically discover network interface for IP address [%s] with error: %s - defaulting to: %s", address, err.Error(), config.Interface)
|
||||
} else if link == nil {
|
||||
log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
|
||||
log.Errorf("failed to automatically discover network interface for IP address [%s] - defaulting to: %s", address, config.Interface)
|
||||
}
|
||||
svcInterface = getAutoInterfaceName(link, config.Interface)
|
||||
case "":
|
||||
@@ -102,7 +100,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
}
|
||||
}
|
||||
|
||||
cidrs := vip.Split(config.VIPSubnet)
|
||||
cidrs := vip.Split(config.VIPCIDR)
|
||||
|
||||
ipv4AutoSubnet := false
|
||||
ipv6AutoSubnet := false
|
||||
@@ -127,7 +125,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
|
||||
}
|
||||
} else {
|
||||
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
|
||||
if cidrs[0] != "" {
|
||||
subnet = cidrs[0]
|
||||
} else {
|
||||
subnet = "32"
|
||||
@@ -140,7 +138,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
|
||||
}
|
||||
} else {
|
||||
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
|
||||
if len(cidrs) > 1 {
|
||||
subnet = cidrs[1]
|
||||
} else {
|
||||
subnet = "128"
|
||||
@@ -148,6 +146,8 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
}
|
||||
}
|
||||
|
||||
//log.Info("new instance", "svc", *svc, "interface", svcInterface)
|
||||
|
||||
// Generate new Virtual IP configuration
|
||||
newVips = append(newVips, &kubevip.Config{
|
||||
VIP: address,
|
||||
@@ -155,7 +155,8 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
SingleNode: true,
|
||||
EnableARP: config.EnableARP,
|
||||
EnableBGP: config.EnableBGP,
|
||||
VIPSubnet: subnet,
|
||||
VIPCIDR: subnet,
|
||||
VIPSubnet: config.VIPSubnet,
|
||||
EnableRoutingTable: config.EnableRoutingTable,
|
||||
RoutingTableID: config.RoutingTableID,
|
||||
RoutingTableType: config.RoutingTableType,
|
||||
@@ -173,27 +174,27 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
|
||||
// Create new service
|
||||
instance := &Instance{
|
||||
//UID: instanceUID,
|
||||
//VIPs: instanceAddresses,
|
||||
ServiceSnapshot: svc,
|
||||
UID: instanceUID,
|
||||
VIPs: instanceAddresses,
|
||||
serviceSnapshot: svc,
|
||||
}
|
||||
for _, port := range svc.Spec.Ports {
|
||||
instance.ExternalPorts = append(instance.ExternalPorts, Port{
|
||||
Port: port.Port,
|
||||
Type: string(port.Protocol),
|
||||
})
|
||||
}
|
||||
// for _, port := range svc.Spec.Ports {
|
||||
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
|
||||
// Port: uint16(port.Port), //nolint
|
||||
// Type: string(port.Protocol),
|
||||
// })
|
||||
// }
|
||||
|
||||
if svc.Annotations != nil {
|
||||
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
|
||||
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
|
||||
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
|
||||
instance.dhcpInterfaceHwaddr = svc.Annotations[hwAddrKey]
|
||||
instance.dhcpInterfaceIP = svc.Annotations[requestedIP]
|
||||
instance.dhcpHostname = svc.Annotations[loadbalancerHostname]
|
||||
}
|
||||
|
||||
configPorts := make([]kubevip.Port, 0)
|
||||
for _, p := range svc.Spec.Ports {
|
||||
for _, p := range instance.ExternalPorts {
|
||||
configPorts = append(configPorts, kubevip.Port{
|
||||
Type: string(p.Protocol),
|
||||
Type: p.Type,
|
||||
Port: int(p.Port),
|
||||
})
|
||||
}
|
||||
@@ -208,7 +209,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
vip.LoadBalancers = append(vip.LoadBalancers, newLB)
|
||||
}
|
||||
// Create Add configuration to the new service
|
||||
instance.VIPConfigs = newVips
|
||||
instance.vipConfigs = newVips
|
||||
|
||||
// If this was purposely created with the address 0.0.0.0,
|
||||
// we will create a macvlan on the main interface and a DHCP client
|
||||
@@ -219,20 +220,20 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case err := <-instance.DHCPClient.ErrorChannel():
|
||||
case err := <-instance.dhcpClient.ErrorChannel():
|
||||
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
|
||||
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
|
||||
case ip := <-instance.DHCPClient.IPChannel():
|
||||
instance.VIPConfigs[0].Interface = instance.DHCPInterface
|
||||
instance.VIPConfigs[0].VIP = ip
|
||||
instance.DHCPInterfaceIP = ip
|
||||
instance.serviceSnapshot.Namespace, instance.serviceSnapshot.Name, err)
|
||||
case ip := <-instance.dhcpClient.IPChannel():
|
||||
instance.vipConfigs[0].Interface = instance.dhcpInterface
|
||||
instance.vipConfigs[0].VIP = ip
|
||||
instance.dhcpInterfaceIP = ip
|
||||
}
|
||||
}
|
||||
|
||||
for _, vipConfig := range instance.VIPConfigs {
|
||||
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
|
||||
for _, vipConfig := range instance.vipConfigs {
|
||||
c, err := cluster.InitCluster(vipConfig, false)
|
||||
if err != nil {
|
||||
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
|
||||
log.Errorf("Failed to add Service %s/%s", svc.Namespace, svc.Name)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -240,8 +241,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
|
||||
c.Network[i].SetServicePorts(svc)
|
||||
}
|
||||
|
||||
instance.Clusters = append(instance.Clusters, c)
|
||||
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
|
||||
instance.clusters = append(instance.clusters, c)
|
||||
log.Infof("(svcs) adding VIP [%s] via %s for [%s/%s]", vipConfig.VIP, vipConfig.Interface, svc.Namespace, svc.Name)
|
||||
|
||||
}
|
||||
|
||||
return instance, nil
|
||||
@@ -305,24 +307,24 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
|
||||
}
|
||||
|
||||
func (i *Instance) startDHCP() error {
|
||||
if len(i.VIPConfigs) != 1 {
|
||||
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
|
||||
if len(i.vipConfigs) != 1 {
|
||||
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.vipConfigs))
|
||||
}
|
||||
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
|
||||
parent, err := netlink.LinkByName(i.vipConfigs[0].Interface)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
|
||||
}
|
||||
|
||||
// Generate name from UID
|
||||
interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
|
||||
interfaceName := fmt.Sprintf("vip-%s", i.UID[0:8])
|
||||
|
||||
// Check if the interface doesn't exist first
|
||||
iface, err := net.InterfaceByName(interfaceName)
|
||||
if err != nil {
|
||||
log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
|
||||
log.Infof("Creating new macvlan interface for DHCP [%s]", interfaceName)
|
||||
|
||||
hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
|
||||
if i.DHCPInterfaceHwaddr != "" && err != nil {
|
||||
hwaddr, err := net.ParseMAC(i.dhcpInterfaceHwaddr)
|
||||
if i.dhcpInterfaceHwaddr != "" && err != nil {
|
||||
return err
|
||||
} else if hwaddr == nil {
|
||||
hwaddr, err = net.ParseMAC(vip.GenerateMac())
|
||||
@@ -331,7 +333,7 @@ func (i *Instance) startDHCP() error {
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
|
||||
log.Infof("New interface [%s] mac is %s", interfaceName, hwaddr)
|
||||
mac := &netlink.Macvlan{
|
||||
LinkAttrs: netlink.LinkAttrs{
|
||||
Name: interfaceName,
|
||||
@@ -356,127 +358,31 @@ func (i *Instance) startDHCP() error {
|
||||
return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
|
||||
}
|
||||
} else {
|
||||
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
|
||||
log.Infof("Using existing macvlan interface for DHCP [%s]", interfaceName)
|
||||
}
|
||||
|
||||
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
|
||||
rpfilterSetting := "0"
|
||||
|
||||
// Check if we need to set an override rp_filter value for the interface
|
||||
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
|
||||
// Check the rp_filter value
|
||||
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
|
||||
if err != nil {
|
||||
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
|
||||
} else {
|
||||
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
|
||||
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
|
||||
} else {
|
||||
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
|
||||
if err != nil {
|
||||
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
|
||||
}
|
||||
var initRebootFlag bool
|
||||
if i.DHCPInterfaceIP != "" {
|
||||
if i.dhcpInterfaceIP != "" {
|
||||
initRebootFlag = true
|
||||
}
|
||||
|
||||
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
|
||||
client := vip.NewDHCPClient(iface, initRebootFlag, i.dhcpInterfaceIP)
|
||||
|
||||
// Add hostname to dhcp client if annotated
|
||||
if i.DHCPHostname != "" {
|
||||
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
|
||||
client.WithHostName(i.DHCPHostname)
|
||||
if i.dhcpHostname != "" {
|
||||
log.Infof("Hostname specified for dhcp lease: [%s] - [%s]", interfaceName, i.dhcpHostname)
|
||||
client.WithHostName(i.dhcpHostname)
|
||||
}
|
||||
|
||||
go client.Start()
|
||||
|
||||
// Set that DHCP is enabled
|
||||
i.IsDHCP = true
|
||||
i.isDHCP = true
|
||||
// Set the name of the interface so that it can be removed on Service deletion
|
||||
i.DHCPInterface = interfaceName
|
||||
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
|
||||
i.dhcpInterface = interfaceName
|
||||
i.dhcpInterfaceHwaddr = iface.HardwareAddr.String()
|
||||
// Add the client so that we can call it to stop function
|
||||
i.DHCPClient = client
|
||||
i.dhcpClient = client
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
|
||||
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
|
||||
// If the service has no status, return empty
|
||||
lbStatusAddresses := []string{}
|
||||
|
||||
if len(s.Status.LoadBalancer.Ingress) == 0 {
|
||||
return lbStatusAddresses
|
||||
}
|
||||
|
||||
for _, ingress := range s.Status.LoadBalancer.Ingress {
|
||||
if ingress.IP != "" {
|
||||
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
|
||||
}
|
||||
// TODO: Handle hostname if needed
|
||||
}
|
||||
|
||||
return lbStatusAddresses
|
||||
}
|
||||
|
||||
// FetchServiceAddresses tries to get the addresses from annotations
|
||||
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
|
||||
func FetchServiceAddresses(s *v1.Service) []string {
|
||||
annotationAvailable := false
|
||||
if s.Annotations != nil {
|
||||
|
||||
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
|
||||
ips := strings.Split(v, ",")
|
||||
var trimmedIPs []string
|
||||
for _, ip := range ips {
|
||||
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
|
||||
}
|
||||
return trimmedIPs
|
||||
}
|
||||
}
|
||||
|
||||
lbStatusAddresses := []string{}
|
||||
if !annotationAvailable {
|
||||
if len(s.Status.LoadBalancer.Ingress) > 0 {
|
||||
for _, ingress := range s.Status.LoadBalancer.Ingress {
|
||||
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
|
||||
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
|
||||
|
||||
if len(lbStatusAddresses) > 0 {
|
||||
for _, a := range lbStatusAddresses {
|
||||
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
|
||||
return []string{s.Spec.LoadBalancerIP}
|
||||
}
|
||||
}
|
||||
return lbStatusAddresses
|
||||
}
|
||||
|
||||
if s.Spec.LoadBalancerIP != "" {
|
||||
return []string{s.Spec.LoadBalancerIP}
|
||||
}
|
||||
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
|
||||
log.Debug("finding service", "UID", svc.UID)
|
||||
for i := range instances {
|
||||
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
|
||||
if instances[i].ServiceSnapshot.UID == svc.UID {
|
||||
return instances[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3,26 +3,24 @@ package manager
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/k8s"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/services"
|
||||
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
|
||||
"github.com/kube-vip/kube-vip/pkg/upnp"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
@@ -39,6 +37,11 @@ type Manager struct {
|
||||
// Manager services
|
||||
// service bool
|
||||
|
||||
// Keeps track of all running instances
|
||||
serviceInstances []*Instance
|
||||
|
||||
// UPNP functionality
|
||||
upnp bool
|
||||
// BGP Manager, this is a singleton that manages all BGP advertisements
|
||||
bgpServer *bgp.Server
|
||||
|
||||
@@ -48,8 +51,6 @@ type Manager struct {
|
||||
// This channel is used to signal a shutdown
|
||||
shutdownChan chan struct{}
|
||||
|
||||
svcProcessor *services.Processor
|
||||
|
||||
// This is a prometheus counter used to count the number of events received
|
||||
// from the service watcher
|
||||
countServiceWatchEvent *prometheus.CounterVec
|
||||
@@ -60,12 +61,6 @@ type Manager struct {
|
||||
|
||||
// This mutex is to protect calls from various goroutines
|
||||
mutex sync.Mutex
|
||||
|
||||
// This tracks used network interfaces and guards them with mutex for concurrent changes.
|
||||
intfMgr *networkinterface.Manager
|
||||
|
||||
// This tracks VIPs and performs ARP/NDP advertisement.
|
||||
arpMgr *arp.Manager
|
||||
}
|
||||
|
||||
// New will create a new managing object
|
||||
@@ -77,14 +72,14 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
// we set it to hostname as a fallback.
|
||||
// This mimics legacy behavior and should work on old kube-vip installations.
|
||||
if config.NodeName == "" {
|
||||
log.Warn("Node name is missing from the config, fall back to hostname")
|
||||
log.Warning("Node name is missing from the config, fall back to hostname")
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get hostname: %v", err)
|
||||
}
|
||||
config.NodeName = hostname
|
||||
}
|
||||
log.Info("using node name", "name", config.NodeName)
|
||||
log.Infof("Using node name [%v]", config.NodeName)
|
||||
|
||||
adminConfigPath := "/etc/kubernetes/admin.conf"
|
||||
homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
|
||||
@@ -98,7 +93,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
// Do nothing, we don't construct a k8s client for etcd leader election
|
||||
case utils.FileExists(adminConfigPath):
|
||||
if config.KubernetesAddr != "" {
|
||||
log.Info("k8s address", "address", config.KubernetesAddr)
|
||||
log.Infof("k8s address [%s]", config.KubernetesAddr)
|
||||
clientConfig, err = k8s.NewRestConfig(adminConfigPath, false, config.KubernetesAddr)
|
||||
} else if config.EnableControlPlane {
|
||||
// If this is a control plane host it will likely have started as a static pod or won't have the
|
||||
@@ -119,7 +114,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
if clientset, err = k8s.NewClientset(clientConfig); err != nil {
|
||||
return nil, fmt.Errorf("could not create k8s clientset: %w", err)
|
||||
}
|
||||
log.Debug("Using external Kubernetes configuration from file", "path", adminConfigPath)
|
||||
log.Debugf("Using external Kubernetes configuration from file: %q", adminConfigPath)
|
||||
case utils.FileExists(homeConfigPath):
|
||||
clientConfig, err = k8s.NewRestConfig(homeConfigPath, false, "")
|
||||
if err != nil {
|
||||
@@ -129,7 +124,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create k8s clientset from external file: %q: %w", homeConfigPath, err)
|
||||
}
|
||||
log.Debug("Using external Kubernetes configuration from file", "path", adminConfigPath)
|
||||
log.Debugf("Using external Kubernetes configuration from file [%s]", homeConfigPath)
|
||||
default:
|
||||
clientConfig, err = k8s.NewRestConfig("", true, "")
|
||||
if err != nil {
|
||||
@@ -171,32 +166,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
// }
|
||||
// }
|
||||
|
||||
// listen for interrupts or the Linux SIGTERM signal and cancel
|
||||
// our context, which the leader election code will observe and
|
||||
// step down
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
// Add Notification for Userland interrupt
|
||||
signal.Notify(signalChan, syscall.SIGINT)
|
||||
|
||||
// Add Notification for SIGTERM (sent from Kubernetes)
|
||||
signal.Notify(signalChan, syscall.SIGTERM)
|
||||
|
||||
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
|
||||
shutdownChan := make(chan struct{})
|
||||
|
||||
intfMgr := networkinterface.NewManager()
|
||||
arpMgr := arp.NewManager(config)
|
||||
|
||||
var bgpServer *bgp.Server
|
||||
if config.EnableBGP {
|
||||
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating BGP server: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
|
||||
|
||||
return &Manager{
|
||||
clientSet: clientset,
|
||||
rwClientSet: rwClientSet,
|
||||
@@ -214,12 +183,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
|
||||
Name: "bgp_session_info",
|
||||
Help: "Display state of session by setting metric for label value with current state to 1",
|
||||
}, []string{"state", "peer"}),
|
||||
signalChan: signalChan,
|
||||
shutdownChan: shutdownChan,
|
||||
svcProcessor: svcProcessor,
|
||||
intfMgr: intfMgr,
|
||||
arpMgr: arpMgr,
|
||||
bgpServer: bgpServer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -238,26 +201,6 @@ func (sm *Manager) Start() error {
|
||||
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
|
||||
sm.shutdownChan = make(chan struct{})
|
||||
|
||||
// HealthCheck
|
||||
if sm.config.HealthCheckPort != 0 {
|
||||
if sm.config.HealthCheckPort < 1024 {
|
||||
return fmt.Errorf("healthcheck port is using a port that is less than 1024 [%d]", sm.config.HealthCheckPort)
|
||||
}
|
||||
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
fmt.Fprintf(w, "OK")
|
||||
})
|
||||
go func() {
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf(":%d", sm.config.HealthCheckPort),
|
||||
ReadHeaderTimeout: 3 * time.Second,
|
||||
}
|
||||
err := server.ListenAndServe()
|
||||
if err != nil {
|
||||
log.Error("healthcheck", "unable to start", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// If BGP is enabled then we start a server instance that will broadcast VIPs
|
||||
if sm.config.EnableBGP {
|
||||
|
||||
@@ -267,48 +210,49 @@ func (sm *Manager) Start() error {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("Starting Kube-vip Manager with the BGP engine")
|
||||
log.Infoln("Starting Kube-vip Manager with the BGP engine")
|
||||
return sm.startBGP()
|
||||
}
|
||||
|
||||
if sm.config.EnableARP || sm.config.EnableWireguard {
|
||||
if sm.config.EnableUPNP {
|
||||
// Before starting the leader Election enable any additional functionality
|
||||
upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
|
||||
|
||||
if upnpEnabled {
|
||||
sm.upnp = true
|
||||
clients := upnp.GetConnectionClients(context.TODO())
|
||||
if len(clients) == 0 {
|
||||
log.Error("Error Enabling UPNP. No Clients found")
|
||||
log.Errorf("Error Enabling UPNP. No Clients found")
|
||||
// Set the struct to false so nothing should use it in future
|
||||
sm.config.EnableUPNP = false
|
||||
sm.upnp = false
|
||||
} else {
|
||||
for _, c := range clients {
|
||||
ip, err := c.GetExternalIPAddress()
|
||||
if err != nil {
|
||||
log.Error("unable to find IGD2 Gateway address", "err", err)
|
||||
}
|
||||
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
|
||||
log.Infof("Found UPNP IGD2 Gateway address[%s] error: [%s]", ip, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO: It would be nice to run the UPNP refresh only on the leader.
|
||||
go sm.svcProcessor.RefreshUPNPForwards()
|
||||
go sm.refreshUPNPForwards()
|
||||
}
|
||||
|
||||
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
|
||||
if sm.config.EnableARP {
|
||||
log.Info("Starting Kube-vip Manager with the ARP engine")
|
||||
log.Infoln("Starting Kube-vip Manager with the ARP engine")
|
||||
return sm.startARP(sm.config.NodeName)
|
||||
}
|
||||
|
||||
if sm.config.EnableWireguard {
|
||||
log.Info("Starting Kube-vip Manager with the Wireguard engine")
|
||||
log.Infoln("Starting Kube-vip Manager with the Wireguard engine")
|
||||
return sm.startWireguard(sm.config.NodeName)
|
||||
}
|
||||
|
||||
if sm.config.EnableRoutingTable {
|
||||
log.Info("Starting Kube-vip Manager with the Routing Table engine")
|
||||
log.Infoln("Starting Kube-vip Manager with the Routing Table engine")
|
||||
return sm.startTableMode(sm.config.NodeName)
|
||||
}
|
||||
|
||||
log.Error("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
|
||||
log.Errorln("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -324,7 +268,7 @@ func returnNameSpace() (string, error) {
|
||||
|
||||
func (sm *Manager) parseAnnotations() error {
|
||||
if sm.config.Annotations == "" {
|
||||
log.Debug("No Node annotations to parse")
|
||||
log.Debugf("No Node annotations to parse")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -334,3 +278,65 @@ func (sm *Manager) parseAnnotations() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) serviceInterface() string {
|
||||
svcIf := sm.config.Interface
|
||||
if sm.config.ServicesInterface != "" {
|
||||
svcIf = sm.config.ServicesInterface
|
||||
}
|
||||
return svcIf
|
||||
}
|
||||
|
||||
func (sm *Manager) startTrafficMirroringIfEnabled() error {
|
||||
if sm.config.MirrorDestInterface != "" {
|
||||
svcIf := sm.serviceInterface()
|
||||
log.Infof("mirroring traffic from interface %s to interface %s", svcIf, sm.config.MirrorDestInterface)
|
||||
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
log.Debug("skip starting traffic mirroring since it's not enabled.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) stopTrafficMirroringIfEnabled() error {
|
||||
if sm.config.MirrorDestInterface != "" {
|
||||
svcIf := sm.serviceInterface()
|
||||
log.Infof("clean up qdisc config on interface %s", svcIf)
|
||||
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
log.Debug("skip stopping traffic mirroring since it's not enabled.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) findServiceInstance(svc *v1.Service) *Instance {
|
||||
svcUID := string(svc.UID)
|
||||
log.Debugf("service UID: %s", svcUID)
|
||||
for i := range sm.serviceInstances {
|
||||
log.Debugf("saved service instance %d UID: %s", i, sm.serviceInstances[i].UID)
|
||||
if sm.serviceInstances[i].UID == svcUID {
|
||||
return sm.serviceInstances[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Refresh UPNP Port Forwards for all Service Instances registered in the SM
|
||||
func (sm *Manager) refreshUPNPForwards() {
|
||||
log.Info("Starting UPNP Port Refresher")
|
||||
for {
|
||||
time.Sleep(300 * time.Second)
|
||||
|
||||
log.Infof("[UPNP] Refreshing %d Instances", len(sm.serviceInstances))
|
||||
for i := range sm.serviceInstances {
|
||||
sm.upnpMap(context.TODO(), sm.serviceInstances[i])
|
||||
if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
|
||||
log.Warnf("[UPNP] Error updating service IPs %s [%s]", sm.serviceInstances[i].serviceSnapshot.Name, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,11 @@ package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
@@ -27,9 +27,6 @@ func (sm *Manager) startARP(id string) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
log.Info("Start ARP/NDP advertisement")
|
||||
go sm.arpMgr.StartAdvertisement(ctx)
|
||||
|
||||
// Shutdown function that will wait on this signal, unless we call it ourselves
|
||||
go func() {
|
||||
<-sm.signalChan
|
||||
@@ -44,7 +41,7 @@ func (sm *Manager) startARP(id string) error {
|
||||
}()
|
||||
|
||||
if sm.config.EnableControlPlane {
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -57,16 +54,17 @@ func (sm *Manager) startARP(id string) error {
|
||||
go func() {
|
||||
err := cpCluster.StartCluster(sm.config, clusterManager, nil)
|
||||
if err != nil {
|
||||
log.Error("starting control plane", "err", err)
|
||||
log.Errorf("Control Plane Error [%v]", err)
|
||||
// Trigger the shutdown of this manager instance
|
||||
sm.signalChan <- syscall.SIGINT
|
||||
|
||||
}
|
||||
}()
|
||||
|
||||
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
|
||||
if !sm.config.EnableServices {
|
||||
<-sm.shutdownChan
|
||||
log.Info("Shutting down Kube-Vip")
|
||||
<-sm.signalChan
|
||||
log.Infof("Shutting down Kube-Vip")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -76,27 +74,27 @@ func (sm *Manager) startARP(id string) error {
|
||||
|
||||
ns, err = returnNameSpace()
|
||||
if err != nil {
|
||||
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", sm.config.Namespace)
|
||||
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
|
||||
ns = sm.config.Namespace
|
||||
}
|
||||
}
|
||||
|
||||
// This will tidy any dangling kube-vip iptables rules
|
||||
if sm.config.EgressClean {
|
||||
if os.Getenv("EGRESS_CLEAN") != "" {
|
||||
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
|
||||
}
|
||||
|
||||
// Start a services watcher (all kube-vip pods will watch services), upon a new service
|
||||
// a lock based upon that service is created that they will all leaderElection on
|
||||
if sm.config.EnableServicesElection {
|
||||
log.Info("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
|
||||
log.Infof("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.startServicesWatchForLeaderElection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
||||
log.Info("beginning services leadership", "namespace", ns, "lock name", sm.config.ServicesLeaseName, "id", id)
|
||||
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, sm.config.ServicesLeaseName, id)
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
@@ -125,21 +123,21 @@ func (sm *Manager) startARP(id string) error {
|
||||
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(ctx context.Context) {
|
||||
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
|
||||
err = sm.servicesWatcher(ctx, sm.syncServices)
|
||||
if err != nil {
|
||||
log.Error("service watcher", "err", err)
|
||||
panic("") // TODO: - emulating log.fatal here
|
||||
log.Fatal(err)
|
||||
}
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
// we can do cleanup here
|
||||
sm.mutex.Lock()
|
||||
defer sm.mutex.Unlock()
|
||||
log.Info("leader lost", "new leader", id)
|
||||
sm.svcProcessor.Stop()
|
||||
log.Infof("leader lost: %s", id)
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("lost leadership, restarting kube-vip")
|
||||
panic("") // TODO: - emulating log.fatal here
|
||||
log.Fatal("lost leadership, restarting kube-vip")
|
||||
},
|
||||
OnNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
@@ -150,7 +148,7 @@ func (sm *Manager) startARP(id string) error {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
log.Info("new leader elected", "new leader", identity)
|
||||
log.Infof("new leader elected: %s", identity)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -3,14 +3,16 @@ package manager
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/cluster"
|
||||
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
|
||||
api "github.com/osrg/gobgp/v3/api"
|
||||
"github.com/packethost/packngo"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Start will begin the Manager, which will start services and watch the configmap
|
||||
@@ -19,15 +21,37 @@ func (sm *Manager) startBGP() error {
|
||||
// var ns string
|
||||
var err error
|
||||
|
||||
if sm.bgpServer == nil {
|
||||
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
|
||||
// If Equinix Metal is enabled then we can begin our preparation work
|
||||
var packetClient *packngo.Client
|
||||
if sm.config.EnableMetal {
|
||||
if sm.config.ProviderConfig != "" {
|
||||
key, project, err := equinixmetal.GetPacketConfig(sm.config.ProviderConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Set the environment variable with the key for the project
|
||||
os.Setenv("PACKET_AUTH_TOKEN", key)
|
||||
// Update the configuration with the project key
|
||||
sm.config.MetalProjectID = project
|
||||
|
||||
}
|
||||
packetClient, err = packngo.NewClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating BGP server: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// We're using Equinix Metal with BGP, populate the Peer information from the API
|
||||
if sm.config.EnableBGP {
|
||||
log.Infoln("Looking up the BGP configuration from Equinix Metal")
|
||||
err = equinixmetal.BGPLookup(packetClient, sm.config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
|
||||
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
|
||||
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
|
||||
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
|
||||
port := uint64(179)
|
||||
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
|
||||
@@ -43,8 +67,9 @@ func (sm *Manager) startBGP() error {
|
||||
"peer": peerDescription,
|
||||
}).Set(metricValue)
|
||||
}
|
||||
}); err != nil {
|
||||
return fmt.Errorf("starting BGP server: %w", err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// use a Go context so we can tell the leaderelection code when we
|
||||
@@ -73,7 +98,7 @@ func (sm *Manager) startBGP() error {
|
||||
}()
|
||||
|
||||
if sm.config.EnableControlPlane {
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -87,10 +112,10 @@ func (sm *Manager) startBGP() error {
|
||||
if sm.config.EnableLeaderElection {
|
||||
err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
|
||||
} else {
|
||||
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
|
||||
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer, packetClient)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("Control Plane", "err", err)
|
||||
log.Errorf("Control Plane Error [%v]", err)
|
||||
// Trigger the shutdown of this manager instance
|
||||
sm.signalChan <- syscall.SIGINT
|
||||
}
|
||||
@@ -99,18 +124,18 @@ func (sm *Manager) startBGP() error {
|
||||
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
|
||||
if !sm.config.EnableServices {
|
||||
<-sm.signalChan
|
||||
log.Info("Shutting down Kube-Vip")
|
||||
log.Infof("Shutting down Kube-Vip")
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
|
||||
err = sm.servicesWatcher(ctx, sm.syncServices)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("Shutting down Kube-Vip")
|
||||
log.Infof("Shutting down Kube-Vip")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,15 +3,15 @@ package manager
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/cluster"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
@@ -27,22 +27,29 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
// want to step down
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
log.Info("destination for routes", "table", sm.config.RoutingTableID, "protocol", sm.config.RoutingProtocol)
|
||||
log.Infof("all routing table entries will exist in table [%d] with protocol [%d]", sm.config.RoutingTableID, sm.config.RoutingProtocol)
|
||||
|
||||
if sm.config.CleanRoutingTable {
|
||||
go func() {
|
||||
// we assume that after 10s all services should be configured so we can delete redundant routes
|
||||
time.Sleep(time.Second * 10)
|
||||
if err := sm.cleanRoutes(); err != nil {
|
||||
log.Error("error checking for old routes", "err", err)
|
||||
log.Errorf("error checking for old routes: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if sm.config.EgressClean {
|
||||
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
|
||||
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
|
||||
log.Debug("IPtables rules cleaned on startup")
|
||||
egressCleanEnv := os.Getenv("EGRESS_CLEAN")
|
||||
if egressCleanEnv != "" {
|
||||
egressClean, err := strconv.ParseBool(egressCleanEnv)
|
||||
if err != nil {
|
||||
log.Warnf("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
|
||||
}
|
||||
if egressClean {
|
||||
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
|
||||
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
|
||||
log.Debug("IPTables rules cleaned on startup")
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown function that will wait on this signal, unless we call it ourselves
|
||||
@@ -58,7 +65,7 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
}()
|
||||
|
||||
if sm.config.EnableControlPlane {
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
|
||||
cpCluster, err = cluster.InitCluster(sm.config, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster initialization error: %w", err)
|
||||
}
|
||||
@@ -68,29 +75,29 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
return fmt.Errorf("cluster manager initialization error: %w", err)
|
||||
}
|
||||
|
||||
if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
|
||||
log.Error("Control Plane", "err", err)
|
||||
if err := cpCluster.StartVipService(sm.config, clusterManager, nil, nil); err != nil {
|
||||
log.Errorf("Control Plane Error [%v]", err)
|
||||
// Trigger the shutdown of this manager instance
|
||||
sm.signalChan <- syscall.SIGINT
|
||||
}
|
||||
} else {
|
||||
ns, err := returnNameSpace()
|
||||
if err != nil {
|
||||
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
|
||||
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
|
||||
ns = sm.config.Namespace
|
||||
}
|
||||
|
||||
// Start a services watcher (all kube-vip pods will watch services), upon a new service
|
||||
// a lock based upon that service is created that they will all leaderElection on
|
||||
if sm.config.EnableServicesElection {
|
||||
log.Info("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
|
||||
log.Infof("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.startServicesWatchForLeaderElection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if sm.config.EnableLeaderElection {
|
||||
|
||||
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
|
||||
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, plunderLock, id)
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
@@ -118,21 +125,21 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(ctx context.Context) {
|
||||
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
|
||||
err = sm.servicesWatcher(ctx, sm.syncServices)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
panic("")
|
||||
log.Fatal(err)
|
||||
}
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
// we can do cleanup here
|
||||
sm.mutex.Lock()
|
||||
defer sm.mutex.Unlock()
|
||||
log.Info("leader lost", "id", id)
|
||||
sm.svcProcessor.Stop()
|
||||
log.Infof("leader lost: %s", id)
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("lost leadership, restarting kube-vip")
|
||||
panic("")
|
||||
log.Fatal("lost leadership, restarting kube-vip")
|
||||
},
|
||||
OnNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
@@ -140,15 +147,15 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
log.Info("new leader elected", "id", identity)
|
||||
log.Infof("new leader elected: %s", identity)
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
log.Info("beginning watching services without leader election")
|
||||
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
|
||||
log.Infof("beginning watching services without leader election")
|
||||
err = sm.servicesWatcher(ctx, sm.syncServices)
|
||||
if err != nil {
|
||||
log.Error("Cannot watch services", "err", err)
|
||||
log.Errorf("Cannot watch services, %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -157,8 +164,6 @@ func (sm *Manager) startTableMode(id string) error {
|
||||
}
|
||||
|
||||
func (sm *Manager) cleanRoutes() error {
|
||||
sm.mutex.Lock()
|
||||
defer sm.mutex.Unlock()
|
||||
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting routes: %w", err)
|
||||
@@ -169,17 +174,41 @@ func (sm *Manager) cleanRoutes() error {
|
||||
if sm.config.EnableControlPlane {
|
||||
found = (routes[i].Dst.IP.String() == sm.config.Address)
|
||||
} else {
|
||||
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
for n := range cluster.Network {
|
||||
r := cluster.Network[n].PrepareRoute()
|
||||
if r.Dst.String() == routes[i].Dst.String() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
err = netlink.RouteDel(&(routes[i]))
|
||||
if err != nil {
|
||||
log.Error("[route] deletion", "route", routes[i], "err", err)
|
||||
log.Errorf("[route] error deleting route: %v", routes[i])
|
||||
}
|
||||
log.Debug("[route] deletion", "route", routes[i])
|
||||
log.Debugf("[route] deleted route: %v", routes[i])
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) countRouteReferences(route *netlink.Route) int {
|
||||
cnt := 0
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
for n := range cluster.Network {
|
||||
r := cluster.Network[n].PrepareRoute()
|
||||
if r.Dst.String() == route.Dst.String() {
|
||||
cnt++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
||||
@@ -4,9 +4,8 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
log "github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
@@ -21,7 +20,7 @@ func (sm *Manager) startWireguard(id string) error {
|
||||
// want to step down
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
log.Info("reading wireguard peer configuration from Kubernetes secret")
|
||||
log.Infoln("reading wireguard peer configuration from Kubernetes secret")
|
||||
s, err := sm.clientSet.CoreV1().Secrets(sm.config.Namespace).Get(ctx, "wireguard", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -48,21 +47,21 @@ func (sm *Manager) startWireguard(id string) error {
|
||||
|
||||
ns, err = returnNameSpace()
|
||||
if err != nil {
|
||||
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
|
||||
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
|
||||
ns = sm.config.Namespace
|
||||
}
|
||||
|
||||
// Start a services watcher (all kube-vip pods will watch services), upon a new service
|
||||
// a lock based upon that service is created that they will all leaderElection on
|
||||
if sm.config.EnableServicesElection {
|
||||
log.Info("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
|
||||
log.Infof("beginning watching services, leaderelection will happen for every service")
|
||||
err = sm.startServicesWatchForLeaderElection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
||||
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
|
||||
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, plunderLock, id)
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
@@ -91,21 +90,21 @@ func (sm *Manager) startWireguard(id string) error {
|
||||
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(ctx context.Context) {
|
||||
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
|
||||
err = sm.servicesWatcher(ctx, sm.syncServices)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
panic("")
|
||||
log.Fatal(err)
|
||||
}
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
// we can do cleanup here
|
||||
sm.mutex.Lock()
|
||||
defer sm.mutex.Unlock()
|
||||
log.Info("leader lost", "id", id)
|
||||
sm.svcProcessor.Stop()
|
||||
log.Infof("leader lost: %s", id)
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("lost leadership, restarting kube-vip")
|
||||
panic("")
|
||||
log.Fatal("lost leadership, restarting kube-vip")
|
||||
},
|
||||
OnNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
@@ -113,7 +112,7 @@ func (sm *Manager) startWireguard(id string) error {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
log.Info("new leader elected", "id", identity)
|
||||
log.Infof("new leader elected: %s", identity)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -5,8 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -29,23 +28,24 @@ func applyNodeLabel(clientSet *kubernetes.Clientset, address, id, identity strin
|
||||
ctx := context.Background()
|
||||
node, err := clientSet.CoreV1().Nodes().Get(ctx, id, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Error("can't query node labels", "node", id, "err", err)
|
||||
log.Errorf("can't query node %s labels. error: %v", id, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug(fmt.Sprintf("node %s labels: %+v", id, node.Labels))
|
||||
log.Debugf("node %s labels: %+v", id, node.Labels)
|
||||
|
||||
value, ok := node.Labels[nodeLabelIndex]
|
||||
path := fmt.Sprintf("/metadata/labels/%s", nodeLabelJSONPath)
|
||||
log.Debug(fmt.Sprintf("Received identity: %s - id: %s", identity, id))
|
||||
if ok && value == address {
|
||||
log.Debug(fmt.Sprintf("removing node label `has-ip=%s` on %s", address, id))
|
||||
if (!ok || value != address) && id == identity {
|
||||
log.Debugf("setting node label `has-ip=%s` on %s", address, id)
|
||||
// Append label
|
||||
applyPatchLabels(ctx, clientSet, id, "add", path, address)
|
||||
} else if ok && value == address {
|
||||
log.Debugf("removing node label `has-ip=%s` on %s", address, id)
|
||||
// Remove label
|
||||
applyPatchLabels(ctx, clientSet, id, "remove", path, address)
|
||||
} else {
|
||||
log.Debug(fmt.Sprintf("setting node label `has-ip=%s` on %s", address, id))
|
||||
// Append label
|
||||
applyPatchLabels(ctx, clientSet, id, "add", path, address)
|
||||
log.Debugf("no node label change needed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,15 +59,15 @@ func applyPatchLabels(ctx context.Context, clientSet *kubernetes.Clientset,
|
||||
}}
|
||||
patchData, err := json.Marshal(patchLabels)
|
||||
if err != nil {
|
||||
log.Error("node patch marshaling failed", "err", err)
|
||||
log.Errorf("node patch marshaling failed. error: %v", err)
|
||||
return
|
||||
}
|
||||
// patch node
|
||||
node, err := clientSet.CoreV1().Nodes().Patch(ctx,
|
||||
name, types.JSONPatchType, patchData, metav1.PatchOptions{})
|
||||
if err != nil {
|
||||
log.Error("node patching failed", "err", err)
|
||||
log.Errorf("can't patch node %s. error: %v", name, err)
|
||||
return
|
||||
}
|
||||
log.Debug("updated", "node", name, "labels", node.Labels)
|
||||
log.Debugf("updated node %s labels: %+v", name, node.Labels)
|
||||
}
|
||||
|
||||
@@ -4,12 +4,5 @@ import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
// PrometheusCollector defines a service watch event counter.
|
||||
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
|
||||
collectors := []prometheus.Collector{}
|
||||
if sm.svcProcessor != nil {
|
||||
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
|
||||
}
|
||||
if sm.bgpServer != nil {
|
||||
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
|
||||
}
|
||||
return collectors
|
||||
return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package services
|
||||
package manager
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@@ -8,12 +8,9 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/nftables"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
@@ -23,7 +20,7 @@ const (
|
||||
defaultServiceCIDR = "10.96.0.0/12"
|
||||
)
|
||||
|
||||
func (p *Processor) iptablesCheck() error {
|
||||
func (sm *Manager) iptablesCheck() error {
|
||||
file, err := os.Open("/proc/modules")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -50,31 +47,6 @@ func (p *Processor) iptablesCheck() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) nftablesCheck() error {
|
||||
file, err := os.Open("/proc/modules")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
var queue, ct bool
|
||||
for scanner.Scan() {
|
||||
line := strings.Fields(scanner.Text())
|
||||
switch line[0] {
|
||||
case "nft_queue":
|
||||
queue = true
|
||||
case "nft_ct":
|
||||
ct = true
|
||||
}
|
||||
}
|
||||
|
||||
if !queue || !ct {
|
||||
return fmt.Errorf("missing nftables modules -> nft_ct [%t] -> ntf_queue [%t]", ct, queue)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
|
||||
cidrs := strings.Split(sourceCidrs, ",")
|
||||
isV6 := vip.IsIPv6(ip)
|
||||
@@ -86,7 +58,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
|
||||
matchingFamily = append(matchingFamily, cidr)
|
||||
selectedCIDR, err := checkCIDR(ip, cidr)
|
||||
if err != nil {
|
||||
log.Warn("IPv6 CIDR check ", "err", err)
|
||||
log.Warnf("IPv6 CIDR check failed: %s", err.Error())
|
||||
continue
|
||||
}
|
||||
if selectedCIDR != "" {
|
||||
@@ -98,7 +70,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
|
||||
matchingFamily = append(matchingFamily, cidr)
|
||||
selectedCidr, err := checkCIDR(ip, cidr)
|
||||
if err != nil {
|
||||
log.Warn("IPv4 CIDR check ", "err", err)
|
||||
log.Warnf("IPv4 CIDR check failed: %s", err.Error())
|
||||
continue
|
||||
}
|
||||
if selectedCidr != "" {
|
||||
@@ -134,27 +106,21 @@ func checkCIDR(ip, cidr string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
|
||||
func (sm *Manager) configureEgress(vipIP, podIP, destinationPorts, namespace string) error {
|
||||
var podCidr, serviceCidr string
|
||||
var autoServiceCIDR, autoPodCIDR string
|
||||
var discoverErr error
|
||||
|
||||
// Look up the destination ports from the annotations on the service
|
||||
destinationPorts := annotations[kubevip.EgressDestinationPorts]
|
||||
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
|
||||
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
|
||||
internalEgress := annotations[kubevip.EgressInternal]
|
||||
|
||||
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
|
||||
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
|
||||
if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
|
||||
autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
|
||||
}
|
||||
|
||||
if discoverErr != nil {
|
||||
log.Warn("autodiscover CIDR", "err", discoverErr)
|
||||
log.Warn(discoverErr)
|
||||
}
|
||||
|
||||
if p.config.EgressPodCidr != "" {
|
||||
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
|
||||
if sm.config.EgressPodCidr != "" {
|
||||
podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
|
||||
} else {
|
||||
if discoverErr == nil {
|
||||
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
|
||||
@@ -164,13 +130,13 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
|
||||
if podCidr == "" {
|
||||
// There's no default IPv6 pod CIDR, therefore we silently back off if CIDR s not specified.
|
||||
if !vip.IsIPv4(podIP) {
|
||||
return fmt.Errorf("error with the CIDR [%s]", podIP)
|
||||
return nil
|
||||
}
|
||||
podCidr = defaultPodCIDR
|
||||
}
|
||||
|
||||
if p.config.EgressServiceCidr != "" {
|
||||
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
|
||||
if sm.config.EgressServiceCidr != "" {
|
||||
serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
|
||||
} else {
|
||||
if discoverErr == nil {
|
||||
serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
|
||||
@@ -185,21 +151,21 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
|
||||
serviceCidr = defaultServiceCIDR
|
||||
}
|
||||
|
||||
log.Info("[Egress]", "podCIDR", podCidr, "serviceCIDR", serviceCidr, "vip", serviceCidr, "pod", podIP)
|
||||
log.Infof("[Egress] pod CIDR [%s], service CIDR [%s] for vip [%s] / pod [%s]", podCidr, serviceCidr, vipIP, podIP)
|
||||
|
||||
// checking if all addresses are of the same IP family
|
||||
if vip.IsIPv4(podIP) != vip.IsIPv4CIDR(podCidr) {
|
||||
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "podCIDR", podCidr)
|
||||
log.Errorf("[Egress] pod's IP [%s] and Pod CIDR [%s] family is not matching. Backing off...", podIP, podCidr)
|
||||
return nil
|
||||
}
|
||||
|
||||
if vip.IsIPv4(vipIP) != vip.IsIPv4CIDR(serviceCidr) {
|
||||
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "serviceCIDR", serviceCidr)
|
||||
log.Errorf("[Egress] VIP's IP [%s] and Service CIDR [%s] family is not matching. Backing off...", podIP, podCidr)
|
||||
return nil
|
||||
}
|
||||
|
||||
if vip.IsIPv4(vipIP) != vip.IsIPv4(podIP) {
|
||||
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "vipIP", vipIP)
|
||||
log.Errorf("[Egress] VIP's IP [%s] and Pod's IP [%s] family is not matching. Backing off...", podIP, podCidr)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -208,30 +174,7 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
|
||||
protocol = iptables.ProtocolIPv6
|
||||
}
|
||||
|
||||
// Use the internal egress implementation
|
||||
if internalEgress != "" {
|
||||
// Create an array of CIDRs that we wont SNAT to.
|
||||
ignoreCIDRs := []string{
|
||||
podCidr,
|
||||
serviceCidr,
|
||||
}
|
||||
|
||||
// Add any specifically denied networks
|
||||
if deniedNetworks != "" {
|
||||
networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
|
||||
ignoreCIDRs = append(ignoreCIDRs, networks...)
|
||||
|
||||
}
|
||||
|
||||
// Apply the SNAT rules
|
||||
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error performing netlink nftables [%s]", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
|
||||
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error Creating iptables client [%s]", err)
|
||||
}
|
||||
@@ -258,35 +201,16 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
|
||||
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
|
||||
if deniedNetworks != "" {
|
||||
networks := strings.Split(deniedNetworks, ",")
|
||||
for x := range networks {
|
||||
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, networks[x])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mask := "/32"
|
||||
if !vip.IsIPv4(podIP) {
|
||||
mask = "/128"
|
||||
}
|
||||
|
||||
if allowedNetworks != "" {
|
||||
networks := strings.Split(allowedNetworks, ",")
|
||||
for x := range networks {
|
||||
err = i.AppendReturnRulesForMarkingForNetwork(vip.MangleChainName, podIP+mask, networks[x])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err = i.AppendReturnRulesForMarking(vip.MangleChainName, podIP+mask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
err = i.AppendReturnRulesForMarking(vip.MangleChainName, podIP+mask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
}
|
||||
|
||||
err = i.InsertMangeTableIntoPrerouting(vip.MangleChainName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding prerouting mangle chain [%s], error [%s]", vip.MangleChainName, err)
|
||||
@@ -330,12 +254,12 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
|
||||
log.Debug("Trying to automatically discover Service and Pod CIDRs")
|
||||
func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
|
||||
log.Debugf("Trying to automatically discover Service and Pod CIDRs")
|
||||
options := v1.ListOptions{
|
||||
LabelSelector: "component=kube-controller-manager",
|
||||
}
|
||||
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
|
||||
podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
|
||||
}
|
||||
@@ -358,3 +282,57 @@ func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (sm *Manager) TeardownEgress(podIP, vipIP, destinationPorts, namespace string) error {
|
||||
protocol := iptables.ProtocolIPv4
|
||||
if vip.IsIPv6(podIP) {
|
||||
protocol = iptables.ProtocolIPv6
|
||||
}
|
||||
|
||||
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error Creating iptables client [%s]", err)
|
||||
}
|
||||
|
||||
// Remove the marking of egress packets
|
||||
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
|
||||
// Clear up SNAT rules
|
||||
if destinationPorts != "" {
|
||||
fixedPorts := strings.Split(destinationPorts, ",")
|
||||
|
||||
for _, fixedPort := range fixedPorts {
|
||||
var proto, port string
|
||||
|
||||
data := strings.Split(fixedPort, ":")
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
} else if len(data) == 1 {
|
||||
proto = "tcp"
|
||||
port = data[0]
|
||||
} else {
|
||||
proto = data[0]
|
||||
port = data[1]
|
||||
}
|
||||
|
||||
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
err = i.DeleteSourceNat(podIP, vipIP)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
}
|
||||
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package services
|
||||
package manager
|
||||
|
||||
import "testing"
|
||||
|
||||
495
pkg/manager/services.go
Normal file
495
pkg/manager/services.go
Normal file
@@ -0,0 +1,495 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/upnp"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
const (
|
||||
hwAddrKey = "kube-vip.io/hwaddr"
|
||||
requestedIP = "kube-vip.io/requestedIP"
|
||||
vipHost = "kube-vip.io/vipHost"
|
||||
egress = "kube-vip.io/egress"
|
||||
egressIPv6 = "kube-vip.io/egress-ipv6"
|
||||
egressDestinationPorts = "kube-vip.io/egress-destination-ports"
|
||||
egressSourcePorts = "kube-vip.io/egress-source-ports"
|
||||
activeEndpoint = "kube-vip.io/active-endpoint"
|
||||
activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
|
||||
flushContrack = "kube-vip.io/flush-conntrack"
|
||||
loadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
|
||||
loadbalancerHostname = "kube-vip.io/loadbalancerHostname"
|
||||
serviceInterface = "kube-vip.io/serviceInterface"
|
||||
upnpEnabled = "kube-vip.io/forwardUPNP"
|
||||
)
|
||||
|
||||
func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service, wg *sync.WaitGroup) error {
|
||||
defer wg.Done()
|
||||
|
||||
log.Debugf("[STARTING] Service Sync")
|
||||
|
||||
// Iterate through the synchronising services
|
||||
foundInstance := false
|
||||
newServiceAddresses := fetchServiceAddresses(svc)
|
||||
newServiceUID := string(svc.UID)
|
||||
|
||||
ingressIPs := []string{}
|
||||
|
||||
for _, ingress := range svc.Status.LoadBalancer.Ingress {
|
||||
ingressIPs = append(ingressIPs, ingress.IP)
|
||||
}
|
||||
|
||||
shouldBreake := false
|
||||
|
||||
for x := range sm.serviceInstances {
|
||||
if shouldBreake {
|
||||
break
|
||||
}
|
||||
for _, newServiceAddress := range newServiceAddresses {
|
||||
log.Debugf("isDHCP: %t, newServiceAddress: %s", sm.serviceInstances[x].isDHCP, newServiceAddress)
|
||||
if sm.serviceInstances[x].UID == newServiceUID {
|
||||
// If the found instance's DHCP configuration doesn't match the new service, delete it.
|
||||
if (sm.serviceInstances[x].isDHCP && newServiceAddress != "0.0.0.0") ||
|
||||
(!sm.serviceInstances[x].isDHCP && newServiceAddress == "0.0.0.0") ||
|
||||
(!sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, newServiceAddress)) ||
|
||||
(len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc)) ||
|
||||
(sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, sm.serviceInstances[x].dhcpInterfaceIP)) {
|
||||
if err := sm.deleteService(newServiceUID); err != nil {
|
||||
return err
|
||||
}
|
||||
shouldBreake = true
|
||||
break
|
||||
}
|
||||
foundInstance = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This instance wasn't found, we need to add it to the manager
|
||||
if !foundInstance && len(newServiceAddresses) > 0 {
|
||||
if err := sm.addService(ctx, svc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func comparePortsAndPortStatuses(svc *v1.Service) bool {
|
||||
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
|
||||
if len(portsStatus) != len(svc.Spec.Ports) {
|
||||
return false
|
||||
}
|
||||
for i, portSpec := range svc.Spec.Ports {
|
||||
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
|
||||
startTime := time.Now()
|
||||
|
||||
newService, err := NewInstance(svc, sm.config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for x := range newService.vipConfigs {
|
||||
newService.clusters[x].StartLoadBalancerService(newService.vipConfigs[x], sm.bgpServer)
|
||||
}
|
||||
|
||||
sm.upnpMap(ctx, newService)
|
||||
|
||||
if newService.isDHCP && len(newService.vipConfigs) == 1 {
|
||||
go func() {
|
||||
for ip := range newService.dhcpClient.IPChannel() {
|
||||
log.Debugf("IP %s may have changed", ip)
|
||||
newService.vipConfigs[0].VIP = ip
|
||||
newService.dhcpInterfaceIP = ip
|
||||
if !sm.config.DisableServiceUpdates {
|
||||
if err := sm.updateStatus(newService); err != nil {
|
||||
log.Warnf("error updating svc: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Debugf("IP update channel closed, stopping")
|
||||
}()
|
||||
}
|
||||
|
||||
sm.serviceInstances = append(sm.serviceInstances, newService)
|
||||
|
||||
if !sm.config.DisableServiceUpdates {
|
||||
log.Debugf("(svcs) will update [%s/%s]", newService.serviceSnapshot.Namespace, newService.serviceSnapshot.Name)
|
||||
if err := sm.updateStatus(newService); err != nil {
|
||||
// delete service to collect garbage
|
||||
if deleteErr := sm.deleteService(newService.UID); deleteErr != nil {
|
||||
return deleteErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
serviceIPs := fetchServiceAddresses(svc)
|
||||
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
|
||||
if svc.Annotations[flushContrack] == "true" {
|
||||
|
||||
log.Debugf("Flushing conntrack rules for service [%s]", svc.Name)
|
||||
for _, serviceIP := range serviceIPs {
|
||||
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
|
||||
if err != nil {
|
||||
log.Errorf("Error flushing any remaining egress connections [%s]", err)
|
||||
}
|
||||
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
|
||||
if err != nil {
|
||||
log.Errorf("Error flushing any remaining ingress connections [%s]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if egress is enabled on the service, if so we'll need to configure some rules
|
||||
if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
|
||||
log.Debugf("Enabling egress for the service [%s]", svc.Name)
|
||||
// We will need to modify the iptables rules
|
||||
err = sm.iptablesCheck()
|
||||
if err != nil {
|
||||
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
|
||||
}
|
||||
var podIP string
|
||||
errList := []error{}
|
||||
|
||||
// Should egress be IPv6
|
||||
if svc.Annotations[egressIPv6] == "true" {
|
||||
// Does the service have an active IPv6 endpoint
|
||||
if svc.Annotations[activeEndpointIPv6] != "" {
|
||||
for _, serviceIP := range serviceIPs {
|
||||
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
|
||||
|
||||
podIP = svc.Annotations[activeEndpointIPv6]
|
||||
|
||||
err = sm.configureEgress(serviceIP, podIP, svc.Annotations[egressDestinationPorts], svc.Namespace)
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
|
||||
for _, serviceIP := range serviceIPs {
|
||||
podIPs := svc.Annotations[activeEndpoint]
|
||||
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
|
||||
podIPs = svc.Annotations[activeEndpointIPv6]
|
||||
}
|
||||
err = sm.configureEgress(serviceIP, podIPs, svc.Annotations[egressDestinationPorts], svc.Namespace)
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(errList) == 0 {
|
||||
var provider epProvider
|
||||
if !sm.config.EnableEndpointSlices {
|
||||
provider = &endpointsProvider{label: "endpoints"}
|
||||
} else {
|
||||
provider = &endpointslicesProvider{label: "endpointslices"}
|
||||
}
|
||||
err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
|
||||
if err != nil {
|
||||
log.Errorf("error configuring egress annotation for loadbalancer [%s]", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
finishTime := time.Since(startTime)
|
||||
log.Infof("[service] synchronised in %dms", finishTime.Milliseconds())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) deleteService(uid string) error {
|
||||
// protect multiple calls
|
||||
sm.mutex.Lock()
|
||||
defer sm.mutex.Unlock()
|
||||
|
||||
var updatedInstances []*Instance
|
||||
var serviceInstance *Instance
|
||||
found := false
|
||||
for x := range sm.serviceInstances {
|
||||
log.Debugf("Looking for [%s], found [%s]", uid, sm.serviceInstances[x].UID)
|
||||
// Add the running services to the new array
|
||||
if sm.serviceInstances[x].UID != uid {
|
||||
updatedInstances = append(updatedInstances, sm.serviceInstances[x])
|
||||
} else {
|
||||
// Flip the found when we match
|
||||
found = true
|
||||
serviceInstance = sm.serviceInstances[x]
|
||||
}
|
||||
}
|
||||
// If we've been through all services and not found the correct one then error
|
||||
if !found {
|
||||
// TODO: - fix UX
|
||||
// return fmt.Errorf("unable to find/stop service [%s]", uid)
|
||||
return nil
|
||||
}
|
||||
shared := false
|
||||
vipSet := make(map[string]interface{})
|
||||
for x := range updatedInstances {
|
||||
for _, vip := range updatedInstances[x].VIPs {
|
||||
vipSet[vip] = nil
|
||||
}
|
||||
}
|
||||
for _, vip := range serviceInstance.VIPs {
|
||||
if _, found := vipSet[vip]; found {
|
||||
shared = true
|
||||
}
|
||||
}
|
||||
if !shared {
|
||||
for x := range serviceInstance.clusters {
|
||||
serviceInstance.clusters[x].Stop()
|
||||
}
|
||||
if serviceInstance.isDHCP {
|
||||
serviceInstance.dhcpClient.Stop()
|
||||
macvlan, err := netlink.LinkByName(serviceInstance.dhcpInterface)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error finding VIP Interface: %v", err)
|
||||
}
|
||||
|
||||
err = netlink.LinkDel(macvlan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deleting DHCP Link : %v", err)
|
||||
}
|
||||
}
|
||||
// TODO: Implement dual-stack loadbalancer support if BGP is enabled
|
||||
for i := range serviceInstance.vipConfigs {
|
||||
if serviceInstance.vipConfigs[i].EnableBGP {
|
||||
cidrVip := fmt.Sprintf("%s/%s", serviceInstance.vipConfigs[i].VIP, serviceInstance.vipConfigs[i].VIPCIDR)
|
||||
err := sm.bgpServer.DelHost(cidrVip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[BGP] error deleting BGP host: %v", err)
|
||||
}
|
||||
log.Debugf("[BGP] deleted host: %s", cidrVip)
|
||||
}
|
||||
}
|
||||
|
||||
// We will need to tear down the egress
|
||||
if serviceInstance.serviceSnapshot.Annotations[egress] == "true" {
|
||||
if serviceInstance.serviceSnapshot.Annotations[activeEndpoint] != "" {
|
||||
log.Infof("service [%s] has an egress re-write enabled", serviceInstance.serviceSnapshot.Name)
|
||||
err := sm.TeardownEgress(serviceInstance.serviceSnapshot.Annotations[activeEndpoint], serviceInstance.serviceSnapshot.Spec.LoadBalancerIP, serviceInstance.serviceSnapshot.Annotations[egressDestinationPorts], serviceInstance.serviceSnapshot.Namespace)
|
||||
if err != nil {
|
||||
log.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the service array
|
||||
sm.serviceInstances = updatedInstances
|
||||
|
||||
log.Infof("Removed [%s] from manager, [%d] advertised services remain", uid, len(sm.serviceInstances))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set up UPNP forwards for a service
|
||||
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
|
||||
func (sm *Manager) upnpMap(ctx context.Context, s *Instance) {
|
||||
if !isUPNPEnabled(s.serviceSnapshot) {
|
||||
// Skip services missing the annotation
|
||||
return
|
||||
}
|
||||
if !sm.upnp {
|
||||
log.Warnf("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding service %s", s.serviceSnapshot.Name)
|
||||
}
|
||||
// If upnp is enabled then update the gateway/router with the address
|
||||
// TODO - check if this implementation for dualstack is correct
|
||||
|
||||
gateways := upnp.GetGatewayClients(ctx)
|
||||
|
||||
// Reset Gateway IPs to remove stale addresses
|
||||
s.upnpGatewayIPs = make([]string, 0)
|
||||
|
||||
for _, vip := range s.VIPs {
|
||||
for _, port := range s.ExternalPorts {
|
||||
for _, gw := range gateways {
|
||||
log.Infof("[UPNP] Adding map to [%s:%d - %s] on gateway %s", vip, port.Port, s.serviceSnapshot.Name, gw.WANIPv6FirewallControlClient.Location)
|
||||
|
||||
forwardSucessful := false
|
||||
if gw.WANIPv6FirewallControlClient != nil {
|
||||
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(port.Type), 3600)
|
||||
if pinholeErr == nil {
|
||||
forwardSucessful = true
|
||||
log.Infof("[UPNP] Service should be accessible externally on port [%d]; PinholeID is [%d]", port.Port, pinholeID)
|
||||
} else {
|
||||
//TODO: Cleanup
|
||||
log.Errorf("[UPNP] Unable to map port to gateway using Pinhole API[%s]", pinholeErr.Error())
|
||||
}
|
||||
}
|
||||
// Fallback to PortForward
|
||||
if !forwardSucessful {
|
||||
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(port.Type), uint16(port.Port), vip, true, s.serviceSnapshot.Name, 3600)
|
||||
if portMappingErr == nil {
|
||||
log.Infof("[UPNP] Service should be accessible externally on port [%d]", port.Port)
|
||||
forwardSucessful = true
|
||||
} else {
|
||||
//TODO: Cleanup
|
||||
log.Errorf("[UPNP] Unable to map port to gateway using PortForward API[%s]", portMappingErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if forwardSucessful {
|
||||
ip, err := gw.ConnectionClient.GetExternalIPAddress()
|
||||
if err == nil {
|
||||
s.upnpGatewayIPs = append(s.upnpGatewayIPs, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove duplicate IPs
|
||||
slices.Sort(s.upnpGatewayIPs)
|
||||
s.upnpGatewayIPs = slices.Compact(s.upnpGatewayIPs)
|
||||
}
|
||||
|
||||
func (sm *Manager) updateStatus(i *Instance) error {
|
||||
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
// Retrieve the latest version of Deployment before attempting update
|
||||
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
|
||||
currentService, err := sm.clientSet.CoreV1().Services(i.serviceSnapshot.Namespace).Get(context.TODO(), i.serviceSnapshot.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentServiceCopy := currentService.DeepCopy()
|
||||
if currentServiceCopy.Annotations == nil {
|
||||
currentServiceCopy.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
|
||||
if sm.config.EnableARP {
|
||||
// Add the current host
|
||||
currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
|
||||
}
|
||||
if i.dhcpInterfaceHwaddr != "" || i.dhcpInterfaceIP != "" {
|
||||
currentServiceCopy.Annotations[hwAddrKey] = i.dhcpInterfaceHwaddr
|
||||
currentServiceCopy.Annotations[requestedIP] = i.dhcpInterfaceIP
|
||||
}
|
||||
|
||||
if !cmp.Equal(currentService, currentServiceCopy) {
|
||||
currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Errorf("Error updating Service Spec [%s] : %v", i.serviceSnapshot.Name, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ports := make([]v1.PortStatus, 0, len(i.serviceSnapshot.Spec.Ports))
|
||||
for _, port := range i.serviceSnapshot.Spec.Ports {
|
||||
ports = append(ports, v1.PortStatus{
|
||||
Port: port.Port,
|
||||
Protocol: port.Protocol,
|
||||
})
|
||||
}
|
||||
|
||||
ingresses := []v1.LoadBalancerIngress{}
|
||||
|
||||
for _, c := range i.vipConfigs {
|
||||
if !vip.IsIP(c.VIP) {
|
||||
ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: ip,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
} else {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: c.VIP,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
if isUPNPEnabled(currentService) {
|
||||
for _, ip := range i.upnpGatewayIPs {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: ip,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
|
||||
currentService.Status.LoadBalancer.Ingress = ingresses
|
||||
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Errorf("Error updating Service %s/%s Status: %v", i.serviceSnapshot.Namespace, i.serviceSnapshot.Name, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if retryErr != nil {
|
||||
log.Errorf("Failed to set Services: %v", retryErr)
|
||||
return retryErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchServiceAddresses tries to get the addresses from annotations
|
||||
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
|
||||
func fetchServiceAddresses(s *v1.Service) []string {
|
||||
annotationAvailable := false
|
||||
if s.Annotations != nil {
|
||||
if v, annotationAvailable := s.Annotations[loadbalancerIPAnnotation]; annotationAvailable {
|
||||
ips := strings.Split(v, ",")
|
||||
var trimmedIPs []string
|
||||
for _, ip := range ips {
|
||||
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
|
||||
}
|
||||
return trimmedIPs
|
||||
}
|
||||
}
|
||||
|
||||
if !annotationAvailable {
|
||||
if len(s.Status.LoadBalancer.Ingress) > 0 {
|
||||
addresses := []string{}
|
||||
for _, ingress := range s.Status.LoadBalancer.Ingress {
|
||||
addresses = append(addresses, ingress.IP)
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
}
|
||||
|
||||
if s.Spec.LoadBalancerIP != "" {
|
||||
return []string{s.Spec.LoadBalancerIP}
|
||||
}
|
||||
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func isUPNPEnabled(s *v1.Service) bool {
|
||||
return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
|
||||
}
|
||||
102
pkg/manager/servicesLeader.go
Normal file
102
pkg/manager/servicesLeader.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
)
|
||||
|
||||
// The startServicesWatchForLeaderElection function will start a services watcher, the
|
||||
func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
|
||||
err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, instance := range sm.serviceInstances {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
_ = cluster.Network[i].DeleteRoute()
|
||||
}
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Shutting down kube-Vip")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// The startServicesWatchForLeaderElection function will start a services watcher, the
|
||||
func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service, wg *sync.WaitGroup) error {
|
||||
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
|
||||
log.Infof("(svc election) service [%s], namespace [%s], lock name [%s], host id [%s]", service.Name, service.Namespace, serviceLease, sm.config.NodeName)
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{
|
||||
Name: serviceLease,
|
||||
Namespace: service.Namespace,
|
||||
},
|
||||
Client: sm.clientSet.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: sm.config.NodeName,
|
||||
},
|
||||
}
|
||||
|
||||
activeService[string(service.UID)] = true
|
||||
// start the leader election code loop
|
||||
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
|
||||
Lock: lock,
|
||||
// IMPORTANT: you MUST ensure that any code you have that
|
||||
// is protected by the lease must terminate **before**
|
||||
// you call cancel. Otherwise, you could have a background
|
||||
// loop still running and another process could
|
||||
// get elected before your background loop finished, violating
|
||||
// the stated goal of the lease.
|
||||
ReleaseOnCancel: true,
|
||||
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
|
||||
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
|
||||
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(ctx context.Context) {
|
||||
// Mark this service as active (as we've started leading)
|
||||
// we run this in background as it's blocking
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
if err := sm.syncServices(ctx, service, wg); err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}()
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
// we can do cleanup here
|
||||
log.Infof("(svc election) service [%s] leader lost: [%s]", service.Name, sm.config.NodeName)
|
||||
if activeService[string(service.UID)] {
|
||||
if err := sm.deleteService(string(service.UID)); err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
// Mark this service is inactive
|
||||
activeService[string(service.UID)] = false
|
||||
},
|
||||
OnNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
if identity == sm.config.NodeName {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
log.Infof("(svc election) new leader elected: %s", identity)
|
||||
},
|
||||
},
|
||||
})
|
||||
log.Infof("(svc election) for service [%s] stopping", service.Name)
|
||||
return nil
|
||||
}
|
||||
@@ -8,9 +8,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -27,7 +26,7 @@ import (
|
||||
// present
|
||||
func (sm *Manager) annotationsWatcher() error {
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", sm.config.Annotations)
|
||||
log.Infof("Kube-Vip is waiting for annotation prefix [%s] to be present on this node", sm.config.Annotations)
|
||||
|
||||
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": sm.config.NodeName}}
|
||||
listOptions := metav1.ListOptions{
|
||||
@@ -55,11 +54,10 @@ func (sm *Manager) annotationsWatcher() error {
|
||||
|
||||
// We got an error with the annotations, falling back to the watch until
|
||||
// they're as needed
|
||||
log.Warn(err.Error())
|
||||
log.Warn(err)
|
||||
|
||||
// TODO, will need refactoring as part of rikatz work
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
|
||||
},
|
||||
})
|
||||
@@ -96,7 +94,7 @@ func (sm *Manager) annotationsWatcher() error {
|
||||
|
||||
bgpConfig, bgpPeer, err := parseBgpAnnotations(sm.config.BGPConfig, node, sm.config.Annotations)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -110,7 +108,7 @@ func (sm *Manager) annotationsWatcher() error {
|
||||
return fmt.Errorf("unable to parse Kubernetes Node from Kubernetes watcher")
|
||||
}
|
||||
|
||||
log.Info("Node has been deleted", "name", node.Name)
|
||||
log.Infof("Node [%s] has been deleted", node.Name)
|
||||
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
@@ -121,17 +119,17 @@ func (sm *Manager) annotationsWatcher() error {
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error(status.String())
|
||||
log.Errorf("%v", status)
|
||||
default:
|
||||
}
|
||||
}
|
||||
close(exitFunction)
|
||||
log.Info("Exiting Annotations watcher")
|
||||
log.Infoln("Exiting Annotations watcher")
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
481
pkg/manager/watch_endpoints.go
Normal file
481
pkg/manager/watch_endpoints.go
Normal file
@@ -0,0 +1,481 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
|
||||
type epProvider interface {
|
||||
createRetryWatcher(context.Context, *Manager,
|
||||
*v1.Service) (*watchtools.RetryWatcher, error)
|
||||
getAllEndpoints() ([]string, error)
|
||||
getLocalEndpoints(string, *kubevip.Config) ([]string, error)
|
||||
getLabel() string
|
||||
updateServiceAnnotation(string, string, *v1.Service, *Manager) error
|
||||
loadObject(runtime.Object, context.CancelFunc) error
|
||||
getProtocol() string
|
||||
}
|
||||
|
||||
type endpointsProvider struct {
|
||||
label string
|
||||
endpoints *v1.Endpoints
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
|
||||
service *v1.Service) (*watchtools.RetryWatcher, error) {
|
||||
opts := metav1.ListOptions{
|
||||
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
|
||||
}
|
||||
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
|
||||
}
|
||||
|
||||
return rw, nil
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
|
||||
eps, ok := endpoints.(*v1.Endpoints)
|
||||
if !ok {
|
||||
cancel()
|
||||
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
|
||||
}
|
||||
ep.endpoints = eps
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
|
||||
result := []string{}
|
||||
for subset := range ep.endpoints.Subsets {
|
||||
for address := range ep.endpoints.Subsets[subset].Addresses {
|
||||
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
|
||||
result = append(result, addr[0])
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
|
||||
var localEndpoints []string
|
||||
|
||||
for _, subset := range ep.endpoints.Subsets {
|
||||
for _, address := range subset.Addresses {
|
||||
log.Debugf("[%s] processing endpoint [%s]", ep.label, address.IP)
|
||||
|
||||
// 1. Compare the Nodename
|
||||
if address.NodeName != nil && id == *address.NodeName {
|
||||
log.Debugf("[%s] found local endpoint - address: %s, hostname: %s, node: %s", ep.label, address.IP, address.Hostname, *address.NodeName)
|
||||
localEndpoints = append(localEndpoints, address.IP)
|
||||
continue
|
||||
}
|
||||
// 2. Compare the Hostname (only useful if address.NodeName is not available)
|
||||
if id == address.Hostname {
|
||||
log.Debugf("[%s] found local endpoint - address: %s, hostname: %s", ep.label, address.IP, address.Hostname)
|
||||
localEndpoints = append(localEndpoints, address.IP)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return localEndpoints, nil
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
|
||||
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
// Retrieve the latest version of Deployment before attempting update
|
||||
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
|
||||
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentServiceCopy := currentService.DeepCopy()
|
||||
if currentServiceCopy.Annotations == nil {
|
||||
currentServiceCopy.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
currentServiceCopy.Annotations[activeEndpoint] = endpoint
|
||||
|
||||
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error updating Service Spec [%s] : %v", ep.getLabel(), currentServiceCopy.Name, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if retryErr != nil {
|
||||
log.Errorf("[%s] failed to set Services: %v", ep.getLabel(), retryErr)
|
||||
return retryErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) getLabel() string {
|
||||
return ep.label
|
||||
}
|
||||
|
||||
func (ep *endpointsProvider) getProtocol() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (sm *Manager) watchEndpoint(ctx context.Context, id string, service *v1.Service, wg *sync.WaitGroup, provider epProvider) error {
|
||||
log.Infof("[%s] watching for service [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
leaderContext, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var leaderElectionActive bool
|
||||
|
||||
rw, err := provider.createRetryWatcher(leaderContext, sm, service)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
|
||||
}
|
||||
|
||||
exitFunction := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debugf("[%s] context cancelled", provider.getLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
case <-sm.shutdownChan:
|
||||
log.Debugf("[%s] shutdown called", provider.getLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
case <-exitFunction:
|
||||
log.Debugf("[%s] function ending", provider.getLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
ch := rw.ResultChan()
|
||||
|
||||
var lastKnownGoodEndpoint string
|
||||
for event := range ch {
|
||||
activeEndpointAnnotation := activeEndpoint
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
|
||||
case watch.Added, watch.Modified:
|
||||
|
||||
if err = provider.loadObject(event.Object, cancel); err != nil {
|
||||
return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
|
||||
}
|
||||
|
||||
if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
|
||||
activeEndpointAnnotation = activeEndpointIPv6
|
||||
}
|
||||
|
||||
// Build endpoints
|
||||
var endpoints []string
|
||||
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
|
||||
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
if endpoints, err = provider.getAllEndpoints(); err != nil {
|
||||
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
|
||||
}
|
||||
} else {
|
||||
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
|
||||
return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Find out if we have any local endpoints
|
||||
// if out endpoint is empty then populate it
|
||||
// if not, go through the endpoints and see if ours still exists
|
||||
// If we have a local endpoint then begin the leader Election, unless it's already running
|
||||
//
|
||||
|
||||
// Check that we have local endpoints
|
||||
if len(endpoints) != 0 {
|
||||
// Ignore IPv4
|
||||
if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// if we haven't populated one, then do so
|
||||
if lastKnownGoodEndpoint != "" {
|
||||
|
||||
// check out previous endpoint exists
|
||||
stillExists := false
|
||||
|
||||
for x := range endpoints {
|
||||
if endpoints[x] == lastKnownGoodEndpoint {
|
||||
stillExists = true
|
||||
}
|
||||
}
|
||||
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
|
||||
if !stillExists {
|
||||
if sm.config.EnableRoutingTable {
|
||||
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
|
||||
service.Annotations[egressDestinationPorts], service.Namespace); err != nil {
|
||||
log.Warnf("error removing redundant egress rules: %s", err.Error())
|
||||
}
|
||||
}
|
||||
if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
|
||||
log.Warnf("[%s] existing [%s] has been removed, restarting leaderElection", provider.getLabel(), lastKnownGoodEndpoint)
|
||||
// Stop the existing leaderElection
|
||||
cancel()
|
||||
// disable last leaderElection flag
|
||||
leaderElectionActive = false
|
||||
}
|
||||
// Set our active endpoint to an existing one
|
||||
lastKnownGoodEndpoint = endpoints[0]
|
||||
}
|
||||
} else {
|
||||
lastKnownGoodEndpoint = endpoints[0]
|
||||
}
|
||||
|
||||
if !leaderElectionActive && sm.config.EnableServicesElection {
|
||||
go func() {
|
||||
leaderContext, cancel = context.WithCancel(ctx)
|
||||
|
||||
// This is a blocking function, that will restart (in the event of failure)
|
||||
for {
|
||||
// if the context isn't cancelled restart
|
||||
if leaderContext.Err() != context.Canceled {
|
||||
leaderElectionActive = true
|
||||
err := sm.StartServicesLeaderElection(leaderContext, service, wg)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
leaderElectionActive = false
|
||||
} else {
|
||||
leaderElectionActive = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
isRouteConfigured, err := isRouteConfigured(service.UID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error while checking if route is configured: %w", provider.getLabel(), err)
|
||||
}
|
||||
// There are local endpoints available on the node
|
||||
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection && !isRouteConfigured {
|
||||
// If routing table mode is enabled - routes should be added per node
|
||||
if sm.config.EnableRoutingTable {
|
||||
if instance := sm.findServiceInstance(service); instance != nil {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
err := cluster.Network[i].AddRoute(false)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EEXIST) {
|
||||
// If route exists try to update it if necessary
|
||||
isUpdated, err := cluster.Network[i].UpdateRoutes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
|
||||
}
|
||||
if isUpdated {
|
||||
log.Debugf("[%s] updated route: %s", provider.getLabel(), cluster.Network[i].IP())
|
||||
}
|
||||
} else {
|
||||
// If other error occurs, return error
|
||||
return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Infof("[%s] added route: %s, service: %s/%s, interface: %s, table: %d",
|
||||
provider.getLabel(), cluster.Network[i].IP(), service.Namespace, service.Name, cluster.Network[i].Interface(), sm.config.RoutingTableID)
|
||||
configuredLocalRoutes.Store(string(service.UID), true)
|
||||
leaderElectionActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If BGP mode is enabled - hosts should be added per node
|
||||
if sm.config.EnableBGP {
|
||||
if instance := sm.findServiceInstance(service); instance != nil {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
|
||||
log.Debugf("[%s] attempting to advertise BGP service: %s", provider.getLabel(), address)
|
||||
err := sm.bgpServer.AddHost(address)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error adding BGP host %s\n", err.Error(), provider.getLabel())
|
||||
} else {
|
||||
log.Infof("[%s] added BGP host: %s, service: %s/%s",
|
||||
provider.getLabel(), address, service.Namespace, service.Name)
|
||||
configuredLocalRoutes.Store(string(service.UID), true)
|
||||
leaderElectionActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// There are no local endpoints
|
||||
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
|
||||
// If routing table mode is enabled - routes should be deleted
|
||||
if sm.config.EnableRoutingTable {
|
||||
if errs := sm.clearRoutes(service); len(errs) == 0 {
|
||||
configuredLocalRoutes.Store(string(service.UID), false)
|
||||
}
|
||||
}
|
||||
|
||||
// If BGP mode is enabled - routes should be deleted
|
||||
if sm.config.EnableBGP {
|
||||
if instance := sm.findServiceInstance(service); instance != nil {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
|
||||
err := sm.bgpServer.DelHost(address)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error deleting BGP host%s: %s\n", provider.getLabel(), address, err.Error())
|
||||
} else {
|
||||
log.Infof("[%s] deleted BGP host: %s, service: %s/%s",
|
||||
provider.getLabel(), address, service.Namespace, service.Name)
|
||||
configuredLocalRoutes.Store(string(service.UID), false)
|
||||
leaderElectionActive = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there are no local endpoints, and we had one then remove it and stop the leaderElection
|
||||
if lastKnownGoodEndpoint != "" && sm.config.EnableRoutingTable {
|
||||
log.Warnf("[%s] existing [%s] has been removed, no remaining endpoints for leaderElection", provider.getLabel(), lastKnownGoodEndpoint)
|
||||
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
|
||||
service.Annotations[egressDestinationPorts], service.Namespace); err != nil {
|
||||
log.Errorf("error removing redundant egress rules: %s", err.Error())
|
||||
}
|
||||
|
||||
lastKnownGoodEndpoint = "" // reset endpoint
|
||||
if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
|
||||
cancel() // stop services watcher
|
||||
}
|
||||
leaderElectionActive = false
|
||||
}
|
||||
}
|
||||
// Set the service accordingly
|
||||
if service.Annotations[egress] == "true" {
|
||||
service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
|
||||
}
|
||||
|
||||
log.Debugf("[%s watcher] service %s/%s: local endpoint(s) [%d], known good [%s], active election [%t]",
|
||||
provider.getLabel(), service.Namespace, service.Name, len(endpoints), lastKnownGoodEndpoint, leaderElectionActive)
|
||||
|
||||
case watch.Deleted:
|
||||
// When no-leader-elecition mode
|
||||
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
|
||||
// find all existing local endpoints
|
||||
var endpoints []string
|
||||
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
|
||||
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
if endpoints, err = provider.getAllEndpoints(); err != nil {
|
||||
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
|
||||
}
|
||||
} else {
|
||||
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
|
||||
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// If there were local endpoints deleted
|
||||
if len(endpoints) > 0 {
|
||||
// Delete all routes in routing table mode
|
||||
if sm.config.EnableRoutingTable {
|
||||
sm.clearRoutes(service)
|
||||
}
|
||||
|
||||
// Delete all hosts in BGP mode
|
||||
if sm.config.EnableBGP {
|
||||
sm.clearBGPHosts(service)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
|
||||
close(exitFunction)
|
||||
log.Infof("[%s] deleted stopping watching for [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
|
||||
|
||||
return nil
|
||||
case watch.Error:
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, _ := errObject.(*apierrors.StatusError)
|
||||
log.Errorf("[%s] -> %v", provider.getLabel(), statusErr)
|
||||
}
|
||||
}
|
||||
close(exitFunction)
|
||||
log.Infof("[%s] stopping watching for [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
|
||||
return nil //nolint:govet
|
||||
}
|
||||
|
||||
func (sm *Manager) clearRoutes(service *v1.Service) []error {
|
||||
errs := []error{}
|
||||
if instance := sm.findServiceInstance(service); instance != nil {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
route := cluster.Network[i].PrepareRoute()
|
||||
// check if route we are about to delete is not referenced by more than one service
|
||||
if sm.countRouteReferences(route) <= 1 {
|
||||
err := cluster.Network[i].DeleteRoute()
|
||||
if err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
log.Errorf("failed to delete route for %s: %s", cluster.Network[i].IP(), err.Error())
|
||||
errs = append(errs, err)
|
||||
}
|
||||
log.Debugf("deleted route: %s, service: %s/%s, interface: %s, table: %d",
|
||||
cluster.Network[i].IP(), service.Namespace, service.Name, cluster.Network[i].Interface(), sm.config.RoutingTableID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func (sm *Manager) clearBGPHosts(service *v1.Service) {
|
||||
if instance := sm.findServiceInstance(service); instance != nil {
|
||||
for _, cluster := range instance.clusters {
|
||||
for i := range cluster.Network {
|
||||
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
|
||||
err := sm.bgpServer.DelHost(address)
|
||||
if err != nil {
|
||||
log.Errorf("[endpoint] error deleting BGP host %s\n", err.Error())
|
||||
} else {
|
||||
log.Debugf("[endpoint] deleted BGP host: %s, service: %s/%s",
|
||||
address, service.Namespace, service.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,36 +1,28 @@
|
||||
package providers
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
|
||||
type Endpointslices struct {
|
||||
type endpointslicesProvider struct {
|
||||
label string
|
||||
endpoints *discoveryv1.EndpointSlice
|
||||
}
|
||||
|
||||
func NewEndpointslices() Provider {
|
||||
return &Endpointslices{
|
||||
label: "endpointslices",
|
||||
}
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
|
||||
func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
|
||||
service *v1.Service) (*watchtools.RetryWatcher, error) {
|
||||
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
|
||||
|
||||
@@ -38,9 +30,9 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
|
||||
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
|
||||
}
|
||||
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -50,7 +42,7 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
|
||||
return rw, nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
|
||||
func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
|
||||
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
|
||||
if !ok {
|
||||
cancel()
|
||||
@@ -60,7 +52,7 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
|
||||
func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
|
||||
result := []string{}
|
||||
for _, ep := range ep.endpoints.Endpoints {
|
||||
result = append(result, ep.Addresses...)
|
||||
@@ -68,21 +60,21 @@ func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
|
||||
func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
|
||||
var localEndpoints []string
|
||||
for _, endpoint := range ep.endpoints.Endpoints {
|
||||
if !*endpoint.Conditions.Serving {
|
||||
continue
|
||||
}
|
||||
for _, address := range endpoint.Addresses {
|
||||
log.Debug("processing endpoint", "provider", ep.label, "ip", address)
|
||||
log.Debugf("[%s] processing endpoint [%s]", ep.label, address)
|
||||
|
||||
// 1. Compare the Nodename
|
||||
if endpoint.NodeName != nil && id == *endpoint.NodeName {
|
||||
if endpoint.Hostname != nil {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
|
||||
log.Debugf("[%s] found endpoint - address: %s, hostname: %s, node: %s", ep.label, address, *endpoint.Hostname, *endpoint.NodeName)
|
||||
} else {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
|
||||
log.Debugf("[%s] found endpoint - address: %s, node: %s", ep.label, address, *endpoint.NodeName)
|
||||
}
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
continue
|
||||
@@ -90,7 +82,7 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
|
||||
|
||||
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
|
||||
if endpoint.Hostname != nil && id == *endpoint.Hostname {
|
||||
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
|
||||
log.Debugf("[%s] found endpoint - address: %s, hostname: %s", ep.label, address, *endpoint.Hostname)
|
||||
localEndpoints = append(localEndpoints, address)
|
||||
}
|
||||
}
|
||||
@@ -98,11 +90,11 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
|
||||
return localEndpoints, nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
|
||||
func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
|
||||
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
// Retrieve the latest version of Deployment before attempting update
|
||||
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
|
||||
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
|
||||
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -112,28 +104,28 @@ func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string,
|
||||
currentServiceCopy.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
|
||||
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
|
||||
currentServiceCopy.Annotations[activeEndpoint] = endpoint
|
||||
currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
|
||||
|
||||
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
|
||||
log.Errorf("[%s] error updating Service Spec [%s] : %v", ep.label, currentServiceCopy.Name, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if retryErr != nil {
|
||||
log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
|
||||
log.Errorf("[%s] failed to set Services: %v", ep.label, retryErr)
|
||||
return retryErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetLabel() string {
|
||||
func (ep *endpointslicesProvider) getLabel() string {
|
||||
return ep.label
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) GetProtocol() string {
|
||||
func (ep *endpointslicesProvider) getProtocol() string {
|
||||
return string(ep.endpoints.AddressType)
|
||||
}
|
||||
380
pkg/manager/watch_services.go
Normal file
380
pkg/manager/watch_services.go
Normal file
@@ -0,0 +1,380 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
)
|
||||
|
||||
// TODO: Fix the naming of these contexts
|
||||
|
||||
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
|
||||
var activeServiceLoadBalancer map[string]context.Context
|
||||
|
||||
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
|
||||
var activeServiceLoadBalancerCancel map[string]func()
|
||||
|
||||
// activeService keeps track of services that already have a leaderElection in place
|
||||
var activeService map[string]bool
|
||||
|
||||
// watchedService keeps track of services that are already being watched
|
||||
var watchedService map[string]bool
|
||||
|
||||
// watchedService keeps track of routes that has been configured on the node
|
||||
var configuredLocalRoutes sync.Map
|
||||
|
||||
func init() {
|
||||
// Set up the caches for monitoring existing active or watched services
|
||||
activeServiceLoadBalancerCancel = make(map[string]func())
|
||||
activeServiceLoadBalancer = make(map[string]context.Context)
|
||||
activeService = make(map[string]bool)
|
||||
watchedService = make(map[string]bool)
|
||||
}
|
||||
|
||||
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
|
||||
func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service, *sync.WaitGroup) error) error {
|
||||
// Watch function
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// first start port mirroring if enabled
|
||||
if err := sm.startTrafficMirroringIfEnabled(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
// clean up traffic mirror related config
|
||||
err := sm.stopTrafficMirroringIfEnabled()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if sm.config.ServiceNamespace == "" {
|
||||
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
|
||||
sm.config.ServiceNamespace = v1.NamespaceAll
|
||||
log.Infof("(svcs) starting services watcher for all namespaces")
|
||||
} else {
|
||||
log.Infof("(svcs) starting services watcher for services in namespace [%s]", sm.config.ServiceNamespace)
|
||||
}
|
||||
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating services watcher: %s", err.Error())
|
||||
}
|
||||
exitFunction := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-sm.shutdownChan:
|
||||
log.Debug("(svcs) shutdown called")
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
return
|
||||
case <-exitFunction:
|
||||
log.Debug("(svcs) function ending")
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
return
|
||||
}
|
||||
}()
|
||||
ch := rw.ResultChan()
|
||||
|
||||
// Used for tracking an active endpoint / pod
|
||||
for event := range ch {
|
||||
sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
|
||||
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
|
||||
// We only care about LoadBalancer services
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
break
|
||||
}
|
||||
|
||||
// Check if we ignore this service
|
||||
if svc.Annotations["kube-vip.io/ignore"] == "true" {
|
||||
log.Infof("(svcs) [%s] has an ignore annotation for kube-vip", svc.Name)
|
||||
break
|
||||
}
|
||||
|
||||
// Select loadbalancer class filtering function
|
||||
lbClassFilterFunc := sm.lbClassFilter
|
||||
if sm.config.LoadBalancerClassLegacyHandling {
|
||||
lbClassFilterFunc = sm.lbClassFilterLegacy
|
||||
}
|
||||
|
||||
// Check the loadBalancer class
|
||||
if lbClassFilterFunc(svc) {
|
||||
break
|
||||
}
|
||||
|
||||
svcAddresses := fetchServiceAddresses(svc)
|
||||
|
||||
// We only care about LoadBalancer services that have been allocated an address
|
||||
if len(svcAddresses) <= 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
|
||||
if event.Type == watch.Modified {
|
||||
for _, addr := range svcAddresses {
|
||||
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
|
||||
f, err := vip.GarbageCollect(sm.config.Interface, addr)
|
||||
if err != nil {
|
||||
log.Errorf("(svcs) cleaning existing address error: [%s]", err.Error())
|
||||
}
|
||||
if f {
|
||||
log.Warnf("(svcs) already found existing address [%s] on adapter [%s]", addr, sm.config.Interface)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Scenarios:
|
||||
// 1.
|
||||
if !activeService[string(svc.UID)] {
|
||||
log.Debugf("(svcs) [%s] has been added/modified with addresses [%s]", svc.Name, fetchServiceAddresses(svc))
|
||||
|
||||
wg.Add(1)
|
||||
activeServiceLoadBalancer[string(svc.UID)], activeServiceLoadBalancerCancel[string(svc.UID)] = context.WithCancel(ctx)
|
||||
// Background the services election
|
||||
// EnableServicesElection enabled
|
||||
// watchEndpoint will do a ServicesElection by Service and understands local endpoints
|
||||
//
|
||||
// EnableRoutingTable enabled and EnableLeaderElection disabled
|
||||
// watchEndpoint will also not do a leaderElection by service.
|
||||
if sm.config.EnableServicesElection ||
|
||||
((sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) {
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
// Start an endpoint watcher if we're not watching it already
|
||||
if !watchedService[string(svc.UID)] {
|
||||
// background the endpoint watcher
|
||||
go func() {
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
// Add Endpoint or EndpointSlices watcher
|
||||
wg.Add(1)
|
||||
var provider epProvider
|
||||
if !sm.config.EnableEndpointSlices {
|
||||
provider = &endpointsProvider{label: "endpoints"}
|
||||
} else {
|
||||
provider = &endpointslicesProvider{label: "endpointslices"}
|
||||
}
|
||||
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, &wg, provider); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
}()
|
||||
|
||||
if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
// We're now watching this service
|
||||
watchedService[string(svc.UID)] = true
|
||||
}
|
||||
} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
|
||||
go func() {
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
// Add Endpoint watcher
|
||||
wg.Add(1)
|
||||
var provider epProvider
|
||||
if !sm.config.EnableEndpointSlices {
|
||||
provider = &endpointsProvider{label: "endpoints"}
|
||||
} else {
|
||||
provider = &endpointslicesProvider{label: "endpointslices"}
|
||||
}
|
||||
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, &wg, provider); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
} else {
|
||||
// Increment the waitGroup before the service Func is called (Done is completed in there)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
} else {
|
||||
// Increment the waitGroup before the service Func is called (Done is completed in there)
|
||||
wg.Add(1)
|
||||
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
activeService[string(svc.UID)] = true
|
||||
}
|
||||
case watch.Deleted:
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if activeService[string(svc.UID)] {
|
||||
|
||||
// We only care about LoadBalancer services
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
break
|
||||
}
|
||||
|
||||
// We can ignore this service
|
||||
if svc.Annotations["kube-vip.io/ignore"] == "true" {
|
||||
log.Infof("(svcs) [%s] has an ignore annotation for kube-vip", svc.Name)
|
||||
break
|
||||
}
|
||||
|
||||
isRouteConfigured, err := isRouteConfigured(svc.UID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error while checkig if route is configured: %w", err)
|
||||
}
|
||||
// If no leader election is enabled, delete routes here
|
||||
if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
|
||||
sm.config.EnableRoutingTable && isRouteConfigured {
|
||||
if errs := sm.clearRoutes(svc); len(errs) == 0 {
|
||||
configuredLocalRoutes.Store(string(svc.UID), false)
|
||||
}
|
||||
}
|
||||
|
||||
// If this is an active service then and additional leaderElection will handle stopping
|
||||
err = sm.deleteService(string(svc.UID))
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
// Calls the cancel function of the context
|
||||
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
|
||||
activeServiceLoadBalancerCancel[string(svc.UID)]()
|
||||
}
|
||||
activeService[string(svc.UID)] = false
|
||||
watchedService[string(svc.UID)] = false
|
||||
}
|
||||
|
||||
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
|
||||
if sm.config.EnableBGP {
|
||||
instance := sm.findServiceInstance(svc)
|
||||
for _, vip := range instance.vipConfigs {
|
||||
vipCidr := fmt.Sprintf("%s/%s", vip.VIP, vip.VIPCIDR)
|
||||
err = sm.bgpServer.DelHost(vipCidr)
|
||||
if err != nil {
|
||||
log.Errorf("error deleting host %s: %s", vipCidr, err.Error())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sm.clearRoutes(svc)
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("(svcs) [%s/%s] has been deleted", svc.Namespace, svc.Name)
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes services")
|
||||
|
||||
// This round trip allows us to handle unstructured status
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Errorf("services -> %v", status)
|
||||
default:
|
||||
}
|
||||
}
|
||||
close(exitFunction)
|
||||
log.Warnln("Stopping watching services for type: LoadBalancer in all namespaces")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
|
||||
if svc == nil {
|
||||
log.Infof("(svcs) service is nil, ignoring")
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass != nil {
|
||||
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
|
||||
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
|
||||
log.Infof("(svcs) [%s] specified the loadBalancer class [%s], ignoring", svc.Name, *svc.Spec.LoadBalancerClass)
|
||||
return true
|
||||
}
|
||||
} else if sm.config.LoadBalancerClassOnly {
|
||||
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
|
||||
log.Infof("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service [%s] didn't specify any loadBalancer class, ignoring", svc.Name)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
|
||||
if svc == nil {
|
||||
log.Infof("(svcs) service is nil, ignoring")
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
|
||||
log.Infof("(svcs) [%s] specified no loadBalancer class, expected [%s], ignoring", svc.Name, sm.config.LoadBalancerClassName)
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
|
||||
return false
|
||||
}
|
||||
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
|
||||
log.Infof("(svcs) [%s] specified loadBalancer class [%s], expected [%s], ignoring", svc.Name, *svc.Spec.LoadBalancerClass, sm.config.LoadBalancerClassName)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isRouteConfigured(serviceUID types.UID) (bool, error) {
|
||||
isConfigured := false
|
||||
value, ok := configuredLocalRoutes.Load(string(serviceUID))
|
||||
if ok {
|
||||
isConfigured, ok = value.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("error converting configuredLocalRoute item to boolean value")
|
||||
}
|
||||
}
|
||||
|
||||
return isConfigured, nil
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package networkinterface
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
interfaces map[string]*Link
|
||||
}
|
||||
|
||||
type Link struct {
|
||||
Lock sync.Mutex
|
||||
Intf netlink.Link
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{
|
||||
interfaces: make(map[string]*Link),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Get(intf netlink.Link) *Link {
|
||||
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
|
||||
return l
|
||||
}
|
||||
result := &Link{
|
||||
Intf: intf,
|
||||
}
|
||||
|
||||
m.interfaces[intf.Attrs().Name] = result
|
||||
return result
|
||||
}
|
||||
@@ -1,458 +0,0 @@
|
||||
package nftables
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/nftables"
|
||||
"github.com/google/nftables/binaryutil"
|
||||
"github.com/google/nftables/expr"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
NatTable = "kube_vip_%s"
|
||||
SNatChain = "kube_vip_snat_%s"
|
||||
)
|
||||
|
||||
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
|
||||
|
||||
conn, err := nftables.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableName string
|
||||
if IPv6 {
|
||||
tableName = fmt.Sprintf(NatTable, "v6")
|
||||
} else {
|
||||
tableName = fmt.Sprintf(NatTable, "v4")
|
||||
}
|
||||
// Look up the table
|
||||
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
|
||||
if t == nil {
|
||||
// If it doesn't exist then create it
|
||||
slog.Debug("[egress]", "Creating Table", tableName)
|
||||
conn.AddTable(GetTable(IPv6))
|
||||
}
|
||||
}
|
||||
slog.Debug("[egress]", "Creating Chain for service", service, "IPv6", IPv6)
|
||||
// These don't return errors, so not 100% sure how to guarantee things were created
|
||||
conn.AddChain(GetSNatChain(IPv6, service))
|
||||
conn.Flush()
|
||||
// Create our nftables rule
|
||||
rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
|
||||
conn.AddRule(rule) // Add the rule
|
||||
|
||||
err = conn.Flush() // Commit the rule to nftables
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return conn.CloseLasting() // Close out any remaining netlink communication
|
||||
}
|
||||
|
||||
func DeleteSNAT(IPv6 bool, service string) error {
|
||||
conn, err := nftables.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var chainName = fmt.Sprintf(SNatChain, service)
|
||||
slog.Info("[egress]", "Looking for", chainName)
|
||||
|
||||
chain, err := conn.ListChain(GetTable(IPv6), chainName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if chain != nil {
|
||||
slog.Info("[egress]", "Deleting chain", chainName)
|
||||
conn.DelChain(chain)
|
||||
return conn.Flush()
|
||||
|
||||
}
|
||||
|
||||
return fmt.Errorf("unable to find chain [%s]", chainName)
|
||||
}
|
||||
|
||||
func GetTable(IPv6 bool) *nftables.Table {
|
||||
var tableName string
|
||||
if IPv6 {
|
||||
tableName = fmt.Sprintf(NatTable, "v6")
|
||||
} else {
|
||||
tableName = fmt.Sprintf(NatTable, "v4")
|
||||
}
|
||||
// Default to IPv4
|
||||
table := &nftables.Table{
|
||||
Family: nftables.TableFamilyIPv4,
|
||||
Name: tableName,
|
||||
}
|
||||
|
||||
// Move to IPv6 if needed
|
||||
if IPv6 {
|
||||
table.Family = nftables.TableFamilyIPv6
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
|
||||
var chainName = fmt.Sprintf(SNatChain, service)
|
||||
policy := nftables.ChainPolicyAccept
|
||||
return &nftables.Chain{
|
||||
Name: chainName,
|
||||
Table: GetTable(IPv6),
|
||||
Type: nftables.ChainTypeNAT,
|
||||
Hooknum: nftables.ChainHookPostrouting,
|
||||
Priority: nftables.ChainPriorityNATSource,
|
||||
Policy: &policy,
|
||||
}
|
||||
}
|
||||
|
||||
func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
|
||||
if IPv6 {
|
||||
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
|
||||
}
|
||||
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
|
||||
}
|
||||
|
||||
// ClearTable will remove the original tables and create new empty ones
|
||||
func ClearTable(conn *nftables.Conn) error {
|
||||
tableName := fmt.Sprintf(NatTable, "v6")
|
||||
if t, err := FilterTable(conn, tableName, false); err != nil {
|
||||
return err
|
||||
} else if t != nil {
|
||||
conn.DelTable(t)
|
||||
}
|
||||
|
||||
// These don't return errors, so not 100% sure how to guarantee things were created
|
||||
conn.AddTable(GetTable(true))
|
||||
tableName = fmt.Sprintf(NatTable, "v4")
|
||||
if t, err := FilterTable(conn, tableName, true); err != nil {
|
||||
return err
|
||||
} else if t != nil {
|
||||
conn.DelTable(t)
|
||||
}
|
||||
|
||||
// These don't return errors, so not 100% sure how to guarantee things were created
|
||||
conn.AddTable(GetTable(false))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create our nftables rule
|
||||
func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
|
||||
|
||||
// Validate pod IP
|
||||
if net.ParseIP(podIP) == nil {
|
||||
return nil, errors.New("ip is invalid")
|
||||
}
|
||||
|
||||
// Validate vip IP
|
||||
if net.ParseIP(vipIP) == nil {
|
||||
return nil, errors.New("output_ip is not a valid ip")
|
||||
}
|
||||
|
||||
// Get the kube-vip table
|
||||
table := GetTable(IPv6)
|
||||
|
||||
// Create our rule
|
||||
rule := &nftables.Rule{
|
||||
Table: table,
|
||||
Exprs: []expr.Any{},
|
||||
}
|
||||
// Set the correct chain
|
||||
rule.Chain = GetSNatChain(IPv6, service)
|
||||
|
||||
// Create a set for our original/source address
|
||||
set := &nftables.Set{
|
||||
Table: table,
|
||||
Anonymous: true,
|
||||
Constant: true,
|
||||
KeyType: nftables.TypeIPAddr,
|
||||
Interval: false,
|
||||
}
|
||||
if IPv6 {
|
||||
set.KeyType = nftables.TypeIP6Addr
|
||||
} else {
|
||||
set.KeyType = nftables.TypeIPAddr
|
||||
}
|
||||
|
||||
// Create an element using our pod IP
|
||||
elements := []nftables.SetElement{}
|
||||
if IPv6 {
|
||||
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
|
||||
} else {
|
||||
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
|
||||
}
|
||||
|
||||
// Add the elements to the set
|
||||
err := conn.AddSet(set, elements)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create the expression using the set
|
||||
expression := []expr.Any{}
|
||||
|
||||
payload := &expr.Payload{
|
||||
OperationType: expr.PayloadLoad,
|
||||
Base: expr.PayloadBaseNetworkHeader,
|
||||
DestRegister: 1,
|
||||
SourceRegister: 0,
|
||||
}
|
||||
|
||||
// Set the length of the data based upon the type of IP version being used
|
||||
if IPv6 {
|
||||
payload.Offset = 8
|
||||
payload.Len = 16
|
||||
} else {
|
||||
payload.Offset = 12
|
||||
payload.Len = 4
|
||||
}
|
||||
lookup := &expr.Lookup{
|
||||
SourceRegister: 1,
|
||||
DestRegister: 0,
|
||||
SetID: set.ID,
|
||||
}
|
||||
|
||||
// Add expressions
|
||||
expression = append(expression, payload)
|
||||
expression = append(expression, lookup)
|
||||
|
||||
// Add expression to the rule
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
|
||||
// If we filter on ports protocols then parse them
|
||||
if destinationPorts != "" {
|
||||
fixedPorts := strings.Split(destinationPorts, ",")
|
||||
|
||||
// Create an element using our pod IP
|
||||
tcpElements := []nftables.SetElement{}
|
||||
udpElements := []nftables.SetElement{}
|
||||
sctpElements := []nftables.SetElement{}
|
||||
|
||||
tcpSet := &nftables.Set{
|
||||
Anonymous: true,
|
||||
Constant: true,
|
||||
Table: table,
|
||||
KeyType: nftables.TypeInetService,
|
||||
}
|
||||
udpSet := &nftables.Set{
|
||||
Anonymous: true,
|
||||
Constant: true,
|
||||
Table: table,
|
||||
KeyType: nftables.TypeInetService,
|
||||
}
|
||||
sctpSet := &nftables.Set{
|
||||
Anonymous: true,
|
||||
Constant: true,
|
||||
Table: table,
|
||||
KeyType: nftables.TypeInetService,
|
||||
}
|
||||
for _, fixedPort := range fixedPorts {
|
||||
data := strings.Split(fixedPort, ":")
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
} else if len(data) == 2 { // Ensure we have two elements { proto:port }
|
||||
// parse the port to a number
|
||||
port, err := strconv.Atoi(data[1])
|
||||
if err != nil {
|
||||
slog.Error("[egress]", "unable to process port", data[1])
|
||||
continue
|
||||
}
|
||||
// Ensure the port is within the valid range for uint16
|
||||
if port < 0 || port > 65535 {
|
||||
slog.Error("[egress]", "port out of range for uint16", data[1])
|
||||
continue
|
||||
}
|
||||
|
||||
switch data[0] {
|
||||
case "tcp":
|
||||
//nolint:gosec
|
||||
tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
|
||||
case "udp":
|
||||
//nolint:gosec
|
||||
udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
|
||||
case "sctp":
|
||||
//nolint:gosec
|
||||
sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
|
||||
default:
|
||||
slog.Error("[egress]", "unknown protocol", data[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add TCP Ports
|
||||
if len(tcpElements) != 0 {
|
||||
err = conn.AddSet(tcpSet, tcpElements)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression := []expr.Any{
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
// [ cmp eq reg 1 0x00000006 ]
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_TCP},
|
||||
},
|
||||
|
||||
// [ payload load 2b @ transport header + 2 => reg 1 ]
|
||||
&expr.Payload{
|
||||
DestRegister: 1,
|
||||
Base: expr.PayloadBaseTransportHeader,
|
||||
Offset: 2,
|
||||
Len: 2,
|
||||
},
|
||||
// [ lookup reg 1 set __set%d ]
|
||||
&expr.Lookup{
|
||||
SourceRegister: 1,
|
||||
SetName: tcpSet.Name,
|
||||
SetID: tcpSet.ID,
|
||||
},
|
||||
}
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
}
|
||||
|
||||
// Add UDP ports
|
||||
if len(udpElements) != 0 {
|
||||
err = conn.AddSet(udpSet, udpElements)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression := []expr.Any{
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
// [ cmp eq reg 1 0x00000006 ]
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_UDP},
|
||||
},
|
||||
|
||||
// [ payload load 2b @ transport header + 2 => reg 1 ]
|
||||
&expr.Payload{
|
||||
DestRegister: 1,
|
||||
Base: expr.PayloadBaseTransportHeader,
|
||||
Offset: 2,
|
||||
Len: 2,
|
||||
},
|
||||
// [ lookup reg 1 set __set%d ]
|
||||
&expr.Lookup{
|
||||
SourceRegister: 1,
|
||||
SetName: udpSet.Name,
|
||||
SetID: udpSet.ID,
|
||||
},
|
||||
}
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
}
|
||||
|
||||
// Add SCTP Ports
|
||||
if len(sctpElements) != 0 {
|
||||
err = conn.AddSet(sctpSet, sctpElements)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression := []expr.Any{
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
// [ cmp eq reg 1 0x00000006 ]
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_SCTP},
|
||||
},
|
||||
|
||||
// [ payload load 2b @ transport header + 2 => reg 1 ]
|
||||
&expr.Payload{
|
||||
DestRegister: 1,
|
||||
Base: expr.PayloadBaseTransportHeader,
|
||||
Offset: 2,
|
||||
Len: 2,
|
||||
},
|
||||
// [ lookup reg 1 set __set%d ]
|
||||
&expr.Lookup{
|
||||
SourceRegister: 1,
|
||||
SetName: sctpSet.Name,
|
||||
SetID: sctpSet.ID,
|
||||
},
|
||||
}
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse which CIDRs we will not SNAT for
|
||||
for _, cidr := range ignoreCIDR {
|
||||
start, end, err := nftables.NetFirstAndLastIP(cidr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression = []expr.Any{}
|
||||
|
||||
payload := &expr.Payload{
|
||||
DestRegister: 1,
|
||||
Base: expr.PayloadBaseNetworkHeader,
|
||||
}
|
||||
notEqualRange := &expr.Range{
|
||||
Op: expr.CmpOpNeq,
|
||||
Register: 1,
|
||||
}
|
||||
|
||||
if IPv6 {
|
||||
payload.Len = 16
|
||||
payload.Offset = 24
|
||||
notEqualRange.FromData = start.To16()
|
||||
notEqualRange.ToData = end.To16()
|
||||
} else {
|
||||
payload.Offset = 16
|
||||
payload.Len = 4
|
||||
notEqualRange.FromData = start.To4()
|
||||
notEqualRange.ToData = end.To4()
|
||||
}
|
||||
// Add expressions
|
||||
expression = append(expression, payload)
|
||||
expression = append(expression, notEqualRange)
|
||||
|
||||
// // Add expression to the rule
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
}
|
||||
|
||||
// Final expression to the rule is the SNAT to the VIP address
|
||||
expression = []expr.Any{}
|
||||
|
||||
immediate := &expr.Immediate{
|
||||
Register: 1,
|
||||
}
|
||||
|
||||
nat := &expr.NAT{
|
||||
Type: expr.NATTypeSourceNAT,
|
||||
RegAddrMin: 1,
|
||||
RegAddrMax: 1,
|
||||
RegProtoMin: 0,
|
||||
RegProtoMax: 0,
|
||||
Random: false,
|
||||
FullyRandom: false,
|
||||
Persistent: false,
|
||||
Prefix: false,
|
||||
}
|
||||
|
||||
if IPv6 {
|
||||
immediate.Data = net.ParseIP(vipIP).To16()
|
||||
nat.Family = unix.NFPROTO_IPV6
|
||||
} else {
|
||||
immediate.Data = net.ParseIP(vipIP).To4()
|
||||
nat.Family = unix.NFPROTO_IPV4
|
||||
}
|
||||
// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
|
||||
// Add expressions
|
||||
expression = append(expression, immediate)
|
||||
expression = append(expression, nat)
|
||||
rule.Exprs = append(rule.Exprs, expression...)
|
||||
|
||||
return rule, nil
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package servicecontext
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Context struct {
|
||||
Ctx context.Context
|
||||
Cancel context.CancelFunc
|
||||
IsActive bool
|
||||
IsWatched bool
|
||||
ConfiguredNetworks sync.Map
|
||||
}
|
||||
|
||||
func New(ctx context.Context) *Context {
|
||||
svcCtx, svcCancel := context.WithCancel(ctx)
|
||||
return &Context{
|
||||
Ctx: svcCtx,
|
||||
Cancel: svcCancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) HasConfiguredNetworks() bool {
|
||||
cnt := 0
|
||||
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
|
||||
cnt++
|
||||
return cnt < 1
|
||||
})
|
||||
return cnt > 0
|
||||
}
|
||||
|
||||
func (ctx *Context) IsNetworkConfigured(ip string) bool {
|
||||
_, exists := ctx.ConfiguredNetworks.Load(ip)
|
||||
return exists
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Context struct {
|
||||
Ctx context.Context
|
||||
Cancel context.CancelFunc
|
||||
IsActive bool
|
||||
IsWatched bool
|
||||
ConfiguredNetworks sync.Map
|
||||
}
|
||||
|
||||
func NewContext(ctx context.Context) *Context {
|
||||
svcCtx, svcCancel := context.WithCancel(ctx)
|
||||
return &Context{
|
||||
Ctx: svcCtx,
|
||||
Cancel: svcCancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) HasConfiguredNetworks() bool {
|
||||
cnt := 0
|
||||
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
|
||||
cnt++
|
||||
return cnt < 1
|
||||
})
|
||||
return cnt > 0
|
||||
}
|
||||
|
||||
func (ctx *Context) IsNetworkConfigured(ip string) bool {
|
||||
_, exists := ctx.ConfiguredNetworks.Load(ip)
|
||||
return exists
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
)
|
||||
|
||||
var (
|
||||
svcLocks map[string]*sync.Mutex
|
||||
)
|
||||
|
||||
func init() {
|
||||
svcLocks = make(map[string]*sync.Mutex)
|
||||
}
|
||||
|
||||
// The StartServicesWatchForLeaderElection function will start a services watcher, the
|
||||
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
|
||||
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, instance := range p.ServiceInstances {
|
||||
for _, cluster := range instance.Clusters {
|
||||
for i := range cluster.Network {
|
||||
_ = cluster.Network[i].DeleteRoute()
|
||||
}
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Shutting down kube-Vip")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// The startServicesWatchForLeaderElection function will start a services watcher, the
|
||||
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
|
||||
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
|
||||
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
|
||||
// we use the Lease lock type since edits to Leases are less common
|
||||
// and fewer objects in the cluster watch "all Leases".
|
||||
lock := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{
|
||||
Name: serviceLease,
|
||||
Namespace: service.Namespace,
|
||||
},
|
||||
Client: p.clientSet.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: p.config.NodeName,
|
||||
},
|
||||
}
|
||||
childCtx, childCancel := context.WithCancel(ctx)
|
||||
defer childCancel()
|
||||
|
||||
if _, ok := svcLocks[serviceLease]; !ok {
|
||||
svcLocks[serviceLease] = new(sync.Mutex)
|
||||
}
|
||||
|
||||
svcLocks[serviceLease].Lock()
|
||||
defer svcLocks[serviceLease].Unlock()
|
||||
|
||||
svcCtx, err := p.getServiceContext(service.UID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
|
||||
}
|
||||
if svcCtx == nil {
|
||||
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
|
||||
}
|
||||
|
||||
svcCtx.IsActive = true
|
||||
|
||||
// start the leader election code loop
|
||||
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
|
||||
Lock: lock,
|
||||
// IMPORTANT: you MUST ensure that any code you have that
|
||||
// is protected by the lease must terminate **before**
|
||||
// you call cancel. Otherwise, you could have a background
|
||||
// loop still running and another process could
|
||||
// get elected before your background loop finished, violating
|
||||
// the stated goal of the lease.
|
||||
ReleaseOnCancel: true,
|
||||
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
|
||||
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
|
||||
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(ctx context.Context) {
|
||||
// Mark this service as active (as we've started leading)
|
||||
// we run this in background as it's blocking
|
||||
if err := p.SyncServices(ctx, service); err != nil {
|
||||
log.Error("service sync", "err", err)
|
||||
childCancel()
|
||||
}
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
// we can do cleanup here
|
||||
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
|
||||
if svcCtx.IsActive {
|
||||
if err := p.deleteService(service.UID); err != nil {
|
||||
log.Error("service deletion", "err", err)
|
||||
}
|
||||
}
|
||||
// Mark this service is inactive
|
||||
svcCtx.IsActive = false
|
||||
},
|
||||
OnNewLeader: func(identity string) {
|
||||
// we're notified when new leader elected
|
||||
if identity == p.config.NodeName {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
log.Info("new leader", "leader", identity)
|
||||
},
|
||||
},
|
||||
})
|
||||
log.Info("stopping leader election", "service", service.Name)
|
||||
return nil
|
||||
}
|
||||
@@ -1,344 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
type Processor struct {
|
||||
config *kubevip.Config
|
||||
lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
|
||||
svcMap sync.Map
|
||||
|
||||
// Keeps track of all running instances
|
||||
ServiceInstances []*instance.Instance
|
||||
|
||||
mutex sync.Mutex
|
||||
bgpServer *bgp.Server
|
||||
|
||||
clientSet *kubernetes.Clientset
|
||||
rwClientSet *kubernetes.Clientset
|
||||
|
||||
shutdownChan chan struct{}
|
||||
|
||||
// This is a prometheus counter used to count the number of events received
|
||||
// from the service watcher
|
||||
CountServiceWatchEvent *prometheus.CounterVec
|
||||
|
||||
intfMgr *networkinterface.Manager
|
||||
arpMgr *arp.Manager
|
||||
}
|
||||
|
||||
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
|
||||
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
|
||||
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
|
||||
lbClassFilterFunc := lbClassFilter
|
||||
if config.LoadBalancerClassLegacyHandling {
|
||||
lbClassFilterFunc = lbClassFilterLegacy
|
||||
}
|
||||
|
||||
return &Processor{
|
||||
config: config,
|
||||
lbClassFilter: lbClassFilterFunc,
|
||||
ServiceInstances: []*instance.Instance{},
|
||||
bgpServer: bgpServer,
|
||||
clientSet: clientSet,
|
||||
rwClientSet: rwClientSet,
|
||||
shutdownChan: shutdownChan,
|
||||
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "kube_vip",
|
||||
Subsystem: "manager",
|
||||
Name: "all_services_events",
|
||||
Help: "Count all events fired by the service watcher categorised by event type",
|
||||
}, []string{"type"}),
|
||||
|
||||
intfMgr: intfMgr,
|
||||
arpMgr: arpMgr,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
|
||||
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
|
||||
// We only care about LoadBalancer services
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Check if we ignore this service
|
||||
if svc.Annotations["kube-vip.io/ignore"] == "true" {
|
||||
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Check the loadBalancer class
|
||||
if p.lbClassFilter(svc, p.config) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
svcAddresses := instance.FetchServiceAddresses(svc)
|
||||
|
||||
// We only care about LoadBalancer services that have been allocated an address
|
||||
if len(svcAddresses) <= 0 {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
svcCtx, err := p.getServiceContext(svc.UID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to get service context: %w", err)
|
||||
}
|
||||
|
||||
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
|
||||
if event.Type == watch.Modified {
|
||||
i := instance.FindServiceInstance(svc, p.ServiceInstances)
|
||||
var originalService []string
|
||||
shouldGarbageCollect := true
|
||||
if i != nil {
|
||||
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
|
||||
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
|
||||
}
|
||||
if shouldGarbageCollect {
|
||||
for _, addr := range svcAddresses {
|
||||
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
|
||||
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
|
||||
if err != nil {
|
||||
log.Error("(svcs) cleaning existing address error", "err", err)
|
||||
}
|
||||
if f {
|
||||
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
|
||||
}
|
||||
}
|
||||
}
|
||||
// This service has been modified, but it was also active.
|
||||
if svcCtx != nil && svcCtx.IsActive {
|
||||
if i != nil {
|
||||
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
|
||||
newService := instance.FetchServiceAddresses(svc)
|
||||
if !reflect.DeepEqual(originalService, newService) {
|
||||
|
||||
// Calls the cancel function of the context
|
||||
if svcCtx != nil {
|
||||
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
|
||||
svcCtx.Cancel()
|
||||
log.Warn("(svcs) waiting for load balancer to finish")
|
||||
<-svcCtx.Ctx.Done()
|
||||
}
|
||||
|
||||
err = p.deleteService(svc.UID)
|
||||
if err != nil {
|
||||
log.Error("(svc) unable to remove", "service", svc.UID)
|
||||
}
|
||||
|
||||
p.svcMap.Delete(svc.UID)
|
||||
}
|
||||
// in theory this should never fail
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
|
||||
|
||||
// Is the service active (bool), if not then process this new service
|
||||
// Does this service use an election per service?
|
||||
//
|
||||
|
||||
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
|
||||
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
|
||||
|
||||
if svcCtx == nil {
|
||||
svcCtx = servicecontext.New(ctx)
|
||||
p.svcMap.Store(svc.UID, svcCtx)
|
||||
}
|
||||
|
||||
if p.config.EnableServicesElection || // Service Election
|
||||
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
|
||||
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
|
||||
|
||||
// If this load balancer Traffic Policy is "local"
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
|
||||
// Start an endpoint watcher if we're not watching it already
|
||||
if !svcCtx.IsWatched {
|
||||
// background the endpoint watcher
|
||||
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
|
||||
err = serviceFunc(svcCtx.Ctx, svc)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
// Add Endpoint or EndpointSlices watcher
|
||||
var provider providers.Provider
|
||||
if p.config.EnableEndpoints {
|
||||
provider = providers.NewEndpoints()
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// We're now watching this service
|
||||
svcCtx.IsWatched = true
|
||||
}
|
||||
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
|
||||
err = serviceFunc(svcCtx.Ctx, svc)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
|
||||
go func() {
|
||||
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
|
||||
// Add Endpoint watcher
|
||||
var provider providers.Provider
|
||||
if p.config.EnableEndpoints {
|
||||
provider = providers.NewEndpoints()
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
// We're now watching this service
|
||||
svcCtx.IsWatched = true
|
||||
} else {
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-svcCtx.Ctx.Done():
|
||||
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
|
||||
return
|
||||
default:
|
||||
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
|
||||
err = serviceFunc(svcCtx.Ctx, svc)
|
||||
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}()
|
||||
}
|
||||
} else {
|
||||
// Increment the waitGroup before the service Func is called (Done is completed in there)
|
||||
err = serviceFunc(svcCtx.Ctx, svc)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
svcCtx.IsActive = true
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Delete(event watch.Event) (bool, error) {
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
svcCtx, err := p.getServiceContext(svc.UID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
|
||||
}
|
||||
if svcCtx != nil && svcCtx.IsActive {
|
||||
// We only care about LoadBalancer services
|
||||
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// We can ignore this service
|
||||
if svc.Annotations["kube-vip.io/ignore"] == "true" {
|
||||
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// If no leader election is enabled, delete routes here
|
||||
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
|
||||
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
|
||||
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
|
||||
svcCtx.ConfiguredNetworks.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
// If this is an active service then and additional leaderElection will handle stopping
|
||||
err = p.deleteService(svc.UID)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
|
||||
// Calls the cancel function of the context
|
||||
log.Warn("(svcs) The load balancer was deleted, cancelling context")
|
||||
svcCtx.Cancel()
|
||||
log.Warn("(svcs) waiting for load balancer to finish")
|
||||
<-svcCtx.Ctx.Done()
|
||||
p.svcMap.Delete(svc.UID)
|
||||
}
|
||||
|
||||
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
|
||||
if p.config.EnableBGP {
|
||||
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
|
||||
} else if p.config.EnableRoutingTable {
|
||||
endpoints.ClearRoutes(svc, &p.ServiceInstances)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Stop() {
|
||||
for _, instance := range p.ServiceInstances {
|
||||
for _, cluster := range instance.Clusters {
|
||||
cluster.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
|
||||
svcCtx, ok := p.svcMap.Load(uid)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
ctx, ok := svcCtx.(*servicecontext.Context)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
|
||||
}
|
||||
return ctx, nil
|
||||
}
|
||||
|
||||
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
|
||||
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
|
||||
}
|
||||
@@ -1,516 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/util/retry"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/egress"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/upnp"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
)
|
||||
|
||||
type ServiceInstanceAction string
|
||||
|
||||
const (
|
||||
ActionDelete ServiceInstanceAction = "delete"
|
||||
ActionAdd ServiceInstanceAction = "add"
|
||||
ActionNone ServiceInstanceAction = "none"
|
||||
)
|
||||
|
||||
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
|
||||
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
|
||||
|
||||
// Iterate through the synchronising services
|
||||
|
||||
action := p.getServiceInstanceAction(svc)
|
||||
switch action {
|
||||
case ActionDelete:
|
||||
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
|
||||
if err := p.deleteService(svc.UID); err != nil {
|
||||
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
|
||||
}
|
||||
case ActionAdd:
|
||||
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
|
||||
if err := p.addService(ctx, svc); err != nil {
|
||||
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
|
||||
}
|
||||
case ActionNone:
|
||||
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
|
||||
}
|
||||
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
|
||||
// protect against multiple calls
|
||||
addresses := instance.FetchServiceAddresses(svc)
|
||||
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
for _, instance := range p.ServiceInstances {
|
||||
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
|
||||
for _, address := range addresses {
|
||||
// handle the case where the service instance needs to be deleted
|
||||
if instance.IsDHCP {
|
||||
if address != "0.0.0.0" {
|
||||
return ActionDelete
|
||||
}
|
||||
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
|
||||
return ActionDelete
|
||||
}
|
||||
}
|
||||
if !instance.IsDHCP {
|
||||
if address == "0.0.0.0" {
|
||||
return ActionDelete
|
||||
}
|
||||
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
|
||||
return ActionDelete
|
||||
}
|
||||
}
|
||||
if !comparePortsAndPortStatuses(svc) {
|
||||
return ActionDelete
|
||||
}
|
||||
}
|
||||
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
|
||||
return ActionNone
|
||||
}
|
||||
}
|
||||
if len(addresses) > 0 {
|
||||
log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
|
||||
return ActionAdd // If no matching instance is found, we need to add a new service instance
|
||||
}
|
||||
return ActionNone
|
||||
}
|
||||
|
||||
func comparePortsAndPortStatuses(svc *v1.Service) bool {
|
||||
if len(svc.Status.LoadBalancer.Ingress) == 0 {
|
||||
return false
|
||||
}
|
||||
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
|
||||
if len(portsStatus) != len(svc.Spec.Ports) {
|
||||
return false
|
||||
}
|
||||
for i, portSpec := range svc.Spec.Ports {
|
||||
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
|
||||
// protect against addService while reading
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for x := range newService.VIPConfigs {
|
||||
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
|
||||
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
|
||||
}
|
||||
|
||||
p.upnpMap(ctx, newService)
|
||||
|
||||
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
|
||||
go func() {
|
||||
for ip := range newService.DHCPClient.IPChannel() {
|
||||
log.Debug("IP changed", "ip", ip)
|
||||
newService.VIPConfigs[0].VIP = ip
|
||||
newService.DHCPInterfaceIP = ip
|
||||
if !p.config.DisableServiceUpdates {
|
||||
if err := p.updateStatus(newService); err != nil {
|
||||
log.Warn("updating svc", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Debug("IP update channel closed, stopping")
|
||||
}()
|
||||
}
|
||||
|
||||
p.ServiceInstances = append(p.ServiceInstances, newService)
|
||||
|
||||
if !p.config.DisableServiceUpdates {
|
||||
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
|
||||
if err := p.updateStatus(newService); err != nil {
|
||||
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
serviceIPs := instance.FetchServiceAddresses(svc)
|
||||
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
|
||||
if svc.Annotations[kubevip.FlushContrack] == "true" {
|
||||
|
||||
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
|
||||
for _, serviceIP := range serviceIPs {
|
||||
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
|
||||
if err != nil {
|
||||
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
|
||||
if err != nil {
|
||||
log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if egress is enabled on the service, if so we'll need to configure some rules
|
||||
if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
|
||||
log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
|
||||
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
|
||||
if p.config.EgressWithNftables {
|
||||
// Ensure that kernel modules are loaded and report back missing modules.
|
||||
err = p.nftablesCheck()
|
||||
if err != nil {
|
||||
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
} else {
|
||||
// Ensure that kernel modules are loaded and report back missing modules.
|
||||
err = p.iptablesCheck()
|
||||
if err != nil {
|
||||
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
var podIP string
|
||||
errList := []error{}
|
||||
|
||||
// Should egress be IPv6
|
||||
if svc.Annotations[kubevip.EgressIPv6] == "true" {
|
||||
// Does the service have an active IPv6 endpoint
|
||||
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
|
||||
for _, serviceIP := range serviceIPs {
|
||||
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
|
||||
|
||||
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
|
||||
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
|
||||
for _, serviceIP := range serviceIPs {
|
||||
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
|
||||
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
|
||||
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
|
||||
}
|
||||
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(errList) == 0 {
|
||||
var provider providers.Provider
|
||||
if p.config.EnableEndpoints {
|
||||
provider = providers.NewEndpoints()
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
|
||||
if err != nil {
|
||||
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
finishTime := time.Since(startTime)
|
||||
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) deleteService(uid types.UID) error {
|
||||
// protect multiple calls
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
var updatedInstances []*instance.Instance
|
||||
var serviceInstance *instance.Instance
|
||||
found := false
|
||||
for x := range p.ServiceInstances {
|
||||
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
|
||||
// Add the running services to the new array
|
||||
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
|
||||
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
|
||||
} else {
|
||||
// Flip the found when we match
|
||||
found = true
|
||||
serviceInstance = p.ServiceInstances[x]
|
||||
}
|
||||
}
|
||||
// If we've been through all services and not found the correct one then error
|
||||
if !found {
|
||||
// TODO: - fix UX
|
||||
// return fmt.Errorf("unable to find/stop service [%s]", uid)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, c := range serviceInstance.Clusters {
|
||||
for n := range c.Network {
|
||||
c.Network[n].SetHasEndpoints(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine if this this VIP is shared with other loadbalancers
|
||||
shared := false
|
||||
vipSet := make(map[string]interface{})
|
||||
for x := range updatedInstances {
|
||||
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
|
||||
vipSet[vip] = nil
|
||||
}
|
||||
}
|
||||
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
|
||||
if _, found := vipSet[vip]; found {
|
||||
shared = true
|
||||
}
|
||||
}
|
||||
if !shared {
|
||||
for x := range serviceInstance.Clusters {
|
||||
serviceInstance.Clusters[x].Stop()
|
||||
}
|
||||
if serviceInstance.IsDHCP {
|
||||
serviceInstance.DHCPClient.Stop()
|
||||
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
|
||||
}
|
||||
|
||||
err = netlink.LinkDel(macvlan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
|
||||
}
|
||||
}
|
||||
for i := range serviceInstance.VIPConfigs {
|
||||
if serviceInstance.VIPConfigs[i].EnableBGP {
|
||||
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
|
||||
}
|
||||
}
|
||||
|
||||
// We will need to tear down the egress
|
||||
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
|
||||
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
|
||||
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
|
||||
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
|
||||
if err != nil {
|
||||
log.Error("[service] egress teardown", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the service array
|
||||
p.ServiceInstances = updatedInstances
|
||||
|
||||
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set up UPNP forwards for a service
|
||||
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
|
||||
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
|
||||
if !isUPNPEnabled(s.ServiceSnapshot) {
|
||||
// Skip services missing the annotation
|
||||
return
|
||||
}
|
||||
if !p.config.EnableUPNP {
|
||||
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
|
||||
return
|
||||
}
|
||||
// If upnp is enabled then update the gateway/router with the address
|
||||
// TODO - check if this implementation for dualstack is correct
|
||||
|
||||
gateways := upnp.GetGatewayClients(ctx)
|
||||
|
||||
// Reset Gateway IPs to remove stale addresses
|
||||
s.UPNPGatewayIPs = make([]string, 0)
|
||||
|
||||
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
|
||||
for _, port := range s.ServiceSnapshot.Spec.Ports {
|
||||
for _, gw := range gateways {
|
||||
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
|
||||
|
||||
forwardSucessful := false
|
||||
if gw.WANIPv6FirewallControlClient != nil {
|
||||
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
|
||||
if pinholeErr == nil {
|
||||
forwardSucessful = true
|
||||
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
|
||||
} else {
|
||||
//TODO: Cleanup
|
||||
log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
|
||||
}
|
||||
}
|
||||
// Fallback to PortForward
|
||||
if !forwardSucessful {
|
||||
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint TODO
|
||||
if portMappingErr == nil {
|
||||
log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
|
||||
forwardSucessful = true
|
||||
} else {
|
||||
//TODO: Cleanup
|
||||
log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if forwardSucessful {
|
||||
ip, err := gw.ConnectionClient.GetExternalIPAddress()
|
||||
if err == nil {
|
||||
s.UPNPGatewayIPs = append(s.UPNPGatewayIPs, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove duplicate IPs
|
||||
slices.Sort(s.UPNPGatewayIPs)
|
||||
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
|
||||
}
|
||||
|
||||
func (p *Processor) updateStatus(i *instance.Instance) error {
|
||||
// let's retry status update every 10ms for 30s
|
||||
retryConfig := wait.Backoff{
|
||||
Steps: 3000,
|
||||
Duration: 10 * time.Millisecond,
|
||||
Factor: 0,
|
||||
Jitter: 0.1,
|
||||
}
|
||||
// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
|
||||
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
|
||||
// Retrieve the latest version of Deployment before attempting update
|
||||
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
|
||||
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentServiceCopy := currentService.DeepCopy()
|
||||
if currentServiceCopy.Annotations == nil {
|
||||
currentServiceCopy.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
|
||||
if p.config.EnableARP {
|
||||
// Add the current host
|
||||
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
|
||||
}
|
||||
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
|
||||
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
|
||||
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
|
||||
}
|
||||
|
||||
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
|
||||
log.Error("(Synthetic error ) updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
|
||||
return fmt.Errorf("(Synthetic) simulating api server errors")
|
||||
}
|
||||
|
||||
if !cmp.Equal(currentService, currentServiceCopy) {
|
||||
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ports := make([]v1.PortStatus, 0, len(i.ServiceSnapshot.Spec.Ports))
|
||||
for _, port := range i.ServiceSnapshot.Spec.Ports {
|
||||
ports = append(ports, v1.PortStatus{
|
||||
Port: port.Port,
|
||||
Protocol: port.Protocol,
|
||||
})
|
||||
}
|
||||
|
||||
ingresses := []v1.LoadBalancerIngress{}
|
||||
|
||||
for _, c := range i.VIPConfigs {
|
||||
if !vip.IsIP(c.VIP) {
|
||||
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: ip,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
} else {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: c.VIP,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
if isUPNPEnabled(currentService) {
|
||||
for _, ip := range i.UPNPGatewayIPs {
|
||||
i := v1.LoadBalancerIngress{
|
||||
IP: ip,
|
||||
Ports: ports,
|
||||
}
|
||||
ingresses = append(ingresses, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
|
||||
currentService.Status.LoadBalancer.Ingress = ingresses
|
||||
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func isUPNPEnabled(s *v1.Service) bool {
|
||||
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
|
||||
}
|
||||
|
||||
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
|
||||
func (p *Processor) RefreshUPNPForwards() {
|
||||
log.Info("Starting UPNP Port Refresher")
|
||||
for {
|
||||
time.Sleep(300 * time.Second)
|
||||
|
||||
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
|
||||
for i := range p.ServiceInstances {
|
||||
p.upnpMap(context.TODO(), p.ServiceInstances[i])
|
||||
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
|
||||
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
|
||||
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
|
||||
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
|
||||
defer cancel()
|
||||
|
||||
var leaderElectionActive bool
|
||||
|
||||
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
exitFunction := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-svcCtx.Ctx.Done():
|
||||
log.Debug("context cancelled", "provider", provider.GetLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
case <-p.shutdownChan:
|
||||
log.Debug("shutdown called", "provider", provider.GetLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
case <-exitFunction:
|
||||
log.Debug("function ending", "provider", provider.GetLabel())
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
// Cancel the context, which will in turn cancel the leadership
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
ch := rw.ResultChan()
|
||||
|
||||
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
|
||||
|
||||
var lastKnownGoodEndpoint string
|
||||
for event := range ch {
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
|
||||
case watch.Added, watch.Modified:
|
||||
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
|
||||
if restart {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
case watch.Deleted:
|
||||
if err := epProcessor.Delete(service, id); err != nil {
|
||||
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
|
||||
}
|
||||
|
||||
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
|
||||
close(exitFunction)
|
||||
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
|
||||
|
||||
return nil
|
||||
case watch.Error:
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, _ := errObject.(*apierrors.StatusError)
|
||||
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
|
||||
}
|
||||
}
|
||||
close(exitFunction)
|
||||
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
|
||||
return nil //nolint:govet
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
)
|
||||
|
||||
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
|
||||
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
|
||||
// first start port mirroring if enabled
|
||||
if err := p.startTrafficMirroringIfEnabled(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
// clean up traffic mirror related config
|
||||
err := p.stopTrafficMirroringIfEnabled()
|
||||
if err != nil {
|
||||
log.Error("Stopping traffic mirroring", "err", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if p.config.ServiceNamespace == "" {
|
||||
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
|
||||
p.config.ServiceNamespace = v1.NamespaceAll
|
||||
log.Info("(svcs) starting services watcher for all namespaces")
|
||||
} else {
|
||||
log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
|
||||
}
|
||||
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating services watcher: %s", err.Error())
|
||||
}
|
||||
exitFunction := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-p.shutdownChan:
|
||||
log.Debug("(svcs) shutdown called")
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
return
|
||||
case <-exitFunction:
|
||||
log.Debug("(svcs) function ending")
|
||||
// Stop the retry watcher
|
||||
rw.Stop()
|
||||
return
|
||||
}
|
||||
}()
|
||||
ch := rw.ResultChan()
|
||||
|
||||
// Used for tracking an active endpoint / pod
|
||||
for event := range ch {
|
||||
p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
|
||||
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
restart, err := p.AddOrModify(ctx, event, serviceFunc)
|
||||
if restart {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("add/modify service error: %w", err)
|
||||
}
|
||||
case watch.Deleted:
|
||||
restart, err := p.Delete(event)
|
||||
if restart {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete service error: %w", err)
|
||||
}
|
||||
case watch.Bookmark:
|
||||
// Un-used
|
||||
case watch.Error:
|
||||
log.Error("Error attempting to watch Kubernetes services")
|
||||
|
||||
// This round trip allows us to handle unstructured status
|
||||
errObject := apierrors.FromObject(event.Object)
|
||||
statusErr, ok := errObject.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
|
||||
}
|
||||
|
||||
status := statusErr.ErrStatus
|
||||
log.Error("services", "err", status)
|
||||
default:
|
||||
}
|
||||
}
|
||||
close(exitFunction)
|
||||
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
|
||||
return nil
|
||||
}
|
||||
|
||||
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
|
||||
if svc == nil {
|
||||
log.Info("(svcs) service is nil, ignoring")
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass != nil {
|
||||
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
|
||||
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
|
||||
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
|
||||
return true
|
||||
}
|
||||
} else if config.LoadBalancerClassOnly {
|
||||
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
|
||||
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
|
||||
if svc == nil {
|
||||
log.Info("(svcs) service is nil, ignoring")
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
|
||||
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
|
||||
return true
|
||||
}
|
||||
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
|
||||
return false
|
||||
}
|
||||
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
|
||||
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Processor) serviceInterface() string {
|
||||
svcIf := p.config.Interface
|
||||
if p.config.ServicesInterface != "" {
|
||||
svcIf = p.config.ServicesInterface
|
||||
}
|
||||
return svcIf
|
||||
}
|
||||
|
||||
func (p *Processor) startTrafficMirroringIfEnabled() error {
|
||||
if p.config.MirrorDestInterface != "" {
|
||||
svcIf := p.serviceInterface()
|
||||
log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
|
||||
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
log.Debug("skip starting traffic mirroring since it's not enabled.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Processor) stopTrafficMirroringIfEnabled() error {
|
||||
if p.config.MirrorDestInterface != "" {
|
||||
svcIf := p.serviceInterface()
|
||||
log.Info("clean up qdisc config", "interface", svcIf)
|
||||
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
log.Debug("skip stopping traffic mirroring since it's not enabled.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func WriteProcSys(path, value string) error {
|
||||
@@ -28,47 +27,3 @@ func WriteProcSys(path, value string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func CheckProcSys(path string) (bool, error) {
|
||||
f, err := os.OpenFile(path, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if cErr := f.Close(); cErr != nil && err == nil {
|
||||
err = fmt.Errorf("failed to close file: %w", cErr)
|
||||
}
|
||||
}()
|
||||
|
||||
buffer := make([]byte, 1)
|
||||
|
||||
if n, err := f.Read(buffer); err != nil || n < len(buffer) {
|
||||
return false, fmt.Errorf("failed to read file: %w", err)
|
||||
}
|
||||
|
||||
var isEnabled bool
|
||||
isEnabled, err = strconv.ParseBool(string(buffer))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse value: %w", err)
|
||||
}
|
||||
|
||||
return isEnabled, nil
|
||||
}
|
||||
|
||||
func EnableProcSys(path string) (bool, error) {
|
||||
isEnabled, err := CheckProcSys(path)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check '%s' status: %w", path, err)
|
||||
}
|
||||
if !isEnabled {
|
||||
if err := WriteProcSys(path, "1"); err != nil {
|
||||
if os.IsPermission(err) {
|
||||
return false, fmt.Errorf("no permission to write to the file '%s' - please ensure that kube-vip is running with proper capabilities/privileged mode to write to sysfs: %w", path, err)
|
||||
}
|
||||
return false, fmt.Errorf("failed to enable '%s': %w", path, err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -4,8 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
@@ -28,15 +27,16 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
|
||||
return fmt.Errorf("failed to find nic %s: %v", toNICName, err)
|
||||
}
|
||||
toNICID := toNIC.Attrs().Index
|
||||
log.Debug(fmt.Sprintf("interface %s has index %d", fromNICName, fromNICID))
|
||||
log.Debug(fmt.Sprintf("interface %s has index %d", toNICName, toNICID))
|
||||
|
||||
log.Debug(fmt.Sprintf("clean up interface %s first in case it has stale qdsic", fromNICName))
|
||||
log.Debugf("interface %s has index %d", fromNICName, fromNICID)
|
||||
log.Debugf("interface %s has index %d", toNICName, toNICID)
|
||||
|
||||
log.Debugf("clean up interface %s first in case it has stale qdsic", fromNICName)
|
||||
if err := CleanupQDSICFromNIC(fromNICName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug(fmt.Sprintf("step 1: tc qdisc add dev %s ingress", fromNICName))
|
||||
log.Debugf("step 1: tc qdisc add dev %s ingress", fromNICName)
|
||||
qdisc1 := &netlink.Ingress{
|
||||
QdiscAttrs: netlink.QdiscAttrs{
|
||||
LinkIndex: fromNICID,
|
||||
@@ -48,7 +48,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
|
||||
return fmt.Errorf("failed to add qdisc for interface %s: %v", fromNICName, err)
|
||||
}
|
||||
|
||||
log.Debug(fmt.Sprintf("step 2: tc filter add dev %s parent ffff: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, toNICName))
|
||||
log.Debugf("step 2: tc filter add dev %s parent ffff: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, toNICName)
|
||||
// add a filter to mirror traffic from index1 to index2
|
||||
filter1 := &netlink.U32{
|
||||
FilterAttrs: netlink.FilterAttrs{
|
||||
@@ -71,7 +71,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
|
||||
return fmt.Errorf("failed to add filter for interface %s: %v", fromNICName, err)
|
||||
}
|
||||
|
||||
log.Debug(fmt.Sprintf("step 3: tc qdisc add dev %s ingress", fromNICName))
|
||||
log.Debugf("step 3: tc qdisc add dev %s ingress", fromNICName)
|
||||
qdiscTemp := netlink.NewPrio(netlink.QdiscAttrs{
|
||||
LinkIndex: fromNICID,
|
||||
Parent: netlink.HANDLE_ROOT,
|
||||
@@ -90,12 +90,12 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug(fmt.Sprintf("step 4: tc filter add dev %s parent %d: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, qdiscID, toNICName))
|
||||
log.Debugf("step 4: tc filter add dev %s parent %d: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, qdiscID, toNICName)
|
||||
|
||||
filter2 := &netlink.U32{
|
||||
FilterAttrs: netlink.FilterAttrs{
|
||||
LinkIndex: fromNICID,
|
||||
Parent: netlink.MakeHandle(uint16(qdiscID), 0), //nolint
|
||||
Parent: netlink.MakeHandle(uint16(qdiscID), 0),
|
||||
Protocol: unix.ETH_P_ALL,
|
||||
},
|
||||
Actions: []netlink.Action{
|
||||
@@ -113,7 +113,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
|
||||
return fmt.Errorf("failed to add filter for interface %s: %v", fromNICName, err)
|
||||
}
|
||||
|
||||
log.Info("traffic mirroring has been set up", "src", toNICName, "dst", fromNICName)
|
||||
log.Infof("traffic mirroring has been set up from interface %s to interface %s", toNICName, fromNICName)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ func CleanupQDSICFromNIC(nicName string) error {
|
||||
}
|
||||
nicID := toNIC.Attrs().Index
|
||||
|
||||
log.Debug(fmt.Sprintf("interface %s has index %d", nicName, nicID))
|
||||
log.Debugf("interface %s has index %d", nicName, nicID)
|
||||
|
||||
log.Debug("step 1: delete ingress qdisc")
|
||||
if err := tryCleanupQdiscByType(nicID, nicName, "ingress"); err != nil {
|
||||
@@ -138,7 +138,7 @@ func CleanupQDSICFromNIC(nicName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("finished cleaning up all qdisc config", "interface", nicName)
|
||||
log.Infof("finished cleaning up all qdisc config on interface %s", nicName)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -154,7 +154,7 @@ func getQdiscFromInterfaceByType(nicID int, nicName string, qType string) (uint3
|
||||
return q.Attrs().Handle, nil
|
||||
}
|
||||
}
|
||||
log.Error("no qdisc", "interface", nicName, "is of type", qType)
|
||||
log.Errorf("no qdisc under interface %s is %s type", nicName, qType)
|
||||
return 0, errQdiscNotFound
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@ func tryCleanupQdiscByType(nicID int, nicName, qType string) error {
|
||||
_, err := getQdiscFromInterfaceByType(nicID, nicName, qType)
|
||||
if err != nil {
|
||||
if err == errQdiscNotFound {
|
||||
log.Debug(fmt.Sprintf("%s type qdisc doesn't exist on interface %s, skip deleting", qType, nicName))
|
||||
log.Debugf("%s type qdisc doesn't exist on interface %s, skip deleting", qType, nicName)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
|
||||
@@ -4,10 +4,9 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/huin/goupnp"
|
||||
"github.com/huin/goupnp/dcps/internetgateway2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -48,7 +47,7 @@ func GetGatewayClients(ctx context.Context) []Gateway {
|
||||
if wanipv6clients, err := internetgateway2.NewWANIPv6FirewallControl1ClientsByURLCtx(ctx, gatewayURL); err == nil {
|
||||
gatewayClients[i].WANIPv6FirewallControlClient = wanipv6clients[0]
|
||||
} else {
|
||||
log.Warn("[UPNP] Unable to find WANIPv6FirewallControl1Clients", "Gateway", gatewayURL, "err", err)
|
||||
log.Warnf("[UPNP] Unable to find WANIPv6FirewallControl1Clients for Gateway %s [%s]", gatewayURL, err.Error())
|
||||
}
|
||||
}
|
||||
return gatewayClients
|
||||
@@ -85,7 +84,7 @@ func GetConnectionClients(ctx context.Context) []ConnectionClient {
|
||||
var routers []ConnectionClient
|
||||
|
||||
if err := tasks.Wait(); err != nil {
|
||||
log.Error("[UPNP] Could not finish querying UPNP connection clients", "err", err.Error())
|
||||
log.Errorf("[UPNP] Could not finish querying UPNP connection clients [%s]", err.Error())
|
||||
return routers
|
||||
}
|
||||
|
||||
@@ -93,7 +92,7 @@ func GetConnectionClients(ctx context.Context) []ConnectionClient {
|
||||
errors = append(errors, ppp1Error...)
|
||||
|
||||
for _, e := range errors {
|
||||
log.Warn("[UPNP] UPNP Gateway responded with an error while querying WAN Connection Client", "err", e)
|
||||
log.Warnf("[UPNP] UPNP Gateway responded with an error while querying WAN Connection Client [%s]", e.Error())
|
||||
}
|
||||
|
||||
for _, c := range ip1Clients {
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
import "os"
|
||||
|
||||
func FileExists(filename string) bool {
|
||||
info, err := os.Stat(filename)
|
||||
@@ -13,15 +9,3 @@ func FileExists(filename string) bool {
|
||||
}
|
||||
return !info.IsDir()
|
||||
}
|
||||
|
||||
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
|
||||
func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
|
||||
|
||||
addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
|
||||
// Check if the input is valid
|
||||
_, _, err := net.ParseCIDR(addr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
@@ -2,24 +2,20 @@ package vip
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"slices"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
"github.com/vishvananda/netlink/nl"
|
||||
"golang.org/x/sys/unix"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -31,15 +27,13 @@ const (
|
||||
|
||||
// Network is an interface that enable managing operations for a given IP
|
||||
type Network interface {
|
||||
AddIP(precheck bool) (bool, error)
|
||||
AddIP(precheck bool) error
|
||||
AddRoute(precheck bool) error
|
||||
DeleteIP() (bool, error)
|
||||
DeleteIP() error
|
||||
DeleteRoute() error
|
||||
UpdateRoutes() (bool, error)
|
||||
IsSet() (bool, error)
|
||||
IP() string
|
||||
CIDR() string
|
||||
IPisLinkLocal() bool
|
||||
PrepareRoute() *netlink.Route
|
||||
SetIP(ip string) error
|
||||
SetServicePorts(service *v1.Service)
|
||||
@@ -50,9 +44,6 @@ type Network interface {
|
||||
DDNSHostName() string
|
||||
DNSName() string
|
||||
SetMask(mask string) error
|
||||
SetHasEndpoints(value bool)
|
||||
HasEndpoints() bool
|
||||
ARPName() string
|
||||
}
|
||||
|
||||
// network - This allows network configuration
|
||||
@@ -60,10 +51,9 @@ type network struct {
|
||||
mu sync.Mutex
|
||||
|
||||
address *netlink.Addr
|
||||
link *networkinterface.Link
|
||||
link netlink.Link
|
||||
ports []v1.ServicePort
|
||||
serviceName string
|
||||
enableSecurity bool
|
||||
ignoreSecurity bool
|
||||
|
||||
dnsName string
|
||||
@@ -75,16 +65,18 @@ type network struct {
|
||||
routeTable int
|
||||
routingTableType int
|
||||
routingProtocol int
|
||||
}
|
||||
|
||||
ipvsEnabled bool
|
||||
|
||||
hasEndpoints bool
|
||||
func netlinkParse(addr string) (*netlink.Addr, error) {
|
||||
mask, err := GetFullMask(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return netlink.ParseAddr(addr + mask)
|
||||
}
|
||||
|
||||
// NewConfig will attempt to provide an interface to the kernel network configuration
|
||||
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
|
||||
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
|
||||
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
|
||||
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int, routingProtocol int, dnsMode, forwardMethod, iptablesBackend string) ([]Network, error) {
|
||||
networks := []Network{}
|
||||
|
||||
link, err := netlink.LinkByName(iface)
|
||||
@@ -92,39 +84,28 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
return networks, errors.Wrapf(err, "could not get link for interface '%s'", iface)
|
||||
}
|
||||
|
||||
networkLink := intfMgr.Get(link)
|
||||
|
||||
if IsIP(address) {
|
||||
result := &network{
|
||||
link: networkLink,
|
||||
link: link,
|
||||
routeTable: tableID,
|
||||
routingTableType: tableType,
|
||||
routingProtocol: routingProtocol,
|
||||
forwardMethod: forwardMethod,
|
||||
iptablesBackend: iptablesBackend,
|
||||
ipvsEnabled: ipvsEnabled,
|
||||
}
|
||||
|
||||
subnet, err = SelectSubnet(address, subnet)
|
||||
if err != nil {
|
||||
return networks, fmt.Errorf("unable to select subnet for IP %q from %q: %w", address, subnet, err)
|
||||
}
|
||||
|
||||
// Check if the subnet needs overriding
|
||||
cidr, err := utils.FormatIPWithSubnetMask(address, subnet)
|
||||
if err != nil {
|
||||
return networks, errors.Wrapf(err, "could not format address '%s' with subnetMask '%s'", address, subnet)
|
||||
if subnet != "" {
|
||||
result.address, err = netlink.ParseAddr(address + subnet)
|
||||
if err != nil {
|
||||
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
|
||||
}
|
||||
} else {
|
||||
result.address, err = netlinkParse(address)
|
||||
if err != nil {
|
||||
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
|
||||
}
|
||||
}
|
||||
result.address, err = netlink.ParseAddr(cidr)
|
||||
if err != nil {
|
||||
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
|
||||
}
|
||||
|
||||
// set address as deprecated so it isn't used as source address according to RFC 3484
|
||||
result.address.PreferedLft = 0
|
||||
|
||||
// Also set ValidLft so the netlink library actually sets them
|
||||
result.address.ValidLft = math.MaxInt
|
||||
|
||||
if iface == "lo" && !loGlobalScope {
|
||||
// set host scope on loopback, otherwise global scope will be used by default
|
||||
@@ -140,7 +121,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
// when leader starts, should do get IP from DHCP for the domain
|
||||
if isDDNS {
|
||||
result := &network{
|
||||
link: networkLink,
|
||||
link: link,
|
||||
routeTable: tableID,
|
||||
routingTableType: tableType,
|
||||
routingProtocol: routingProtocol,
|
||||
@@ -148,8 +129,6 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
iptablesBackend: iptablesBackend,
|
||||
isDDNS: isDDNS,
|
||||
dnsName: address,
|
||||
ipvsEnabled: ipvsEnabled,
|
||||
enableSecurity: enableSecurity,
|
||||
}
|
||||
|
||||
networks = append(networks, result)
|
||||
@@ -160,7 +139,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
|
||||
for _, ip := range ips {
|
||||
result := &network{
|
||||
link: networkLink,
|
||||
link: link,
|
||||
routeTable: tableID,
|
||||
routingTableType: tableType,
|
||||
routingProtocol: routingProtocol,
|
||||
@@ -168,21 +147,15 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
|
||||
iptablesBackend: iptablesBackend,
|
||||
isDDNS: isDDNS,
|
||||
dnsName: address,
|
||||
ipvsEnabled: ipvsEnabled,
|
||||
enableSecurity: enableSecurity,
|
||||
}
|
||||
|
||||
// we're able to resolve store this as the initial IP
|
||||
|
||||
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, subnet)); err != nil {
|
||||
if result.address, err = netlinkParse(ip); err != nil {
|
||||
return networks, err
|
||||
}
|
||||
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
|
||||
result.address.ValidLft = defaultValidLft
|
||||
|
||||
// set address as deprecated so it isn't used as source address according to RFC 3484
|
||||
result.address.PreferedLft = 0
|
||||
|
||||
networks = append(networks, result)
|
||||
}
|
||||
|
||||
@@ -225,7 +198,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
|
||||
route := &netlink.Route{
|
||||
Scope: routeScope,
|
||||
Dst: configurator.address.IPNet,
|
||||
LinkIndex: configurator.link.Intf.Attrs().Index,
|
||||
LinkIndex: configurator.link.Attrs().Index,
|
||||
Table: configurator.routeTable,
|
||||
Type: configurator.routingTableType,
|
||||
Protocol: netlink.RouteProtocol(configurator.routingProtocol),
|
||||
@@ -235,8 +208,6 @@ func (configurator *network) PrepareRoute() *netlink.Route {
|
||||
|
||||
// AddRoute - Add an IP address to a route table
|
||||
func (configurator *network) AddRoute(precheck bool) error {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
route := configurator.PrepareRoute()
|
||||
|
||||
exists := false
|
||||
@@ -258,7 +229,7 @@ func (configurator *network) AddRoute(precheck bool) error {
|
||||
}
|
||||
|
||||
func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
|
||||
routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
|
||||
routes, err := netlink.RouteList(configurator.link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to list routes")
|
||||
}
|
||||
@@ -308,41 +279,36 @@ func (configurator *network) UpdateRoutes() (bool, error) {
|
||||
}
|
||||
|
||||
// AddIP - Add an IP address to the interface
|
||||
func (configurator *network) AddIP(precheck bool) (bool, error) {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
func (configurator *network) AddIP(precheck bool) error {
|
||||
exists := false
|
||||
var err error
|
||||
if precheck {
|
||||
if exists, err = configurator.IsSet(); err != nil {
|
||||
return false, errors.Wrap(err, "could not check if address exists")
|
||||
if exists, err = configurator.addressExists(); err != nil {
|
||||
return errors.Wrap(err, "could not check if address exists")
|
||||
}
|
||||
}
|
||||
|
||||
if exists {
|
||||
return false, nil
|
||||
if !exists {
|
||||
if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
|
||||
return errors.Wrap(err, "could not add ip")
|
||||
}
|
||||
|
||||
if err := configurator.configureIPTables(); err != nil {
|
||||
return errors.Wrap(err, "could not configure IPTables")
|
||||
}
|
||||
}
|
||||
|
||||
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
|
||||
return false, errors.Wrap(err, "could not add ip")
|
||||
}
|
||||
|
||||
if err := configurator.configureIPTables(); err != nil {
|
||||
return true, errors.Wrap(err, "could not configure IPTables")
|
||||
}
|
||||
|
||||
return true, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (configurator *network) configureIPTables() error {
|
||||
if configurator.enableSecurity && !configurator.ignoreSecurity {
|
||||
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
|
||||
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
|
||||
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
|
||||
}
|
||||
}
|
||||
|
||||
// It seems that masquerading is only reuired with IPv4 for IPVS to work.
|
||||
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
|
||||
if configurator.forwardMethod == "masquerade" {
|
||||
if err := configurator.addIptablesRulesForMasquerade(); err != nil {
|
||||
return errors.Wrap(err, "could not add iptables rules for masquerade")
|
||||
}
|
||||
@@ -351,6 +317,21 @@ func (configurator *network) configureIPTables() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (configurator *network) addressExists() (bool, error) {
|
||||
addrs, err := netlink.AddrList(configurator.link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "could not list addresses")
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
if addr.Equal(*configurator.address) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
|
||||
ipt, err := iptables.New()
|
||||
if err != nil {
|
||||
@@ -362,7 +343,7 @@ func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
|
||||
if err := insertCommonIPTablesRules(ipt, vip, comment); err != nil {
|
||||
return fmt.Errorf("could not add common iptables rules: %w", err)
|
||||
}
|
||||
log.Debug("add iptables rules", "vip", vip, "ports", configurator.ports)
|
||||
log.Debugf("add iptables rules, vip: %s, ports: %+v", vip, configurator.ports)
|
||||
if err := configurator.insertIPTablesRulesForServicePorts(ipt, vip, comment); err != nil {
|
||||
return fmt.Errorf("could not add iptables rules for service ports: %v", err)
|
||||
}
|
||||
@@ -467,7 +448,7 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
|
||||
return fmt.Errorf("could not delete common iptables rules: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("remove iptables rules", "vip", vip, "ports", configurator.ports)
|
||||
log.Debugf("remove iptables rules, vip: %s, ports: %+v", vip, configurator.ports)
|
||||
for _, port := range configurator.ports {
|
||||
// iptables -D INPUT -d <VIP> -p <protocol> --dport <port> -j ACCEPT
|
||||
if err := ipt.DeleteIfExists(iptables.TableFilter, iptables.ChainInput, "-d", vip, "-p", string(port.Protocol),
|
||||
@@ -480,37 +461,34 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
|
||||
}
|
||||
|
||||
// DeleteIP - Remove an IP address from the interface
|
||||
func (configurator *network) DeleteIP() (bool, error) {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
|
||||
func (configurator *network) DeleteIP() error {
|
||||
result, err := configurator.IsSet()
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "ip check in DeleteIP failed")
|
||||
return errors.Wrap(err, "ip check in DeleteIP failed")
|
||||
}
|
||||
|
||||
// Nothing to delete
|
||||
if !result {
|
||||
return false, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
|
||||
return false, errors.Wrap(err, "could not delete ip")
|
||||
if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
|
||||
return errors.Wrap(err, "could not delete ip")
|
||||
}
|
||||
|
||||
if configurator.enableSecurity && !configurator.ignoreSecurity {
|
||||
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
|
||||
if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
|
||||
return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
|
||||
return errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
|
||||
}
|
||||
}
|
||||
|
||||
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
|
||||
if configurator.forwardMethod == "masquerade" {
|
||||
if err := configurator.removeIptablesRulesForMasquerade(); err != nil {
|
||||
return true, errors.Wrap(err, "could not remove iptables masquerade rules ")
|
||||
return errors.Wrap(err, "could not remove iptables masquerade rules ")
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (configurator *network) addIptablesRulesForMasquerade() error {
|
||||
@@ -577,15 +555,12 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
|
||||
|
||||
// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
|
||||
func (configurator *network) IsDADFAIL() bool {
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
|
||||
if configurator.address == nil || !IsIPv6(configurator.address.IP.String()) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Get all the address
|
||||
addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
|
||||
addresses, err := netlink.AddrList(configurator.link, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
@@ -604,7 +579,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
|
||||
return address.Flags&unix.IFA_F_DADFAILED != 0
|
||||
}
|
||||
|
||||
// isSet - Check to see if VIP is set
|
||||
// IsSet - Check to see if VIP is set
|
||||
func (configurator *network) IsSet() (result bool, err error) {
|
||||
var addresses []netlink.Addr
|
||||
|
||||
@@ -612,11 +587,11 @@ func (configurator *network) IsSet() (result bool, err error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
|
||||
addresses, err = netlink.AddrList(configurator.link, 0)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "could not list addresses")
|
||||
|
||||
return false, err
|
||||
return
|
||||
}
|
||||
|
||||
for _, address := range addresses {
|
||||
@@ -633,30 +608,13 @@ func (configurator *network) SetIP(ip string) error {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
|
||||
configurator.link.Lock.Lock()
|
||||
defer configurator.link.Lock.Unlock()
|
||||
|
||||
if strings.Contains("/", ip) {
|
||||
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
|
||||
}
|
||||
ones, _ := configurator.address.Mask.Size()
|
||||
cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not format address '%s' with subnetMask '%s'", ip, strconv.Itoa(ones))
|
||||
}
|
||||
addr, err := netlink.ParseAddr(cidr)
|
||||
addr, err := netlinkParse(ip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if configurator.address != nil && configurator.IsDNS() {
|
||||
addr.ValidLft = defaultValidLft
|
||||
} else {
|
||||
addr.ValidLft = math.MaxInt
|
||||
}
|
||||
|
||||
// set address as deprecated so it isn't used as source address according to RFC 3484
|
||||
addr.PreferedLft = 0
|
||||
|
||||
configurator.address = addr
|
||||
return nil
|
||||
}
|
||||
@@ -680,21 +638,6 @@ func (configurator *network) IP() string {
|
||||
return configurator.address.IP.String()
|
||||
}
|
||||
|
||||
func (configurator *network) CIDR() string {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
|
||||
return configurator.address.IPNet.String()
|
||||
}
|
||||
|
||||
// IP - return the IP Address
|
||||
func (configurator *network) IPisLinkLocal() bool {
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
|
||||
return configurator.address.IP.IsLinkLocalUnicast()
|
||||
}
|
||||
|
||||
// DNSName return the configured dnsName when use DNS
|
||||
func (configurator *network) DNSName() string {
|
||||
return configurator.dnsName
|
||||
@@ -720,23 +663,19 @@ func (configurator *network) DDNSHostName() string {
|
||||
|
||||
// Interface - return the Interface name
|
||||
func (configurator *network) Interface() string {
|
||||
return configurator.link.Intf.Attrs().Name
|
||||
return configurator.link.Attrs().Name
|
||||
}
|
||||
|
||||
func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
|
||||
func GarbageCollect(adapter, address string) (found bool, err error) {
|
||||
|
||||
// Get adapter
|
||||
link, err := netlink.LinkByName(adapter)
|
||||
if err != nil {
|
||||
return true, errors.Wrapf(err, "could not get link for interface '%s'", adapter)
|
||||
}
|
||||
|
||||
l := intfMgr.Get(link)
|
||||
|
||||
l.Lock.Lock()
|
||||
defer l.Lock.Unlock()
|
||||
|
||||
// Get addresses on adapter
|
||||
addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
|
||||
addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -748,7 +687,7 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
|
||||
found = true
|
||||
// linting issue
|
||||
existing := existing
|
||||
if err = netlink.AddrDel(l.Intf, &existing); err != nil {
|
||||
if err = netlink.AddrDel(link, &existing); err != nil {
|
||||
return true, errors.Wrap(err, "could not delete ip")
|
||||
}
|
||||
}
|
||||
@@ -757,83 +696,14 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
|
||||
}
|
||||
|
||||
func (configurator *network) SetMask(mask string) error {
|
||||
selectedMask, err := SelectSubnet(configurator.IP(), mask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to select mask %q: %w", mask, err)
|
||||
}
|
||||
|
||||
m, err := strconv.Atoi(selectedMask)
|
||||
m, err := strconv.Atoi(mask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
size := 32
|
||||
family := "IPv4"
|
||||
|
||||
if IsIPv6(configurator.IP()) {
|
||||
if IsIPv6(configurator.address.IP.String()) {
|
||||
size = 128
|
||||
family = "IPv6"
|
||||
}
|
||||
|
||||
if m > size {
|
||||
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
|
||||
}
|
||||
|
||||
toSet := net.CIDRMask(m, size)
|
||||
if toSet == nil {
|
||||
return fmt.Errorf("failed to create mask /%d", m)
|
||||
}
|
||||
|
||||
configurator.mu.Lock()
|
||||
defer configurator.mu.Unlock()
|
||||
|
||||
configurator.address.Mask = toSet
|
||||
configurator.address.IPNet.Mask = net.CIDRMask(m, size)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (configurator *network) SetHasEndpoints(value bool) {
|
||||
log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
|
||||
configurator.hasEndpoints = value
|
||||
}
|
||||
|
||||
func (configurator *network) HasEndpoints() bool {
|
||||
log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
|
||||
return configurator.hasEndpoints
|
||||
}
|
||||
|
||||
func (configurator *network) ARPName() string {
|
||||
return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
|
||||
}
|
||||
|
||||
// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
|
||||
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
|
||||
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {
|
||||
// Split the SubnetMasks input into DualStack or SingleStack
|
||||
// If the input is "32,128", it will be split into ["32", "128"]
|
||||
subnetMasksParts := strings.Split(subnetMasks, ",")
|
||||
if len(subnetMasksParts) == 0 {
|
||||
return "", fmt.Errorf("no subnetMasks provided got: %q", subnetMasks)
|
||||
} else if len(subnetMasksParts) > 2 {
|
||||
return "", fmt.Errorf("invalid subnetMasks provided got: %q", subnetMasks)
|
||||
}
|
||||
if slices.Contains(subnetMasksParts, "auto") {
|
||||
return "", fmt.Errorf("auto subnet discovery only works for services: %q", subnetMasks)
|
||||
}
|
||||
|
||||
// Parse the raw IP address
|
||||
ip := net.ParseIP(rawIP)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("invalid IP address: %s", rawIP)
|
||||
}
|
||||
if ip.To4() != nil {
|
||||
return subnetMasksParts[0], nil
|
||||
}
|
||||
if ip.To16() != nil {
|
||||
subnetMask := subnetMasksParts[0]
|
||||
if len(subnetMasksParts) == 2 {
|
||||
subnetMask = subnetMasksParts[1]
|
||||
}
|
||||
return subnetMask, nil
|
||||
}
|
||||
return "", fmt.Errorf("unable to select subnet mask for: IP %q and masks %q", rawIP, subnetMasks)
|
||||
}
|
||||
|
||||
@@ -135,7 +135,9 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
|
||||
Halen: m.hardwareAddressLength,
|
||||
}
|
||||
target := ethernetBroadcast
|
||||
copy(ll.Addr[:], target)
|
||||
for i := 0; i < len(target); i++ { //nolint
|
||||
ll.Addr[i] = target[i]
|
||||
}
|
||||
|
||||
b, err := m.bytes()
|
||||
if err != nil {
|
||||
|
||||
@@ -5,9 +5,8 @@ import (
|
||||
"net"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DDNSManager will start a dhclient to retrieve and keep the lease for the IP
|
||||
@@ -56,7 +55,7 @@ func (ddns *ddnsManager) Start() (string, error) {
|
||||
client.Stop()
|
||||
return "", errors.New("failed to get IP from dhcp for ddns in 1 minutes")
|
||||
case ip = <-client.IPChannel():
|
||||
log.Info("got address from dhcp", "ip", ip)
|
||||
log.Info("got ip from dhcp: ", ip)
|
||||
}
|
||||
|
||||
// lease.FixedAddress.String() could return <nil>
|
||||
@@ -75,7 +74,7 @@ func (ddns *ddnsManager) Start() (string, error) {
|
||||
client.Stop()
|
||||
return
|
||||
case ip := <-client.IPChannel():
|
||||
log.Info("got address from dhcp", "ip", ip)
|
||||
log.Info("got ip from dhcp: ", ip)
|
||||
}
|
||||
}
|
||||
}(ddns.ctx)
|
||||
|
||||
@@ -8,11 +8,10 @@ import (
|
||||
"net"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/insomniacslk/dhcp/dhcpv4"
|
||||
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
|
||||
"github.com/jpillora/backoff"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const dhcpClientPort = "68"
|
||||
@@ -130,7 +129,7 @@ func (c *DHCPClient) Start() {
|
||||
// Set up two ticker to renew/rebind regularly
|
||||
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
|
||||
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
|
||||
log.Debug("dhcp timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
|
||||
log.Debugf("t1 %v t2 %v", t1Timeout, t2Timeout)
|
||||
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
|
||||
|
||||
for {
|
||||
@@ -144,25 +143,25 @@ func (c *DHCPClient) Start() {
|
||||
lease, err := c.renew()
|
||||
if err == nil {
|
||||
c.lease = lease
|
||||
log.Info("renew", "lease", lease)
|
||||
log.Infof("renew, lease: %+v", lease)
|
||||
t2.Reset(t2Timeout)
|
||||
} else {
|
||||
log.Error("renew failed", "err", err)
|
||||
log.Errorf("renew failed, error: %s", err.Error())
|
||||
}
|
||||
case <-t2.C:
|
||||
// rebind is just like a request, but forcing to provide a new IP address
|
||||
lease, err := c.request(true)
|
||||
if err == nil {
|
||||
c.lease = lease
|
||||
log.Info("rebind", "lease", lease)
|
||||
log.Infof("rebind, lease: %+v", lease)
|
||||
} else {
|
||||
if _, ok := err.(*nclient4.ErrNak); !ok {
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
log.Error("rebind failed", "err", err)
|
||||
log.Errorf("rebind failed, error: %s", err.Error())
|
||||
return
|
||||
}
|
||||
log.Warn("ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
|
||||
log.Warnf("ip %s may have changed: %s", c.lease.ACK.YourIPAddr, err.Error())
|
||||
c.initRebootFlag = false
|
||||
c.lease = c.requestWithBackoff()
|
||||
}
|
||||
@@ -172,9 +171,9 @@ func (c *DHCPClient) Start() {
|
||||
case <-c.stopChan:
|
||||
// release is a unicast request of the IP release.
|
||||
if err := c.release(); err != nil {
|
||||
log.Error("release lease failed", "lease", lease, "err", err)
|
||||
log.Errorf("release lease failed, error: %s, lease: %+v", err.Error(), c.lease)
|
||||
} else {
|
||||
log.Info("release", "lease", lease)
|
||||
log.Infof("release, lease: %+v", c.lease)
|
||||
}
|
||||
t1.Stop()
|
||||
t2.Stop()
|
||||
@@ -206,18 +205,18 @@ func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
|
||||
var err error
|
||||
|
||||
for {
|
||||
log.Debug("trying to get a new IP", "attempt", backoff.Attempt())
|
||||
log.Debugf("trying to get a new IP, attempt %f", backoff.Attempt())
|
||||
lease, err = c.request(false)
|
||||
if err != nil {
|
||||
dur := backoff.Duration()
|
||||
if backoff.Attempt() > maxBackoffAttempts-1 {
|
||||
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
|
||||
log.Error(errMsg.Error())
|
||||
log.Error(errMsg)
|
||||
c.errorChan <- errMsg
|
||||
c.Stop()
|
||||
return nil
|
||||
}
|
||||
log.Error("request failed", "err", err.Error(), "waiting", dur)
|
||||
log.Errorf("request failed, error: %s (waiting %v)", err.Error(), dur)
|
||||
time.Sleep(dur)
|
||||
continue
|
||||
}
|
||||
@@ -226,7 +225,7 @@ func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
|
||||
}
|
||||
|
||||
if c.ipChan != nil {
|
||||
log.Debug("using channel")
|
||||
log.Debugf("using channel")
|
||||
c.ipChan <- lease.ACK.YourIPAddr.String()
|
||||
}
|
||||
|
||||
@@ -252,13 +251,13 @@ func (c *DHCPClient) request(rebind bool) (*nclient4.Lease, error) {
|
||||
|
||||
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
|
||||
if c.initRebootFlag {
|
||||
log.Debug("init-reboot", "ip", c.requestedIP)
|
||||
log.Debugf("init-reboot ip %s", c.requestedIP)
|
||||
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.requestedIP)))
|
||||
}
|
||||
|
||||
// if this is a rebind, then the IP we should set is the one that already exists in lease
|
||||
if rebind {
|
||||
log.Debug("rebinding", "ip", c.lease.ACK.YourIPAddr)
|
||||
log.Debugf("rebinding ip %s", c.lease.ACK.YourIPAddr)
|
||||
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.lease.ACK.YourIPAddr)))
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// IPUpdater is the interface to plug dns updaters
|
||||
@@ -29,7 +29,7 @@ func (d *ipUpdater) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info("stop ipUpdater")
|
||||
log.Infof("stop ipUpdater")
|
||||
return
|
||||
default:
|
||||
mode := "ipv4"
|
||||
@@ -39,18 +39,18 @@ func (d *ipUpdater) Run(ctx context.Context) {
|
||||
|
||||
ip, err := LookupHost(d.vip.DNSName(), mode)
|
||||
if err != nil {
|
||||
log.Warn("cannot lookup", "name", d.vip.DNSName(), "err", err)
|
||||
log.Warnf("cannot lookup %s: %v", d.vip.DNSName(), err)
|
||||
// fallback to renewing the existing IP
|
||||
ip = []string{d.vip.IP()}
|
||||
}
|
||||
|
||||
log.Info("setting IP", "address", ip)
|
||||
log.Infof("setting %s as an IP", ip)
|
||||
if err := d.vip.SetIP(ip[0]); err != nil {
|
||||
log.Error("setting IP", "address", ip, "err", err)
|
||||
log.Errorf("setting %s as an IP: %v", ip, err)
|
||||
}
|
||||
|
||||
if _, err := d.vip.AddIP(false); err != nil {
|
||||
log.Error("error adding virtual IP", "err", err)
|
||||
if err := d.vip.AddIP(false); err != nil {
|
||||
log.Errorf("error adding virtual IP: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,9 +5,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
iptables "github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
ct "github.com/florianl/go-conntrack"
|
||||
)
|
||||
@@ -40,7 +39,7 @@ func CreateIptablesClient(nftables bool, namespace string, protocol iptables.Pro
|
||||
if protocol == iptables.ProtocolIPv6 {
|
||||
proto = "IPv6"
|
||||
}
|
||||
log.Info("[egress] Creating an iptables client", "nftables", nftables, "protocol", proto)
|
||||
log.Infof("[egress] Creating an iptables client, nftables mode [%t], protocol [%s]", nftables, proto)
|
||||
e := new(Egress)
|
||||
var err error
|
||||
|
||||
@@ -57,7 +56,7 @@ func CreateIptablesClient(nftables bool, namespace string, protocol iptables.Pro
|
||||
}
|
||||
|
||||
func (e *Egress) CheckMangleChain(name string) (bool, error) {
|
||||
log.Info("[egress] chain exists", "name", name)
|
||||
log.Infof("[egress] Checking for Chain [%s]", name)
|
||||
return e.ipTablesClient.ChainExists("mangle", name)
|
||||
}
|
||||
|
||||
@@ -69,12 +68,8 @@ func (e *Egress) DeleteManglePrerouting(name string) error {
|
||||
return e.ipTablesClient.Delete("mangle", "PREROUTING", "-j", name)
|
||||
}
|
||||
|
||||
func (e *Egress) DeleteMangleReturnForNetwork(name, network string) error {
|
||||
return e.ipTablesClient.Delete("mangle", name, "-d", network, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
|
||||
}
|
||||
|
||||
func (e *Egress) DeleteMangleMarking(podIP, name string) error {
|
||||
log.Info("[egress] Stopping marking packets on network", "podIP", podIP)
|
||||
log.Infof("[egress] Stopping marking packets on network [%s]", podIP)
|
||||
|
||||
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", podIP, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
|
||||
@@ -84,19 +79,8 @@ func (e *Egress) DeleteMangleMarking(podIP, name string) error {
|
||||
return e.ipTablesClient.Delete("mangle", name, "-s", podIP, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
}
|
||||
|
||||
func (e *Egress) DeleteMangleMarkingForNetwork(podIP, name, network string) error {
|
||||
log.Info("[egress] Stopping marking packets", "podIP", podIP, "network", network)
|
||||
|
||||
// exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", podIP, "-d", network, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
|
||||
// if !exists {
|
||||
// return fmt.Errorf("unable to find source Mangle rule for [%s]", podIP)
|
||||
// }
|
||||
return e.ipTablesClient.Delete("mangle", name, "-s", podIP, "-d", network, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
}
|
||||
|
||||
func (e *Egress) DeleteSourceNat(podIP, vip string) error {
|
||||
log.Info("[egress] Removing source nat", "podIP", podIP, "vip", vip)
|
||||
log.Infof("[egress] Removing source nat from [%s] => [%s]", podIP, vip)
|
||||
|
||||
exists, _ := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-m", "comment", "--comment", e.comment)
|
||||
|
||||
@@ -107,7 +91,7 @@ func (e *Egress) DeleteSourceNat(podIP, vip string) error {
|
||||
}
|
||||
|
||||
func (e *Egress) DeleteSourceNatForDestinationPort(podIP, vip, port, proto string) error {
|
||||
log.Info("[egress] Removing source nat", "podIP", podIP, "vip", vip, "destination port", port)
|
||||
log.Infof("[egress] Removing source nat from [%s] => [%s], with destination port [%s]", podIP, vip, port)
|
||||
|
||||
exists, _ := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-p", proto, "--dport", port, "-m", "comment", "--comment", e.comment)
|
||||
|
||||
@@ -119,13 +103,13 @@ func (e *Egress) DeleteSourceNatForDestinationPort(podIP, vip, port, proto strin
|
||||
|
||||
func (e *Egress) CreateMangleChain(name string) error {
|
||||
|
||||
log.Info("[egress] Creating Chain", "name", name)
|
||||
log.Infof("[egress] Creating Chain [%s]", name)
|
||||
// Creates a new chain in the mangle table
|
||||
return e.ipTablesClient.NewChain("mangle", name)
|
||||
|
||||
}
|
||||
func (e *Egress) AppendReturnRulesForDestinationSubnet(name, subnet string) error {
|
||||
log.Info("[egress] Adding jump for subnet to RETURN to previous chain/rules", "subnet", subnet)
|
||||
log.Infof("[egress] Adding jump for subnet [%s] to RETURN to previous chain/rules", subnet)
|
||||
exists, _ := e.ipTablesClient.Exists("mangle", name, "-d", subnet, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
|
||||
if !exists {
|
||||
return e.ipTablesClient.Append("mangle", name, "-d", subnet, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
|
||||
@@ -134,7 +118,7 @@ func (e *Egress) AppendReturnRulesForDestinationSubnet(name, subnet string) erro
|
||||
}
|
||||
|
||||
func (e *Egress) AppendReturnRulesForMarking(name, subnet string) error {
|
||||
log.Info("[egress] Marking packets on network", "subnet", subnet)
|
||||
log.Infof("[egress] Marking packets on network [%s]", subnet)
|
||||
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", subnet, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
if !exists {
|
||||
return e.ipTablesClient.Append("mangle", name, "-s", subnet, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
@@ -142,17 +126,8 @@ func (e *Egress) AppendReturnRulesForMarking(name, subnet string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Egress) AppendReturnRulesForMarkingForNetwork(name, subnet, destination string) error {
|
||||
log.Info("[egress] Marking packets on network", "subnet", subnet)
|
||||
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", subnet, "-d", destination, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
if !exists {
|
||||
return e.ipTablesClient.Append("mangle", name, "-s", subnet, "-d", destination, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Egress) InsertMangeTableIntoPrerouting(name string) error {
|
||||
log.Info("[egress] Adding jump from mangle prerouting", "destination", name)
|
||||
log.Infof("[egress] Adding jump from mangle prerouting to [%s]", name)
|
||||
if exists, err := e.ipTablesClient.Exists("mangle", "PREROUTING", "-j", name, "-m", "comment", "--comment", e.comment); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
@@ -165,7 +140,7 @@ func (e *Egress) InsertMangeTableIntoPrerouting(name string) error {
|
||||
}
|
||||
|
||||
func (e *Egress) InsertSourceNat(vip, podIP string) error {
|
||||
log.Info("[egress] Adding source nat", "original source", podIP, "new source", vip)
|
||||
log.Infof("[egress] Adding source nat from [%s] => [%s]", podIP, vip)
|
||||
if exists, err := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-m", "comment", "--comment", e.comment); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
@@ -178,17 +153,17 @@ func (e *Egress) InsertSourceNat(vip, podIP string) error {
|
||||
}
|
||||
|
||||
func (e *Egress) InsertSourceNatForDestinationPort(vip, podIP, port, proto string) error {
|
||||
log.Info("[egress] Adding source nat", "from", podIP, "to", vip, "port", port)
|
||||
log.Infof("[egress] Adding source nat from [%s] => [%s], with destination port [%s]", podIP, vip, port)
|
||||
natRules, err := e.ipTablesClient.List("nat", "POSTROUTING")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundNatRules := e.findExistingVIP(natRules, vip)
|
||||
log.Warn("[egress] Cleaning existing postrouting nat rules for vip", "rulecount", len(foundNatRules), "vip", vip)
|
||||
log.Warnf("[egress] Cleaning [%d] existing postrouting nat rules for vip [%s]", len(foundNatRules), vip)
|
||||
for x := range foundNatRules {
|
||||
err = e.ipTablesClient.Delete("nat", "POSTROUTING", foundNatRules[x][2:]...)
|
||||
if err != nil {
|
||||
log.Error("[egress] removing rule", "err", err)
|
||||
log.Errorf("[egress] Error removing rule [%v]", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,13 +182,13 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
|
||||
nfct, err := ct.Open(&ct.Config{})
|
||||
if err != nil {
|
||||
log.Error("create conntrack client", "err", err)
|
||||
log.Errorf("could not create nfct: %v", err)
|
||||
return err
|
||||
}
|
||||
defer nfct.Close()
|
||||
sessions, err := nfct.Dump(ct.Conntrack, ct.IPv4)
|
||||
if err != nil {
|
||||
log.Error("could not dump sessions", "err", err)
|
||||
log.Errorf("could not dump sessions: %v", err)
|
||||
return err
|
||||
}
|
||||
destPortProtocol := make(map[uint16]uint8)
|
||||
@@ -240,7 +215,7 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
case strings.ToLower("sctp"):
|
||||
destPortProtocol[uint16(port)] = ProtocolSCTP
|
||||
default:
|
||||
log.Error("[egress] annotation protocol isn't supported", "protocol ID", data[0])
|
||||
log.Errorf("[egress] annotation protocol [%s] isn't supported", data[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,7 +241,7 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
case strings.ToLower("sctp"):
|
||||
srcPortProtocol[uint16(port)] = ProtocolSCTP
|
||||
default:
|
||||
log.Error("[egress] annotation protocol isn't supported", "protocol ID", data[0])
|
||||
log.Errorf("[egress] annotation protocol [%s] isn't supported", data[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -279,14 +254,14 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
if destinationPorts != "" {
|
||||
proto := destPortProtocol[*session.Origin.Proto.DstPort]
|
||||
if proto == *session.Origin.Proto.Number {
|
||||
log.Info("[egress] cleaning existing connection", "src", session.Origin.Src.String(), "dst", session.Origin.Dst.String(), "dst port", *session.Origin.Proto.DstPort, "protocol", *session.Origin.Proto.Number)
|
||||
log.Infof("[egress] cleaning existing connection Source [%s] -> [%s:%d] proto: [%d] ", session.Origin.Src.String(), session.Origin.Dst.String(), *session.Origin.Proto.DstPort, *session.Origin.Proto.Number)
|
||||
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
|
||||
}
|
||||
} else {
|
||||
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("could not delete sessions", "err", err)
|
||||
log.Errorf("could not delete sessions: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -299,14 +274,14 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
if srcPorts != "" {
|
||||
proto := srcPortProtocol[*session.Origin.Proto.DstPort]
|
||||
if proto == *session.Origin.Proto.Number {
|
||||
log.Info("[egress] cleaning existing connection", "src", session.Origin.Src.String(), "dst", session.Origin.Dst.String(), "dst port", *session.Origin.Proto.DstPort, "protocol", *session.Origin.Proto.Number)
|
||||
log.Infof("[egress] cleaning existing connection Source [%s] -> [%s:%d] proto: [%d] ", session.Origin.Src.String(), session.Origin.Dst.String(), *session.Origin.Proto.DstPort, *session.Origin.Proto.Number)
|
||||
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
|
||||
}
|
||||
} else {
|
||||
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("could not delete sessions", "err", err)
|
||||
log.Errorf("could not delete sessions: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -318,13 +293,13 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
|
||||
// Debug functions
|
||||
|
||||
func (e *Egress) DumpChain(name string) error {
|
||||
log.Info("Dumping chain", "name", name)
|
||||
log.Infof("Dumping chain [%s]", name)
|
||||
c, err := e.ipTablesClient.List("mangle", name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for x := range c {
|
||||
log.Info("", "rule", c[x])
|
||||
log.Infof("Rule -> %s", c[x])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -335,16 +310,16 @@ func (e *Egress) CleanIPtables() error {
|
||||
return err
|
||||
}
|
||||
foundNatRules := e.findRules(natRules)
|
||||
log.Warn("[egress] Cleaning dangling postrouting nat rules", "rulecount", len(foundNatRules))
|
||||
log.Warnf("[egress] Cleaning [%d] dangling postrouting nat rules", len(foundNatRules))
|
||||
for x := range foundNatRules {
|
||||
err = e.ipTablesClient.Delete("nat", "POSTROUTING", foundNatRules[x][2:]...)
|
||||
if err != nil {
|
||||
log.Error("[egress] Error removing rule", "err", err)
|
||||
log.Errorf("[egress] Error removing rule [%v]", err)
|
||||
}
|
||||
}
|
||||
exists, err := e.CheckMangleChain(MangleChainName)
|
||||
if err != nil {
|
||||
log.Debug("[egress] No Mangle chain exists", "err", err)
|
||||
log.Debugf("[egress] No Mangle chain exists [%v]", err)
|
||||
}
|
||||
if exists {
|
||||
mangleRules, err := e.ipTablesClient.List("mangle", MangleChainName)
|
||||
@@ -352,11 +327,11 @@ func (e *Egress) CleanIPtables() error {
|
||||
return err
|
||||
}
|
||||
foundNatRules = e.findRules(mangleRules)
|
||||
log.Warn("[egress] Cleaning dangling prerouting mangle rules", "rulecount", len(foundNatRules))
|
||||
log.Warnf("[egress] Cleaning [%d] dangling prerouting mangle rules", len(foundNatRules))
|
||||
for x := range foundNatRules {
|
||||
err = e.ipTablesClient.Delete("mangle", MangleChainName, foundNatRules[x][2:]...)
|
||||
if err != nil {
|
||||
log.Error("[egress] Error removing rule", "err", err)
|
||||
log.Errorf("[egress] Error removing rule [%v]", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,7 +343,7 @@ func (e *Egress) CleanIPtables() error {
|
||||
// log.Errorf("[egress] Error removing flushing table [%v]", err)
|
||||
// }
|
||||
} else {
|
||||
log.Warn("No existing mangle chain exists", "chain name", MangleChainName)
|
||||
log.Warnf("No existing mangle chain [%s] exists", MangleChainName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -409,13 +384,13 @@ func (e *Egress) findExistingVIP(rules []string, vip string) [][]string {
|
||||
func ClearIPTables(useNftables bool, namespace string, protocol iptables.Protocol) {
|
||||
i, err := CreateIptablesClient(useNftables, namespace, protocol)
|
||||
if err != nil {
|
||||
log.Warn("[egress] Unable to clean any dangling egress rules", "err", err)
|
||||
log.Warnf("[egress] Unable to clean any dangling egress rules [%v]", err)
|
||||
log.Warn("[egress] Can be ignored in non iptables release of kube-vip")
|
||||
} else {
|
||||
log.Info("[egress] Cleaning any dangling kube-vip egress rules")
|
||||
cleanErr := i.CleanIPtables()
|
||||
if cleanErr != nil {
|
||||
log.Error("Error cleaning rules", "err", cleanErr)
|
||||
log.Errorf("Error cleaning rules [%v]", cleanErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
|
||||
"github.com/mdlayher/ndp"
|
||||
|
||||
log "log/slog"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// NdpResponder defines the parameters for the NDP connection.
|
||||
@@ -50,7 +50,7 @@ func (n *NdpResponder) SendGratuitous(address string) error {
|
||||
return fmt.Errorf("failed to parse address %s", ip)
|
||||
}
|
||||
|
||||
log.Info("Broadcasting NDP update", "ip", address, "hwaddr", n.hardwareAddr, "interface", n.intf)
|
||||
log.Infof("Broadcasting NDP update for %s (%s) via %s", address, n.hardwareAddr, n.intf)
|
||||
return n.advertise(netip.IPv6LinkLocalAllNodes(), ip, true)
|
||||
}
|
||||
|
||||
@@ -66,7 +66,6 @@ func (n *NdpResponder) advertise(dst, target netip.Addr, gratuitous bool) error
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
log.Debug("ndp", "advertisement", m)
|
||||
log.Infof("ndp: %v", m)
|
||||
return n.conn.WriteTo(m, nil, dst)
|
||||
}
|
||||
|
||||
@@ -8,9 +8,8 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
@@ -121,6 +120,17 @@ func IsIPv6CIDR(cidr string) bool {
|
||||
return ip.To4() == nil
|
||||
}
|
||||
|
||||
// GetFullMask returns /32 for an IPv4 address and /128 for an IPv6 address
|
||||
func GetFullMask(address string) (string, error) {
|
||||
if IsIPv4(address) {
|
||||
return "/32", nil
|
||||
}
|
||||
if IsIPv6(address) {
|
||||
return "/128", nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to parse %s as either IPv4 or IPv6", address)
|
||||
}
|
||||
|
||||
// GetDefaultGatewayInterface return default gateway interface link
|
||||
func GetDefaultGatewayInterface() (*net.Interface, error) {
|
||||
routes, err := netlink.RouteList(nil, syscall.AF_INET)
|
||||
@@ -157,7 +167,7 @@ func MonitorDefaultInterface(ctx context.Context, defaultIF *net.Interface) erro
|
||||
for {
|
||||
select {
|
||||
case r := <-routeCh:
|
||||
log.Debug(fmt.Sprintf("type: %d, route: %+v", r.Type, r.Route))
|
||||
log.Debugf("type: %d, route: %+v", r.Type, r.Route)
|
||||
if r.Type == syscall.RTM_DELROUTE && (r.Dst == nil || r.Dst.String() == "0.0.0.0/0") && r.LinkIndex == defaultIF.Index {
|
||||
return fmt.Errorf("default route deleted and the default interface may be invalid")
|
||||
}
|
||||
@@ -181,7 +191,7 @@ func GenerateMac() (mac string) {
|
||||
* - https://macaddress.io/database-download
|
||||
*/
|
||||
mac = fmt.Sprintf("%s:%s:%s:%02x:%02x:%02x", "00", "00", "6C", buf[0], buf[1], buf[2])
|
||||
log.Info("Generated mac", "address", mac)
|
||||
log.Infof("Generated mac: %s", mac)
|
||||
return mac
|
||||
}
|
||||
|
||||
@@ -192,50 +202,3 @@ func Split(values string) []string {
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetInterfaceByIP returns the network interface that has the specified IP address assigned.
|
||||
func GetInterfaceByIP(ipAddr string) (*netlink.Link, error) {
|
||||
ip := net.ParseIP(ipAddr)
|
||||
if ip == nil {
|
||||
return nil, fmt.Errorf("invalid IP address: %s", ipAddr)
|
||||
}
|
||||
|
||||
links, err := netlink.LinkList()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list network interfaces: %v", err)
|
||||
}
|
||||
|
||||
for i := range links {
|
||||
addrs, err := netlink.AddrList(links[i], netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list addresses for interface %s: %v", links[i].Attrs().Name, err)
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
if addr.IP.Equal(ip) {
|
||||
return &links[i], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no interface found with IP address: %s", ipAddr)
|
||||
}
|
||||
|
||||
// GetNonLinkLocalIP returns the first non link-local IPv4/IPv6 address on the given interface.
|
||||
func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
|
||||
a, err := netlink.AddrList(*iface, family)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to list addresses for interface %s: %v", (*iface).Attrs().Name, err)
|
||||
}
|
||||
|
||||
for _, addr := range a {
|
||||
if addr.IPNet != nil {
|
||||
ip := addr.IPNet.IP
|
||||
if !ip.IsLinkLocalUnicast() {
|
||||
return ip.String(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
|
||||
}
|
||||
|
||||
13
test.sh
13
test.sh
@@ -1,13 +0,0 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
rm -f logs.txt
|
||||
|
||||
for i in $(seq 1 100);
|
||||
do
|
||||
echo "RUN $i"
|
||||
GOMAXPROCS=4 make e2e-tests129
|
||||
if [[ "$?" -ne 0 ]]; then
|
||||
echo "FAILED AT RUN $i"
|
||||
break
|
||||
fi
|
||||
done
|
||||
@@ -1,3 +0,0 @@
|
||||
[global.config]
|
||||
as = {{ .AS }}
|
||||
router-id = "1.1.1.1"
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -1,683 +0,0 @@
|
||||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
package e2e_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/klog/v2"
|
||||
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
|
||||
"sigs.k8s.io/kind/pkg/log"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/kube-vip/kube-vip/testing/e2e"
|
||||
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/client"
|
||||
api "github.com/osrg/gobgp/v3/api"
|
||||
)
|
||||
|
||||
const (
|
||||
goBGPAS uint32 = 65500
|
||||
kubevipAS uint32 = 65501
|
||||
goBGPPort uint32 = 50051
|
||||
|
||||
defaultFixedNexthopv6 = "fc00:1000:1000:1000::100"
|
||||
defaultFixedNexthopv4 = "172.18.0.100"
|
||||
)
|
||||
|
||||
var _ = Describe("kube-vip BGP mode", Ordered, func() {
|
||||
if Mode == ModeBGP {
|
||||
var (
|
||||
logger log.Logger
|
||||
imagePath string
|
||||
k8sImagePath string
|
||||
configPath string
|
||||
kubeVIPBGPManifestTemplate *template.Template
|
||||
goBGPConfigTemplate *template.Template
|
||||
tempDirPath string
|
||||
v129 bool
|
||||
localIPv4 string
|
||||
localIPv6 string
|
||||
curDir string
|
||||
networkInterface string
|
||||
|
||||
bgpKill chan any
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
klog.SetOutput(GinkgoWriter)
|
||||
logger = e2e.TestLogger{}
|
||||
|
||||
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
|
||||
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
|
||||
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
|
||||
if configPath == "" {
|
||||
configPath = "/etc/kubernetes/admin.conf"
|
||||
}
|
||||
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
|
||||
networkInterface = "br-"
|
||||
}
|
||||
|
||||
_, v129 = os.LookupEnv("V129")
|
||||
var err error
|
||||
curDir, err = os.Getwd()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
templateBGPPath := filepath.Join(curDir, "kube-vip-bgp.yaml.tmpl")
|
||||
kubeVIPBGPManifestTemplate, err = template.New("kube-vip-bgp.yaml.tmpl").ParseFiles(templateBGPPath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
v4addr, _, err := deployment.GetLocalIPv4(networkInterface)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
localIPv4 = v4addr.String()
|
||||
|
||||
v6addr, _, err := deployment.GetLocalIPv6(networkInterface)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
localIPv6 = v6addr.String()
|
||||
|
||||
goBGPConfig := &e2e.BGPPeerValues{
|
||||
AS: goBGPAS,
|
||||
}
|
||||
|
||||
bgpKill = make(chan any)
|
||||
|
||||
goBGPConfigPath := filepath.Join(filepath.Join(curDir, "bgp"), "config.toml.tmpl")
|
||||
goBGPConfigTemplate, err = template.New("config.toml.tmpl").ParseFiles(goBGPConfigPath)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
goBGPConfigPath = filepath.Join(tempDirPath, "config.toml")
|
||||
|
||||
f, err := os.OpenFile(goBGPConfigPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer f.Close()
|
||||
|
||||
err = goBGPConfigTemplate.Execute(f, goBGPConfig)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
go startGoBGP(goBGPConfigPath, bgpKill)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
close(bgpKill)
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv4 services BGP mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
|
||||
k8sImagePath, e2e.IPv4Family, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129,
|
||||
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, p := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: p.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv4 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv6 services BGP mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
|
||||
k8sImagePath, e2e.IPv6Family, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129,
|
||||
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, p := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: p.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv6 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - fixed nexthop", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
|
||||
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
|
||||
logger, nodesNumber, "fixed", "mpbgp-ipv4")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, p := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: p.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv6 routes over IPv4 session",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv6)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv6)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
|
||||
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
|
||||
logger, nodesNumber, "fixed", "mpbgp-ipv6")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, n := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: n.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv4 routes over IPv6 session",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv4)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv4)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - auto_sourceif nexthop", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
containerIP string
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
_, containerIP = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
|
||||
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
|
||||
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv4")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, p := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: p.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv6 routes over IPv4 session",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
gobgpClient api.GobgpApiClient
|
||||
gobgpPeers []*e2e.BGPPeerValues
|
||||
containerIP string
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
containerIP, _ = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
|
||||
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
|
||||
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv6")
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
for _, n := range gobgpPeers {
|
||||
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
|
||||
Address: n.IP,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("advertise IPv4 routes over IPv6 session",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
func testBGP(offset uint, lbFamily string, afiFamily api.Family_Afi, svcFamily []corev1.IPFamily, svcName string,
|
||||
trafficPolicy corev1.ServiceExternalTrafficPolicy, client kubernetes.Interface, numberOfServices int, gobgpClient api.GobgpApiClient, expectedNexthop string) {
|
||||
lbAddress := e2e.GenerateVIP(lbFamily, offset)
|
||||
routeCheckFamily := &api.Family{
|
||||
Afi: afiFamily,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
}
|
||||
testServiceBGP(svcName, lbAddress, trafficPolicy, client, svcFamily, numberOfServices, gobgpClient, routeCheckFamily, expectedNexthop)
|
||||
}
|
||||
|
||||
func setupEnv(tempDirPath, cpVIP, clusterName *string, manifestValues *e2e.KubevipManifestValues,
|
||||
localIPv4, localIPv6, imagePath, configPath, k8sImagePath, clusterAddrFamily, bgpClientAddrFamily string, peerAddrFamily []string, client *kubernetes.Interface,
|
||||
gobgpPeers *[]*e2e.BGPPeerValues, v129 bool, kubeVIPBGPManifestTemplate *template.Template, gobgpClient *api.GobgpApiClient,
|
||||
logger log.Logger, nodesNumber int, mpbgpnexthop, clusterNameSuffix string) (string, string) {
|
||||
var err error
|
||||
*tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
*cpVIP = e2e.GenerateVIP(clusterAddrFamily, SOffset.Get())
|
||||
|
||||
var clusterIPFamily kindconfigv1alpha4.ClusterIPFamily
|
||||
var podSubnet, serviceSubnet string
|
||||
switch clusterAddrFamily {
|
||||
case e2e.IPv6Family:
|
||||
clusterIPFamily = kindconfigv1alpha4.IPv6Family
|
||||
case e2e.DualstackFamily:
|
||||
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
|
||||
case e2e.DualstackFamilyIPv6:
|
||||
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
|
||||
podSubnet = "fd00:10:244::/56,10.244.0.0/16"
|
||||
serviceSubnet = "fd00:10:96::/112,10.96.0.0/16"
|
||||
default:
|
||||
clusterIPFamily = kindconfigv1alpha4.IPv4Family
|
||||
}
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: clusterIPFamily,
|
||||
}
|
||||
|
||||
if podSubnet != "" && serviceSubnet != "" {
|
||||
networking.PodSubnet = podSubnet
|
||||
networking.ServiceSubnet = serviceSubnet
|
||||
}
|
||||
|
||||
kvPeers := []*e2e.BGPPeerValues{}
|
||||
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
|
||||
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
|
||||
IP: localIPv4,
|
||||
AS: goBGPAS,
|
||||
IPFamily: e2e.IPv4Family,
|
||||
})
|
||||
}
|
||||
|
||||
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
|
||||
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
|
||||
IP: localIPv6,
|
||||
AS: goBGPAS,
|
||||
IPFamily: e2e.IPv6Family,
|
||||
})
|
||||
}
|
||||
|
||||
kvPeersStr := []string{}
|
||||
for _, p := range kvPeers {
|
||||
kvPeersStr = append(kvPeersStr, p.String())
|
||||
}
|
||||
|
||||
manifestValues = &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: *cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "false",
|
||||
SvcEnable: "true",
|
||||
SvcElectionEnable: "false",
|
||||
BGPAS: kubevipAS,
|
||||
BGPPeers: strings.Join(kvPeersStr, ","),
|
||||
MPBGPNexthop: mpbgpnexthop,
|
||||
MPBGPNexthopIPv4: defaultFixedNexthopv4,
|
||||
MPBGPNexthopIPv6: defaultFixedNexthopv6,
|
||||
}
|
||||
|
||||
By(manifestValues.BGPPeers)
|
||||
|
||||
*clusterName, *client = prepareCluster(*tempDirPath, clusterNameSuffix, k8sImagePath, v129, kubeVIPBGPManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
|
||||
container := fmt.Sprintf("%s-control-plane", *clusterName)
|
||||
|
||||
containerIPv4, containerIPv6, err := GetContainerIPs(container)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
|
||||
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
|
||||
IP: containerIPv4,
|
||||
AS: kubevipAS,
|
||||
IPFamily: e2e.IPv4Family,
|
||||
})
|
||||
}
|
||||
|
||||
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
|
||||
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
|
||||
IP: containerIPv6,
|
||||
AS: kubevipAS,
|
||||
IPFamily: e2e.IPv6Family,
|
||||
})
|
||||
}
|
||||
|
||||
if bgpClientAddrFamily == e2e.IPv6Family {
|
||||
*gobgpClient, err = newGoBGPClient(localIPv6, goBGPPort)
|
||||
} else {
|
||||
*gobgpClient, err = newGoBGPClient(localIPv4, goBGPPort)
|
||||
}
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for _, p := range *gobgpPeers {
|
||||
if slices.Contains(peerAddrFamily, p.IPFamily) {
|
||||
peerCtx := context.TODO()
|
||||
Eventually(peerCtx, func() error {
|
||||
_, err = (*gobgpClient).AddPeer(context.TODO(), &api.AddPeerRequest{
|
||||
Peer: &api.Peer{
|
||||
Conf: &api.PeerConf{
|
||||
NeighborAddress: p.IP,
|
||||
PeerAsn: uint32(p.AS),
|
||||
},
|
||||
AfiSafis: []*api.AfiSafi{
|
||||
{
|
||||
Config: &api.AfiSafiConfig{
|
||||
Enabled: true,
|
||||
Family: &api.Family{
|
||||
Afi: api.Family_AFI_IP6,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Config: &api.AfiSafiConfig{
|
||||
Enabled: true,
|
||||
Family: &api.Family{
|
||||
Afi: api.Family_AFI_IP,
|
||||
Safi: api.Family_SAFI_UNICAST,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
return err
|
||||
}, "120s", "100ms").Should(Succeed())
|
||||
}
|
||||
}
|
||||
|
||||
return containerIPv4, containerIPv6
|
||||
}
|
||||
|
||||
func testServiceBGP(svcName, lbAddress string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
|
||||
client kubernetes.Interface, serviceAddrFamily []corev1.IPFamily, numberOfServices int,
|
||||
gobgpClient api.GobgpApiClient, gobgpFamily *api.Family, expectedNexthop string) {
|
||||
lbAddresses := vip.Split(lbAddress)
|
||||
|
||||
services := []string{}
|
||||
for i := range numberOfServices {
|
||||
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
|
||||
}
|
||||
|
||||
for _, svc := range services {
|
||||
createTestService(svc, dsNamespace, dsName, lbAddress,
|
||||
client, corev1.IPFamilyPolicyPreferDualStack, serviceAddrFamily, trafficPolicy)
|
||||
}
|
||||
|
||||
for _, addr := range lbAddresses {
|
||||
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
|
||||
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
|
||||
if expectedNexthop != "" {
|
||||
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
|
||||
}
|
||||
}
|
||||
|
||||
for i := range numberOfServices {
|
||||
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
if i < numberOfServices-1 {
|
||||
for _, addr := range lbAddresses {
|
||||
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
|
||||
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
|
||||
if expectedNexthop != "" {
|
||||
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, addr := range lbAddresses {
|
||||
checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 0)
|
||||
}
|
||||
}
|
||||
|
||||
func GetContainerIPs(containerName string) (string, string, error) {
|
||||
cli, err := client.NewClientWithOpts(client.FromEnv)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
containers, err := cli.ContainerList(context.Background(), container.ListOptions{})
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to list containers: %w", err)
|
||||
}
|
||||
|
||||
for _, c := range containers {
|
||||
for _, n := range c.Names {
|
||||
if n[1:] == containerName {
|
||||
fmt.Println(n)
|
||||
for _, n := range c.NetworkSettings.Networks {
|
||||
return n.IPAddress, n.GlobalIPv6Address, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
func newGoBGPClient(address string, port uint32) (api.GobgpApiClient, error) {
|
||||
grpcOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
|
||||
target := net.JoinHostPort(address, strconv.Itoa(int(port)))
|
||||
conn, err := grpc.NewClient(target, grpcOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to GoBGP server %q: %w", target, err)
|
||||
}
|
||||
|
||||
return api.NewGobgpApiClient(conn), nil
|
||||
}
|
||||
|
||||
func checkGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedPaths int) []*api.Destination {
|
||||
var paths []*api.Destination
|
||||
Eventually(func() error {
|
||||
var err error
|
||||
paths, err = getGoBGPPaths(ctx, client, family, prefixes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(paths) != expectedPaths {
|
||||
return fmt.Errorf("expected %d paths, but found %d", expectedPaths, len(paths))
|
||||
}
|
||||
return nil
|
||||
}, "120s").ShouldNot(HaveOccurred())
|
||||
return paths
|
||||
}
|
||||
|
||||
func getGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix) ([]*api.Destination, error) {
|
||||
pathCtx, cancel := context.WithTimeout(ctx, time.Second*5)
|
||||
defer cancel()
|
||||
stream, err := client.ListPath(pathCtx, &api.ListPathRequest{
|
||||
TableType: api.TableType_GLOBAL,
|
||||
Family: family,
|
||||
Name: "",
|
||||
Prefixes: prefixes,
|
||||
SortType: api.ListPathRequest_PREFIX,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rib := make([]*api.Destination, 0)
|
||||
for {
|
||||
r, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rib = append(rib, r.Destination)
|
||||
}
|
||||
|
||||
return rib, nil
|
||||
}
|
||||
|
||||
func startGoBGP(config string, kill chan any) {
|
||||
By("starting GoBGP server")
|
||||
cmd := exec.Command("../../bin/gobgpd", "-f", config)
|
||||
go cmd.Run()
|
||||
<-kill
|
||||
By("stopping GoBGP server")
|
||||
err := cmd.Process.Kill()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
@@ -1,583 +0,0 @@
|
||||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
package e2e_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/klog/v2"
|
||||
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
|
||||
"sigs.k8s.io/kind/pkg/log"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/kube-vip/kube-vip/testing/e2e"
|
||||
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
|
||||
)
|
||||
|
||||
var _ = Describe("kube-vip routing table mode", func() {
|
||||
if Mode == ModeRT {
|
||||
var (
|
||||
logger log.Logger
|
||||
imagePath string
|
||||
k8sImagePath string
|
||||
configPath string
|
||||
kubeVIPRoutingTableManifestTemplate *template.Template
|
||||
tempDirPath string
|
||||
v129 bool
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
klog.SetOutput(GinkgoWriter)
|
||||
logger = e2e.TestLogger{}
|
||||
|
||||
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
|
||||
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
|
||||
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
|
||||
if configPath == "" {
|
||||
configPath = "/etc/kubernetes/admin.conf"
|
||||
}
|
||||
_, v129 = os.LookupEnv("V129")
|
||||
curDir, err := os.Getwd()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
templateRoutingTablePath := filepath.Join(curDir, "kube-vip-routing-table.yaml.tmpl")
|
||||
kubeVIPRoutingTableManifestTemplate, err = template.New("kube-vip-routing-table.yaml.tmpl").ParseFiles(templateRoutingTablePath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv4 control-plane routing table mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
|
||||
nodesNumber = 3
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.IPv4Family,
|
||||
}
|
||||
|
||||
manifestValues := &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "true",
|
||||
SvcEnable: "false",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
It("setups IPv4 address and route on control-plane node", func() {
|
||||
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
for i := 1; i <= nodesNumber; i++ {
|
||||
var container string
|
||||
if i > 1 {
|
||||
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
|
||||
} else {
|
||||
container = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
}
|
||||
|
||||
checkIPAddress(cpVIP, container, true)
|
||||
e2e.CheckRoutePresence(cpVIP, container, true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv6 control-plane routing table mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
|
||||
nodesNumber = 3
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
|
||||
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.IPv6Family,
|
||||
}
|
||||
|
||||
manifestValues := &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "true",
|
||||
SvcEnable: "false",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
It("setups IPv6 address and route on control-plane node", func() {
|
||||
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
for i := 1; i <= nodesNumber; i++ {
|
||||
var container string
|
||||
if i > 1 {
|
||||
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
|
||||
} else {
|
||||
container = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
}
|
||||
|
||||
checkIPAddress(cpVIP, container, true)
|
||||
e2e.CheckRoutePresence(cpVIP, container, true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv4 primary", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
|
||||
nodesNumber = 3
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.DualStackFamily,
|
||||
}
|
||||
|
||||
manifestValues := &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "true",
|
||||
SvcEnable: "false",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
networkInterface := ""
|
||||
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
|
||||
networkInterface = "br-"
|
||||
}
|
||||
|
||||
localIPv6, localIPv6Net, err := deployment.GetLocalIPv6(networkInterface)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
addSAN := &san{
|
||||
ip: localIPv6,
|
||||
ipnet: localIPv6Net,
|
||||
}
|
||||
|
||||
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
It("setups DualStack addresses and routes on control-plane nodes", func() {
|
||||
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
for i := 1; i <= nodesNumber; i++ {
|
||||
var container string
|
||||
if i > 1 {
|
||||
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
|
||||
} else {
|
||||
container = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
}
|
||||
|
||||
addresses := vip.Split(cpVIP)
|
||||
|
||||
for _, addr := range addresses {
|
||||
checkIPAddress(addr, container, true)
|
||||
e2e.CheckRoutePresence(addr, container, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv6 primary", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
|
||||
nodesNumber = 3
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.DualStackFamily,
|
||||
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
|
||||
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
|
||||
}
|
||||
|
||||
manifestValues := &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "true",
|
||||
SvcEnable: "false",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
networkInterface := ""
|
||||
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
|
||||
networkInterface = "br-"
|
||||
}
|
||||
|
||||
localIPv4, localIPv4Net, err := deployment.GetLocalIPv4(networkInterface)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
addSAN := &san{
|
||||
ip: localIPv4,
|
||||
ipnet: localIPv4Net,
|
||||
}
|
||||
|
||||
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
It("setups DualStack addresses and routes on control-plane nodes", func() {
|
||||
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
for i := 1; i <= nodesNumber; i++ {
|
||||
var container string
|
||||
if i > 1 {
|
||||
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
|
||||
} else {
|
||||
container = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
}
|
||||
|
||||
addresses := vip.Split(cpVIP)
|
||||
|
||||
for _, addr := range addresses {
|
||||
checkIPAddress(addr, container, true)
|
||||
e2e.CheckRoutePresence(addr, container, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv4 services routing table mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
svcElection bool
|
||||
ipFamily []corev1.IPFamily
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.IPv4Family,
|
||||
}
|
||||
|
||||
manifestValues = &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "false",
|
||||
SvcEnable: "true",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
var err error
|
||||
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol}
|
||||
|
||||
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("configures an IPv4 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
|
||||
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
|
||||
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip IPv6 services routing table mode functionality", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
svcElection bool
|
||||
ipFamily []corev1.IPFamily
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.IPv6Family,
|
||||
}
|
||||
|
||||
manifestValues = &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "false",
|
||||
SvcEnable: "true",
|
||||
SvcElectionEnable: "false",
|
||||
}
|
||||
|
||||
var err error
|
||||
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
ipFamily = []corev1.IPFamily{corev1.IPv6Protocol}
|
||||
|
||||
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("configures an IPv6 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
|
||||
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
|
||||
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services routing table mode functionality - IPv4 primary", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
svcElection bool
|
||||
ipFamily []corev1.IPFamily
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.DualStackFamily,
|
||||
}
|
||||
|
||||
manifestValues = &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "false",
|
||||
SvcEnable: "true",
|
||||
SvcElectionEnable: "false",
|
||||
EnableEndpointslices: "true",
|
||||
}
|
||||
|
||||
var err error
|
||||
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
|
||||
|
||||
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("configures an IPv4 and IPv6 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateDualStackVIP(offset)
|
||||
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateDualStackVIP(offset)
|
||||
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("kube-vip DualStack services routing table mode functionality - IPv6 primary", Ordered, func() {
|
||||
var (
|
||||
cpVIP string
|
||||
clusterName string
|
||||
client kubernetes.Interface
|
||||
manifestValues *e2e.KubevipManifestValues
|
||||
svcElection bool
|
||||
ipFamily []corev1.IPFamily
|
||||
|
||||
nodesNumber = 1
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
|
||||
|
||||
networking := &kindconfigv1alpha4.Networking{
|
||||
IPFamily: kindconfigv1alpha4.DualStackFamily,
|
||||
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
|
||||
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
|
||||
}
|
||||
|
||||
manifestValues = &e2e.KubevipManifestValues{
|
||||
ControlPlaneVIP: cpVIP,
|
||||
ImagePath: imagePath,
|
||||
ConfigPath: configPath,
|
||||
ControlPlaneEnable: "false",
|
||||
SvcEnable: "true",
|
||||
SvcElectionEnable: "false",
|
||||
EnableEndpointslices: "true",
|
||||
}
|
||||
|
||||
var err error
|
||||
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
|
||||
|
||||
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
|
||||
})
|
||||
|
||||
DescribeTable("configures an IPv4 and IPv6 routes for services",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateDualStackVIP(offset)
|
||||
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
|
||||
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
|
||||
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
|
||||
lbAddress := e2e.GenerateDualStackVIP(offset)
|
||||
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
|
||||
},
|
||||
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
|
||||
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
func testServiceRT(svcName, lbAddress, leaseName, leaseNamespace, clusterName string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
|
||||
client kubernetes.Interface, serviceElection bool, ipFamily []corev1.IPFamily, numberOfServices int) {
|
||||
lbAddresses := vip.Split(lbAddress)
|
||||
|
||||
services := []string{}
|
||||
for i := range numberOfServices {
|
||||
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
|
||||
}
|
||||
|
||||
for _, svc := range services {
|
||||
createTestService(svc, dsNamespace, dsName, lbAddress,
|
||||
client, corev1.IPFamilyPolicyPreferDualStack, ipFamily, trafficPolicy)
|
||||
}
|
||||
|
||||
var container string
|
||||
if serviceElection {
|
||||
container = e2e.GetLeaseHolder(leaseName, leaseNamespace, client)
|
||||
} else {
|
||||
container = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
}
|
||||
|
||||
for _, addr := range lbAddresses {
|
||||
e2e.CheckRoutePresence(addr, container, true)
|
||||
}
|
||||
|
||||
for i := range numberOfServices {
|
||||
expected := i < numberOfServices-1
|
||||
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for _, addr := range lbAddresses {
|
||||
e2e.CheckRoutePresence(addr, container, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type san struct {
|
||||
ip *net.IP
|
||||
ipnet *net.IPNet
|
||||
}
|
||||
@@ -4,41 +4,15 @@
|
||||
package e2e_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/kube-vip/kube-vip/testing/e2e"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeEnv = "TEST_MODE"
|
||||
ModeARP = "arp"
|
||||
ModeRT = "rt"
|
||||
ModeBGP = "bgp"
|
||||
)
|
||||
|
||||
var (
|
||||
SOffset *e2e.SecureOffset
|
||||
ConfigMtx *sync.Mutex
|
||||
Mode string
|
||||
"github.com/kube-vip/kube-vip/testing/e2e"
|
||||
)
|
||||
|
||||
func TestE2E(t *testing.T) {
|
||||
mode := os.Getenv(ModeEnv)
|
||||
if mode == "" {
|
||||
Mode = ModeARP
|
||||
} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
|
||||
log.Fatal("invalid", "mode", mode)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
Mode = mode
|
||||
}
|
||||
SOffset = e2e.NewOffset(5)
|
||||
ConfigMtx = &sync.Mutex{}
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "E2E Suite")
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -268,6 +268,15 @@ func (c *Cluster) Delete() {
|
||||
Expect(c.provider.Delete(c.Name, "")).To(Succeed())
|
||||
}
|
||||
|
||||
func startKubeletForEtcd(node nodes.Node) {
|
||||
e2e.RunInNode(node,
|
||||
"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
|
||||
)
|
||||
e2e.RunInNode(node,
|
||||
"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
|
||||
)
|
||||
}
|
||||
|
||||
func createCerts(node nodes.Node) {
|
||||
e2e.RunInNode(node,
|
||||
"kubeadm",
|
||||
|
||||
@@ -60,7 +60,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
|
||||
|
||||
test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
|
||||
|
||||
test.vip = e2e.GenerateVIP(e2e.IPv4Family, 5)
|
||||
test.vip = e2e.GenerateIPv4VIP()
|
||||
test.logger.Printf("Selected VIP %s", test.vip)
|
||||
|
||||
test.currentDir, err = os.Getwd()
|
||||
|
||||
@@ -32,7 +32,7 @@ spec:
|
||||
- name: cp_enable
|
||||
value: "true"
|
||||
- name: vip_loglevel
|
||||
value: "-4"
|
||||
value: "5"
|
||||
- name: vip_nodename
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
|
||||
@@ -6,34 +6,22 @@ package e2e
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/onsi/gomega/gexec"
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
|
||||
"sigs.k8s.io/kind/pkg/cluster"
|
||||
)
|
||||
|
||||
const (
|
||||
IPv4Family = "IPv4"
|
||||
IPv6Family = "IPv6"
|
||||
DualstackFamily = "DualStack"
|
||||
DualstackFamilyIPv6 = "DualStackIPv6"
|
||||
)
|
||||
|
||||
func EnsureKindNetwork() {
|
||||
By("checking if the Docker \"kind\" network exists")
|
||||
cmd := exec.Command("docker", "inspect", "kind")
|
||||
@@ -70,47 +58,53 @@ func EnsureKindNetwork() {
|
||||
Eventually(session).Should(gexec.Exit(0))
|
||||
}
|
||||
|
||||
func GenerateVIP(family string, offset uint) string {
|
||||
if family == DualstackFamily || family == DualstackFamilyIPv6 {
|
||||
return fmt.Sprintf("%s,%s", GenerateVIP(IPv4Family, offset), GenerateVIP(IPv6Family, offset))
|
||||
}
|
||||
|
||||
func GenerateIPv6VIP() string {
|
||||
cidrs := getKindNetworkSubnetCIDRs()
|
||||
|
||||
for _, cidr := range cidrs {
|
||||
if cidr != "" {
|
||||
ip, ipNet, parseErr := net.ParseCIDR(cidr)
|
||||
Expect(parseErr).NotTo(HaveOccurred())
|
||||
ip, ipNet, parseErr := net.ParseCIDR(cidr)
|
||||
Expect(parseErr).NotTo(HaveOccurred())
|
||||
|
||||
if ip.To4() == nil && family == IPv6Family {
|
||||
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
|
||||
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
|
||||
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
|
||||
if ip.To4() == nil {
|
||||
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
|
||||
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
|
||||
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
|
||||
|
||||
chosenVIP := make([]byte, 16)
|
||||
// Copy upper half into chosenVIP
|
||||
copy(chosenVIP, ipNet.IP[0:8])
|
||||
// Copy lower half into chosenVIP
|
||||
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
|
||||
return net.IP(chosenVIP).String()
|
||||
} else if ip.To4() != nil && family == IPv4Family {
|
||||
mask := binary.BigEndian.Uint32(ipNet.Mask)
|
||||
start := binary.BigEndian.Uint32(ipNet.IP)
|
||||
end := (start & mask) | (^mask)
|
||||
|
||||
chosenVIP := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
|
||||
return net.IP(chosenVIP).String()
|
||||
}
|
||||
chosenVIP := make([]byte, 16)
|
||||
// Copy upper half into chosenVIP
|
||||
copy(chosenVIP, ipNet.IP[0:8])
|
||||
// Copy lower half into chosenVIP
|
||||
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
|
||||
return net.IP(chosenVIP).String()
|
||||
}
|
||||
}
|
||||
|
||||
Fail("Could not find any " + family + " CIDRs in the Docker \"kind\" network")
|
||||
Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
|
||||
return ""
|
||||
}
|
||||
|
||||
func GenerateDualStackVIP(offset uint) string {
|
||||
return GenerateVIP(IPv4Family, offset) + "," + GenerateVIP(IPv6Family, offset)
|
||||
func GenerateIPv4VIP() string {
|
||||
cidrs := getKindNetworkSubnetCIDRs()
|
||||
|
||||
for _, cidr := range cidrs {
|
||||
ip, ipNet, parseErr := net.ParseCIDR(cidr)
|
||||
Expect(parseErr).NotTo(HaveOccurred())
|
||||
|
||||
if ip.To4() != nil {
|
||||
mask := binary.BigEndian.Uint32(ipNet.Mask)
|
||||
start := binary.BigEndian.Uint32(ipNet.IP)
|
||||
end := (start & mask) | (^mask)
|
||||
|
||||
chosenVIP := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(chosenVIP, end-5)
|
||||
return net.IP(chosenVIP).String()
|
||||
}
|
||||
}
|
||||
Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
|
||||
return ""
|
||||
}
|
||||
|
||||
func GenerateDualStackVIP() string {
|
||||
return GenerateIPv4VIP() + "," + GenerateIPv6VIP()
|
||||
}
|
||||
|
||||
func getKindNetworkSubnetCIDRs() []string {
|
||||
@@ -139,88 +133,30 @@ func getKindNetworkSubnetCIDRs() []string {
|
||||
return cidrs
|
||||
}
|
||||
|
||||
func CheckIPAddressPresence(ip string, container string, expected bool) bool {
|
||||
func CheckIPAddressPresence(ip string, container string) bool {
|
||||
cmdOut := new(bytes.Buffer)
|
||||
family := "-4"
|
||||
if vip.IsIPv6(ip) {
|
||||
family = "-6"
|
||||
}
|
||||
|
||||
cmd := exec.Command(
|
||||
"docker", "exec", container, "ip", family, "addr", "show", "dev", "eth0",
|
||||
)
|
||||
|
||||
cmdOut := new(bytes.Buffer)
|
||||
cmdErr := new(bytes.Buffer)
|
||||
cmd.Stdout = cmdOut
|
||||
cmd.Stderr = cmdErr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(cmdOut.String(), ip) == expected
|
||||
Eventually(cmd.Run(), "20s").Should(Succeed())
|
||||
return strings.Contains(cmdOut.String(), ip)
|
||||
}
|
||||
|
||||
func CheckIPAddressPresenceByLease(name, namespace, ip string, client kubernetes.Interface, expected bool) bool {
|
||||
container := GetLeaseHolder(name, namespace, client)
|
||||
if container == "" {
|
||||
return false
|
||||
}
|
||||
return CheckIPAddressPresence(ip, container, expected)
|
||||
}
|
||||
|
||||
func CheckRoutePresence(ip string, container string, expected bool) bool {
|
||||
func CheckRoutePresence(ip string, container string) bool {
|
||||
cmdOut := new(bytes.Buffer)
|
||||
family := "-4"
|
||||
if vip.IsIPv6(ip) {
|
||||
family = "-6"
|
||||
}
|
||||
|
||||
result := false
|
||||
Eventually(func() bool {
|
||||
cmdOut := new(bytes.Buffer)
|
||||
cmdErr := new(bytes.Buffer)
|
||||
|
||||
cmd := exec.Command(
|
||||
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
|
||||
)
|
||||
|
||||
cmd.Stdout = cmdOut
|
||||
cmd.Stderr = cmdErr
|
||||
cmd.Run()
|
||||
|
||||
result = strings.Contains(cmdOut.String(), ip) == expected
|
||||
|
||||
return result
|
||||
}, "60s", "1s").Should(BeTrue())
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func GetLeaseHolder(name, namespace string, client kubernetes.Interface) string {
|
||||
var lease *coordinationv1.Lease
|
||||
Eventually(func() error {
|
||||
var err error
|
||||
lease, err = client.CoordinationV1().Leases(namespace).Get(context.TODO(), name, metav1.GetOptions{})
|
||||
return err
|
||||
}, "300s").ShouldNot(HaveOccurred())
|
||||
|
||||
Expect(lease).ToNot(BeNil())
|
||||
return *lease.Spec.HolderIdentity
|
||||
}
|
||||
|
||||
type SecureOffset struct {
|
||||
value int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewOffset(value int) *SecureOffset {
|
||||
return &SecureOffset{
|
||||
value: value,
|
||||
}
|
||||
}
|
||||
|
||||
func (so *SecureOffset) Get() uint {
|
||||
so.mu.Lock()
|
||||
defer so.mu.Unlock()
|
||||
so.value++
|
||||
return uint(so.value - 1)
|
||||
cmd := exec.Command(
|
||||
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
|
||||
)
|
||||
cmd.Stdout = cmdOut
|
||||
Eventually(cmd.Run(), "20s").Should(Succeed())
|
||||
return strings.Contains(cmdOut.String(), ip)
|
||||
}
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kube-vip
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- name: kube-vip
|
||||
args:
|
||||
- manager
|
||||
- --prometheusHTTPServer
|
||||
- ""
|
||||
env:
|
||||
- name: vip_loglevel
|
||||
value: "-4"
|
||||
- name: lb_class_legacy_handling
|
||||
value: "false"
|
||||
- name: lb_class_name
|
||||
- name: prometheus_server
|
||||
value: :2112
|
||||
- name: disable_service_updates
|
||||
value: "false"
|
||||
- name: vip_arp
|
||||
value: "false"
|
||||
- name: port
|
||||
value: "6443"
|
||||
- name: vip_cidr
|
||||
value: "32"
|
||||
- name: vip_leaderelection
|
||||
value: "false"
|
||||
- name: vip_interface
|
||||
value: lo
|
||||
- name: bgp_enable
|
||||
value: "true"
|
||||
- name: bgp_hold_time
|
||||
value: "3"
|
||||
- name: bgp_keepalive_interval
|
||||
value: "1"
|
||||
- name: bgp_sourceif
|
||||
value: "eth0"
|
||||
- name: bgp_routerid
|
||||
value: "2.2.2.2"
|
||||
- name: bgp_as
|
||||
value: "{{ .BGPAS }}"
|
||||
- name: bgp_peers
|
||||
value: "{{ .BGPPeers }}"
|
||||
- name: mpbgp_nexthop
|
||||
value: "{{ .MPBGPNexthop }}"
|
||||
- name: mpbgp_ipv4
|
||||
value: "{{ .MPBGPNexthopIPv4 }}"
|
||||
- name: mpbgp_ipv6
|
||||
value: "{{ .MPBGPNexthopIPv6 }}"
|
||||
- name: address
|
||||
value: "{{ .ControlPlaneVIP }}"
|
||||
- name: cp_enable
|
||||
value: "{{ .ControlPlaneEnable }}"
|
||||
- name: svc_enable
|
||||
value: "{{ .SvcEnable }}"
|
||||
- name: svc_election
|
||||
value: "{{ .SvcElectionEnable }}"
|
||||
- name: enable_endpointslices
|
||||
value: "{{ .EnableEndpointslices }}"
|
||||
image: "{{ .ImagePath }}"
|
||||
imagePullPolicy: Never
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes/admin.conf
|
||||
name: kubeconfig
|
||||
hostAliases:
|
||||
- hostnames:
|
||||
- kubernetes
|
||||
ip: 127.0.0.1
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: "{{ .ConfigPath }}"
|
||||
name: kubeconfig
|
||||
@@ -12,8 +12,6 @@ spec:
|
||||
- --prometheusHTTPServer
|
||||
- ""
|
||||
env:
|
||||
- name: vip_loglevel
|
||||
value: "-4"
|
||||
- name: vip_arp
|
||||
value: "false"
|
||||
- name: vip_interface
|
||||
@@ -29,23 +27,19 @@ spec:
|
||||
- name: vip_retryperiod
|
||||
value: "1"
|
||||
- name: cp_enable
|
||||
value: "{{ .ControlPlaneEnable }}"
|
||||
value: "true"
|
||||
- name: vip_nodename
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: vip_subnet
|
||||
value: '32,128'
|
||||
- name: vip_cidr
|
||||
value: '32'
|
||||
- name: cp_namespace
|
||||
value: kube-system
|
||||
- name: svc_enable
|
||||
value: 'false'
|
||||
- name: vip_routingtable
|
||||
value: 'true'
|
||||
- name: svc_enable
|
||||
value: "{{ .SvcEnable }}"
|
||||
- name: svc_election
|
||||
value: "{{ .SvcElectionEnable }}"
|
||||
- name: enable_endpointslices
|
||||
value: "{{ .EnableEndpointslices }}"
|
||||
image: "{{ .ImagePath }}"
|
||||
imagePullPolicy: Never
|
||||
securityContext:
|
||||
|
||||
@@ -12,8 +12,6 @@ spec:
|
||||
- --prometheusHTTPServer
|
||||
- ""
|
||||
env:
|
||||
- name: vip_loglevel
|
||||
value: "-4"
|
||||
- name: vip_arp
|
||||
value: "true"
|
||||
- name: vip_interface
|
||||
@@ -34,12 +32,6 @@ spec:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: svc_enable
|
||||
value: "{{ .SvcEnable }}"
|
||||
- name: svc_election
|
||||
value: "{{ .SvcElectionEnable }}"
|
||||
- name: enable_endpointslices
|
||||
value: "{{ .EnableEndpointslices }}"
|
||||
image: "{{ .ImagePath }}"
|
||||
imagePullPolicy: Never
|
||||
securityContext:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package deployment
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gookit/slog"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func getKindNetworkSubnetCIDRs() ([]string, error) {
|
||||
@@ -95,7 +95,7 @@ func generateIPv6VIP() (string, error) {
|
||||
|
||||
}
|
||||
|
||||
func (config *TestConfig) manifestGen() error {
|
||||
func (config *testConfig) manifestGen() error {
|
||||
curDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -140,7 +140,7 @@ func (config *TestConfig) manifestGen() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
slog.Infof("🗃️ Manifest path %s", config.ManifestPath)
|
||||
log.Infof("🗃️ Manifest path %s", config.ManifestPath)
|
||||
err = kubeVIPManifestTemplate.Execute(manifestFile, kubevipManifestValues{
|
||||
ControlPlaneVIP: config.ControlPlaneAddress,
|
||||
ImagePath: config.ImagePath,
|
||||
@@ -1,15 +1,13 @@
|
||||
package deployment
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gookit/slog"
|
||||
"github.com/vishvananda/netlink"
|
||||
log "github.com/sirupsen/logrus"
|
||||
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
|
||||
"sigs.k8s.io/kind/pkg/cluster"
|
||||
"sigs.k8s.io/kind/pkg/cmd"
|
||||
@@ -18,11 +16,6 @@ import (
|
||||
|
||||
var provider *cluster.Provider
|
||||
|
||||
const (
|
||||
defaultIPv4Pool = "172.18.100.10-172.18.100.30"
|
||||
defaultIPv6Pool = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
|
||||
)
|
||||
|
||||
type kubevipManifestValues struct {
|
||||
ControlPlaneVIP string
|
||||
ImagePath string
|
||||
@@ -33,7 +26,7 @@ type nodeAddresses struct {
|
||||
addresses []string
|
||||
}
|
||||
|
||||
func (config *TestConfig) CreateKind() error {
|
||||
func (config *testConfig) createKind() error {
|
||||
|
||||
clusterConfig := kindconfigv1alpha4.Cluster{
|
||||
Networking: kindconfigv1alpha4.Networking{
|
||||
@@ -45,17 +38,13 @@ func (config *TestConfig) CreateKind() error {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if config.DualStack {
|
||||
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
|
||||
} else {
|
||||
if config.IPv6 || !config.Egress && config.EgressIPv6 {
|
||||
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
|
||||
}
|
||||
if config.IPv6 {
|
||||
// Change Networking Family
|
||||
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
|
||||
}
|
||||
|
||||
if config.Cilium {
|
||||
clusterConfig.Networking.DisableDefaultCNI = true
|
||||
if config.Dualstack {
|
||||
// Change Networking Family
|
||||
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
|
||||
}
|
||||
|
||||
if config.ControlPlane {
|
||||
@@ -85,13 +74,6 @@ func (config *TestConfig) CreateKind() error {
|
||||
//clusterConfig.Nodes = append(clusterConfig.Nodes, kindconfigv1alpha4.Node{Role: kindconfigv1alpha4.WorkerRole})
|
||||
}
|
||||
|
||||
// Change the default image if required
|
||||
if config.KindVersionImage != "" {
|
||||
for x := range clusterConfig.Nodes {
|
||||
clusterConfig.Nodes[x].Image = config.KindVersionImage
|
||||
}
|
||||
}
|
||||
|
||||
provider = cluster.NewProvider(cluster.ProviderWithLogger(cmd.NewLogger()), cluster.ProviderWithDocker())
|
||||
clusters, err := provider.List()
|
||||
if err != nil {
|
||||
@@ -100,7 +82,7 @@ func (config *TestConfig) CreateKind() error {
|
||||
found := false
|
||||
for x := range clusters {
|
||||
if clusters[x] == "services" {
|
||||
slog.Infof("Cluster already exists")
|
||||
log.Infof("Cluster already exists")
|
||||
found = true
|
||||
}
|
||||
}
|
||||
@@ -121,8 +103,8 @@ func (config *TestConfig) CreateKind() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if !config.SkipHostnameChange {
|
||||
slog.Infof("⚙️ changing hostnames on nodes to force using proper node names for service selection")
|
||||
if !config.skipHostnameChange {
|
||||
log.Infof("⚙️ changing hostnames on nodes to force using proper node names for service selection")
|
||||
for _, node := range nodes {
|
||||
nodeName := node.String()
|
||||
cmd := exec.Command("docker", "exec", nodeName, "hostname", nodeName+"-modified")
|
||||
@@ -132,11 +114,6 @@ func (config *TestConfig) CreateKind() error {
|
||||
}
|
||||
}
|
||||
|
||||
if config.Cilium {
|
||||
cmd := exec.Command("cilium", "install", "--helm-set", "ipv6.enabled=true", "--helm-set", "--enable-ipv6-ndp=true")
|
||||
_, _ = cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
// HMMM, if we want to run workloads on the control planes (todo)
|
||||
if config.ControlPlane {
|
||||
for _, node := range nodes {
|
||||
@@ -146,52 +123,11 @@ func (config *TestConfig) CreateKind() error {
|
||||
}
|
||||
}
|
||||
|
||||
ipv4Pool := defaultIPv4Pool
|
||||
ipv6Pool := defaultIPv6Pool
|
||||
|
||||
var ipv4Net *net.IPNet
|
||||
var ipv6Net *net.IPNet
|
||||
|
||||
links, err := netlink.LinkList()
|
||||
if err != nil {
|
||||
return fmt.Errorf("netlink: failed to list links: %w", err)
|
||||
}
|
||||
|
||||
for _, link := range links {
|
||||
if strings.Contains(link.Attrs().Name, config.DockerNIC) {
|
||||
_, ipv4Net, err = getNetwork(link, netlink.FAMILY_V4)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get IPv4 network: %w", err)
|
||||
}
|
||||
if ipv4Net == nil {
|
||||
return fmt.Errorf("failed to find IPv4 network on the docker interface")
|
||||
}
|
||||
|
||||
ipv4Pool = generateRange(ipv4Net)
|
||||
|
||||
_, ipv6Net, err = getNetwork(link, netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get IPv6 network: %w", err)
|
||||
}
|
||||
if ipv6Net == nil {
|
||||
return fmt.Errorf("failed to find IPv6 network on the docker interface")
|
||||
}
|
||||
|
||||
ipv6Pool = generateRange(ipv6Net)
|
||||
}
|
||||
}
|
||||
|
||||
globalRange := ipv4Pool
|
||||
globalRange := "172.18.100.10-172.18.100.30"
|
||||
if config.IPv6 {
|
||||
globalRange = ipv6Pool
|
||||
globalRange = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
|
||||
}
|
||||
|
||||
if config.DualStack {
|
||||
globalRange = fmt.Sprintf("%s,%s", ipv4Pool, ipv6Pool)
|
||||
}
|
||||
|
||||
slog.Infof("Cloud provider's range: %s", globalRange)
|
||||
|
||||
cmd := exec.Command("kubectl", "create", "configmap", "--namespace", "kube-system", "kubevip", "--from-literal", "range-global="+globalRange)
|
||||
if _, err := cmd.CombinedOutput(); err != nil {
|
||||
return err
|
||||
@@ -204,51 +140,14 @@ func (config *TestConfig) CreateKind() error {
|
||||
if _, err := cmd.CombinedOutput(); err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Infof("💤 sleeping for a few seconds to let controllers start")
|
||||
log.Infof("💤 sleeping for a few seconds to let controllers start")
|
||||
time.Sleep(time.Second * 5)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNetwork(link netlink.Link, family int) (*net.IP, *net.IPNet, error) {
|
||||
addrs, err := netlink.AddrList(link, family)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("netlink: failed to get addresses for link %q: %w", link.Attrs().Name, err)
|
||||
}
|
||||
if len(addrs) > 0 {
|
||||
for _, a := range addrs {
|
||||
if a.Scope == int(netlink.SCOPE_UNIVERSE) {
|
||||
ip, cidr, err := net.ParseCIDR(a.IPNet.String())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse CIDR: %w", err)
|
||||
}
|
||||
return &ip, cidr, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func generateRange(network *net.IPNet) string {
|
||||
if network.IP.To4() != nil {
|
||||
parts := strings.Split(network.IP.String(), ".")
|
||||
p1 := fmt.Sprintf("%s.%s.100.10", parts[0], parts[1])
|
||||
p2 := fmt.Sprintf("%s.%s.100.100", parts[0], parts[1])
|
||||
return fmt.Sprintf("%s-%s", p1, p2)
|
||||
}
|
||||
|
||||
if network.IP.To16() != nil {
|
||||
parts := strings.Split(network.IP.String(), ":")
|
||||
p1 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0010", parts[0], parts[1], parts[2], parts[3])
|
||||
p2 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0100", parts[0], parts[1], parts[2], parts[3])
|
||||
return fmt.Sprintf("%s-%s", p1, p2)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func DeleteKind() error {
|
||||
slog.Info("🧽 deleting Kind cluster")
|
||||
func deleteKind() error {
|
||||
log.Info("🧽 deleting Kind cluster")
|
||||
return provider.Delete("services", "")
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
package deployment
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gookit/slog"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
@@ -17,19 +17,18 @@ import (
|
||||
)
|
||||
|
||||
// service defines the settings for a new service
|
||||
type Service struct {
|
||||
name string
|
||||
egress bool // enable egress
|
||||
egressInternal bool // enable Internal egress
|
||||
egressIPv6 bool // egress should be IPv6
|
||||
policyLocal bool // set the policy to local pods
|
||||
testHTTP bool
|
||||
testDualstack bool // test dualstack loadbalancer services
|
||||
timeout int // how long to wait for the service to be created
|
||||
type service struct {
|
||||
name string
|
||||
egress bool // enable egress
|
||||
egressIPv6 bool // egress should be IPv6
|
||||
policyLocal bool // set the policy to local pods
|
||||
testHTTP bool
|
||||
testDualstack bool // test dualstack loadbalancer services
|
||||
timeout int // how long to wait for the service to be created
|
||||
}
|
||||
|
||||
type Deployment struct {
|
||||
replicas int32
|
||||
type deployment struct {
|
||||
replicas int
|
||||
server bool
|
||||
client bool
|
||||
address string
|
||||
@@ -37,7 +36,7 @@ type Deployment struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clientset, imagepath string) error {
|
||||
func (d *deployment) createKVDs(ctx context.Context, clientset *kubernetes.Clientset, imagepath string) error {
|
||||
ds := appsv1.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "kube-vip-ds",
|
||||
@@ -72,7 +71,7 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
|
||||
Value: "true",
|
||||
},
|
||||
{
|
||||
Name: "vip_subnet",
|
||||
Name: "vip_cidr",
|
||||
Value: "32",
|
||||
},
|
||||
{
|
||||
@@ -93,7 +92,7 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
|
||||
},
|
||||
{
|
||||
Name: "vip_loglevel",
|
||||
Value: "-4",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "egress_withnftables",
|
||||
@@ -133,8 +132,8 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
|
||||
return nil
|
||||
|
||||
}
|
||||
func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
|
||||
replicas := d.replicas
|
||||
func (d *deployment) createDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
|
||||
replicas := int32(d.replicas)
|
||||
deployment := &appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: d.name,
|
||||
@@ -156,7 +155,7 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "kube-vip-web",
|
||||
Image: "docker.io/plndr/e2e:0.0.1",
|
||||
Image: "plndr/e2e:0.0.1",
|
||||
Ports: []v1.ContainerPort{
|
||||
{
|
||||
Name: "http",
|
||||
@@ -205,11 +204,11 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Infof("📝 created deployment [%s]", result.GetObjectMeta().GetName())
|
||||
log.Infof("📝 created deployment [%s]", result.GetObjectMeta().GetName())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clientset) (currentLeader string, loadBalancerAddresses []string, err error) {
|
||||
func (s *service) createService(ctx context.Context, clientset *kubernetes.Clientset) (currentLeader string, loadBalancerAddresses []string, err error) {
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: s.name,
|
||||
@@ -239,20 +238,10 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
|
||||
"kube-vip.io/egress": "true",
|
||||
}
|
||||
}
|
||||
|
||||
if s.egressInternal {
|
||||
svc.Annotations["kube-vip.io/egress-internal"] = "true"
|
||||
}
|
||||
|
||||
if s.egressIPv6 {
|
||||
svc.Annotations["kube-vip.io/egress-ipv6"] = "true"
|
||||
}
|
||||
|
||||
//svc.Annotations["kube-vip.io/egress-denied-networks"] = "172.18.0.0/24"
|
||||
//svc.Annotations["kube-vip.io/egress-allowed-networks"] = "172.18.0.0/24"
|
||||
|
||||
//svc.Annotations["kube-vip.io/egress-allowed-networks"] = "192.168.0.0/24, 172.18.0.0/24"
|
||||
|
||||
if s.policyLocal {
|
||||
svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
|
||||
}
|
||||
@@ -263,11 +252,11 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
|
||||
}
|
||||
ipv4VIP, err := generateIPv4VIP()
|
||||
if err != nil {
|
||||
slog.Fatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
ipv6VIP, err := generateIPv6VIP()
|
||||
if err != nil {
|
||||
slog.Fatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
svc.Annotations["kube-vip.io/loadbalancerIPs"] = fmt.Sprintf("%s,%s", ipv4VIP, ipv6VIP)
|
||||
svc.Labels["implementation"] = "kube-vip"
|
||||
@@ -276,19 +265,19 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
|
||||
svc.Spec.IPFamilyPolicy = &ipfPolicy
|
||||
}
|
||||
|
||||
slog.Infof("🌍 creating service [%s]", svc.Name)
|
||||
log.Infof("🌍 creating service [%s]", svc.Name)
|
||||
_, err = clientset.CoreV1().Services(v1.NamespaceDefault).Create(ctx, svc, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
slog.Fatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
|
||||
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
slog.Fatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
ch := rw.ResultChan()
|
||||
go func() {
|
||||
@@ -303,16 +292,17 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
slog.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
log.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if svc.Name == s.name {
|
||||
if len(svc.Status.LoadBalancer.Ingress) != 0 {
|
||||
for _, ingress := range svc.Status.LoadBalancer.Ingress {
|
||||
loadBalancerAddresses = append(loadBalancerAddresses, ingress.IP)
|
||||
}
|
||||
slog.Infof("🔎 found load balancer addresses [%s] on node [%s]", loadBalancerAddresses, svc.Annotations["kube-vip.io/vipHost"])
|
||||
log.Infof("🔎 found load balancer addresses [%s] on node [%s]", loadBalancerAddresses, svc.Annotations["kube-vip.io/vipHost"])
|
||||
ready = true
|
||||
currentLeader = svc.Annotations["kube-vip.io/vipHost"]
|
||||
}
|
||||
@@ -328,7 +318,6 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
|
||||
for _, lbAddress := range loadBalancerAddresses {
|
||||
err = httpTest(lbAddress)
|
||||
if err != nil {
|
||||
slog.Infof("web retrieval err: %s", err.Error())
|
||||
return "", nil, fmt.Errorf("web retrieval timeout ")
|
||||
|
||||
}
|
||||
725
testing/e2e/services/services.go
Normal file
725
testing/e2e/services/services.go
Normal file
@@ -0,0 +1,725 @@
|
||||
//nolint:govet
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
)
|
||||
|
||||
// Methodology
|
||||
|
||||
// 1. Create a deployment
|
||||
// 2. Expose the deployment
|
||||
func (config *testConfig) startServiceTest(ctx context.Context, clientset *kubernetes.Clientset) {
|
||||
nodeTolerate := os.Getenv("NODE_TOLERATE")
|
||||
|
||||
d := "kube-vip-deploy"
|
||||
s := "kube-vip-service"
|
||||
l := "kube-vip-deploy-leader"
|
||||
|
||||
if !config.ignoreSimple {
|
||||
// Simple Deployment test
|
||||
log.Infof("🧪 ---> simple deployment <---")
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 2,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
svc := service{
|
||||
name: s,
|
||||
testHTTP: true,
|
||||
timeout: 10,
|
||||
}
|
||||
_, _, err = svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
} else {
|
||||
config.successCounter++
|
||||
}
|
||||
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !config.ignoreDeployments {
|
||||
// Multiple deployment tests
|
||||
log.Infof("🧪 ---> multiple deployments <---")
|
||||
deploy := deployment{
|
||||
name: l,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 2,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for i := 1; i < 5; i++ {
|
||||
svc := service{
|
||||
name: fmt.Sprintf("%s-%d", s, i),
|
||||
testHTTP: true,
|
||||
timeout: 30,
|
||||
}
|
||||
_, _, err = svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
config.successCounter++
|
||||
}
|
||||
for i := 1; i < 5; i++ {
|
||||
log.Infof("🧹 deleting service [%s]", fmt.Sprintf("%s-%d", s, i))
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, fmt.Sprintf("%s-%d", s, i), metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
log.Infof("🧹 deleting deployment [%s]", d)
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, l, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !config.ignoreLeaderFailover {
|
||||
// Failover tests
|
||||
log.Infof("🧪 ---> leader failover deployment (local policy) <---")
|
||||
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 2,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
svc := service{
|
||||
name: s,
|
||||
egress: false,
|
||||
policyLocal: true,
|
||||
testHTTP: true,
|
||||
timeout: 180,
|
||||
}
|
||||
leader, lbAddresses, err := svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
lbAddress := lbAddresses[0]
|
||||
|
||||
err = leaderFailover(ctx, &s, &leader, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
} else {
|
||||
config.successCounter++
|
||||
}
|
||||
|
||||
// Get all addresses on all nodes
|
||||
nodes, err := getAddressesOnNodes()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
// Make sure we don't exist in two places
|
||||
err = checkNodesForDuplicateAddresses(nodes, lbAddress)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !config.ignoreLeaderActive {
|
||||
// pod Failover tests
|
||||
log.Infof("🧪 ---> active pod failover deployment (local policy) <---")
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 1,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
svc := service{
|
||||
name: s,
|
||||
policyLocal: true,
|
||||
testHTTP: true,
|
||||
timeout: 30,
|
||||
}
|
||||
leader, _, err := svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
err = podFailover(ctx, &s, &leader, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
} else {
|
||||
config.successCounter++
|
||||
}
|
||||
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !config.ignoreLocalDeploy {
|
||||
// Multiple deployment tests
|
||||
log.Infof("🧪 ---> multiple deployments (local policy) <---")
|
||||
timeout := 30
|
||||
deploy := deployment{
|
||||
name: l,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 2,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for i := 1; i < 5; i++ {
|
||||
svc := service{
|
||||
policyLocal: true,
|
||||
name: fmt.Sprintf("%s-%d", s, i),
|
||||
testHTTP: true,
|
||||
timeout: timeout,
|
||||
}
|
||||
_, lbAddresses, err := svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
lbAddress := lbAddresses[0]
|
||||
|
||||
config.successCounter++
|
||||
nodes, err := getAddressesOnNodes()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
err = checkNodesForDuplicateAddresses(nodes, lbAddress)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := 1; i < 5; i++ {
|
||||
log.Infof("🧹 deleting service [%s]", fmt.Sprintf("%s-%d", s, i))
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, fmt.Sprintf("%s-%d", s, i), metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
log.Infof("🧹 deleting deployment [%s]", d)
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, l, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !config.ignoreEgress {
|
||||
// pod Failover tests
|
||||
log.Infof("🧪 ---> egress IP re-write (local policy) <---")
|
||||
var egress string
|
||||
var found bool
|
||||
timeout := 30
|
||||
// Set up a local listener
|
||||
go func() {
|
||||
found = tcpServer(false, &egress, timeout)
|
||||
}()
|
||||
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 1,
|
||||
client: true,
|
||||
}
|
||||
|
||||
// Find this machines IP address
|
||||
deploy.address = GetLocalIPv4()
|
||||
if deploy.address == "" {
|
||||
log.Fatalf("Unable to detect local IP address")
|
||||
}
|
||||
log.Infof("📠 found local address [%s]", deploy.address)
|
||||
// Create a deployment that connects back to this machines IP address
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
svc := service{
|
||||
policyLocal: true,
|
||||
name: s,
|
||||
egress: true,
|
||||
testHTTP: false,
|
||||
timeout: 30,
|
||||
}
|
||||
|
||||
_, lbAddresses, err := svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
egress = lbAddresses[0]
|
||||
|
||||
for i := 1; i < 10; i++ {
|
||||
if found {
|
||||
log.Infof("🕵️ egress has correct IP address")
|
||||
config.successCounter++
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second * 1)
|
||||
}
|
||||
|
||||
if !found {
|
||||
log.Error("😱 No traffic found from loadbalancer address ")
|
||||
}
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !config.ignoreEgressIPv6 {
|
||||
// pod Failover tests
|
||||
log.Infof("🧪 ---> egress IP re-write IPv6 (local policy) <---")
|
||||
var egress string
|
||||
var found bool
|
||||
timeout := 60
|
||||
// Set up a local listener
|
||||
go func() {
|
||||
found = tcpServer(true, &egress, timeout)
|
||||
}()
|
||||
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 1,
|
||||
client: true,
|
||||
}
|
||||
|
||||
// Find this machines IP address
|
||||
deploy.address = GetLocalIPv6()
|
||||
if deploy.address == "" {
|
||||
log.Fatalf("Unable to detect local IP address")
|
||||
}
|
||||
log.Infof("📠 found local address [%s]", deploy.address)
|
||||
// Create a deployment that connects back to this machines IP address
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
svc := service{
|
||||
policyLocal: true,
|
||||
name: s,
|
||||
egress: true,
|
||||
egressIPv6: true,
|
||||
testHTTP: false,
|
||||
timeout: timeout,
|
||||
testDualstack: true,
|
||||
}
|
||||
|
||||
_, lbAddresses, err := svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for x := range lbAddresses {
|
||||
ip := net.ParseIP(lbAddresses[x])
|
||||
// if ip == nil {
|
||||
// return errors.New("invalid address")
|
||||
// }
|
||||
if ip.To4() == nil {
|
||||
// use brackets for IPv6 address
|
||||
egress = lbAddresses[x]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for i := 1; i < timeout; i++ {
|
||||
if found {
|
||||
log.Infof("🕵️ egress has correct IP address")
|
||||
config.successCounter++
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second * 1)
|
||||
}
|
||||
|
||||
if !found {
|
||||
log.Error("😱 No traffic found from loadbalancer address ")
|
||||
}
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !config.ignoreDualStack {
|
||||
// Dualstack loadbalancer test
|
||||
log.Infof("🧪 ---> testing dualstack loadbalancer service <---")
|
||||
deploy := deployment{
|
||||
name: d,
|
||||
nodeAffinity: nodeTolerate,
|
||||
replicas: 2,
|
||||
server: true,
|
||||
}
|
||||
err := deploy.createDeployment(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
svc := service{
|
||||
name: s,
|
||||
testHTTP: true,
|
||||
testDualstack: true,
|
||||
timeout: 30,
|
||||
}
|
||||
_, _, err = svc.createService(ctx, clientset)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
} else {
|
||||
config.successCounter++
|
||||
}
|
||||
|
||||
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
|
||||
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.Infof("🏆 Testing Complete [%d] passed", config.successCounter)
|
||||
}
|
||||
|
||||
func httpTest(address string) error {
|
||||
log.Infof("🕷️ testing HTTP request against [%s]", address)
|
||||
Client := http.Client{
|
||||
Timeout: 2 * time.Second,
|
||||
}
|
||||
ip := net.ParseIP(address)
|
||||
if ip == nil {
|
||||
return errors.New("invalid address")
|
||||
}
|
||||
if ip.To4() == nil {
|
||||
// use brackets for IPv6 address
|
||||
address = fmt.Sprintf("[%s]", address)
|
||||
}
|
||||
var err error
|
||||
for i := 0; i < 5; i++ {
|
||||
var r *http.Response
|
||||
//nolint
|
||||
r, err = Client.Get(fmt.Sprintf("http://%s", address)) //nolint
|
||||
|
||||
if err == nil {
|
||||
log.Infof("🕸️ successfully retrieved web data in [%ds]", i)
|
||||
r.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func leaderFailover(ctx context.Context, name, leaderNode *string, clientset *kubernetes.Clientset) error {
|
||||
go func() {
|
||||
log.Infof("💀 killing leader five times")
|
||||
for i := 0; i < 5; i++ {
|
||||
p, err := clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for x := range p.Items {
|
||||
if p.Items[x].Spec.NodeName == *leaderNode {
|
||||
if p.Items[x].Spec.Containers[0].Name == "kube-vip" {
|
||||
err = clientset.CoreV1().Pods("kube-system").Delete(ctx, p.Items[x].Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Infof("🔪 leader pod [%s] has been deleted", p.Items[x].Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second * 5)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Infof("👀 service [%s] for updates", *name)
|
||||
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ch := rw.ResultChan()
|
||||
|
||||
go func() {
|
||||
time.Sleep(time.Second * 30)
|
||||
rw.Stop()
|
||||
}()
|
||||
|
||||
// Used for tracking an active endpoint / pod
|
||||
for event := range ch {
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added:
|
||||
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
log.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if svc.Name == *name {
|
||||
if len(svc.Status.LoadBalancer.Ingress) != 0 {
|
||||
log.Infof("🔎 found load balancer address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
|
||||
}
|
||||
}
|
||||
case watch.Modified:
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
log.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if svc.Name == *name {
|
||||
if len(svc.Status.LoadBalancer.Ingress) != 0 {
|
||||
log.Infof("🔍 updated with address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
|
||||
err = httpTest(svc.Status.LoadBalancer.Ingress[0].IP)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*leaderNode = svc.Annotations["kube-vip.io/vipHost"]
|
||||
}
|
||||
}
|
||||
default:
|
||||
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func podFailover(ctx context.Context, name, leaderNode *string, clientset *kubernetes.Clientset) error {
|
||||
go func() {
|
||||
log.Infof("💀 killing active pod five times")
|
||||
for i := 0; i < 5; i++ {
|
||||
p, err := clientset.CoreV1().Pods(v1.NamespaceDefault).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for x := range p.Items {
|
||||
if p.Items[x].Spec.NodeName == *leaderNode {
|
||||
if p.Items[x].Spec.Containers[0].Name == "kube-vip-web" {
|
||||
found = true
|
||||
err = clientset.CoreV1().Pods(v1.NamespaceDefault).Delete(ctx, p.Items[x].Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Infof("🔪 active pod [%s] on [%s] has been deleted", p.Items[x].Name, p.Items[x].Spec.NodeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
log.Warnf("😱 No Pod found on [%s]", *leaderNode)
|
||||
}
|
||||
time.Sleep(time.Second * 5)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Infof("👀 service [%s] for updates", *name)
|
||||
|
||||
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
|
||||
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ch := rw.ResultChan()
|
||||
|
||||
go func() {
|
||||
time.Sleep(time.Second * 30)
|
||||
rw.Stop()
|
||||
}()
|
||||
|
||||
// Used for tracking an active endpoint / pod
|
||||
for event := range ch {
|
||||
// We need to inspect the event and get ResourceVersion out of it
|
||||
switch event.Type {
|
||||
case watch.Added:
|
||||
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
log.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if svc.Name == *name {
|
||||
if len(svc.Status.LoadBalancer.Ingress) != 0 {
|
||||
log.Infof("🔎 found load balancer address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
|
||||
}
|
||||
}
|
||||
case watch.Modified:
|
||||
svc, ok := event.Object.(*v1.Service)
|
||||
if !ok {
|
||||
log.Fatalf("unable to parse Kubernetes services from API watcher")
|
||||
}
|
||||
if svc.Name == *name {
|
||||
if len(svc.Status.LoadBalancer.Ingress) != 0 {
|
||||
log.Infof("🔍 updated with address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
|
||||
err = httpTest(svc.Status.LoadBalancer.Ingress[0].IP)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
*leaderNode = svc.Annotations["kube-vip.io/vipHost"]
|
||||
}
|
||||
}
|
||||
default:
|
||||
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tcpServer(IPv6 bool, egressAddress *string, timeout int) bool {
|
||||
var listen net.Listener
|
||||
var err error
|
||||
if !IPv6 {
|
||||
listen, err = net.Listen("tcp", ":12345") //nolint
|
||||
} else {
|
||||
listen, err = net.Listen("tcp6", ":12346") //nolint
|
||||
|
||||
}
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
// close listener
|
||||
go func() {
|
||||
time.Sleep(time.Second * time.Duration(timeout))
|
||||
listen.Close()
|
||||
}()
|
||||
for {
|
||||
conn, err := listen.Accept()
|
||||
if err != nil {
|
||||
return false
|
||||
// log.Fatal(err)
|
||||
}
|
||||
remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
|
||||
// If it is IPv6 expand the address
|
||||
// if IPv6 {
|
||||
// addr, _ := netip.ParseAddr(remoteAddress)
|
||||
// remoteAddress = addr.StringExpanded()
|
||||
// }
|
||||
|
||||
if remoteAddress == *egressAddress {
|
||||
log.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
|
||||
return true
|
||||
}
|
||||
log.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
|
||||
go handleRequest(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func handleRequest(conn net.Conn) {
|
||||
// incoming request
|
||||
buffer := make([]byte, 1024)
|
||||
_, err := conn.Read(buffer)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
// write data to response
|
||||
time := time.Now().Format(time.ANSIC)
|
||||
responseStr := fmt.Sprintf("Your message is: %v. Received time: %v", string(buffer[:]), time)
|
||||
_, err = conn.Write([]byte(responseStr))
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
// close conn
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
func GetLocalIPv4() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, address := range addrs {
|
||||
// check the address type and if it is not a loopback the display it
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To4() != nil {
|
||||
return ipnet.IP.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func GetLocalIPv6() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, address := range addrs {
|
||||
// check the address type and if it is not a loopback the display it
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() == nil && !ipnet.IP.IsLinkLocalUnicast() {
|
||||
if ipnet.IP.To16() != nil {
|
||||
return ipnet.IP.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user