Compare commits

..

1 Commits

Author SHA1 Message Date
Marcel Fest
37e3446c5a Use the new release version in Makefile
Signed-off-by: Marcel Fest <marcel.fest@telekom.de>
2024-11-11 08:44:45 +01:00
110 changed files with 5301 additions and 9406 deletions

View File

@@ -26,6 +26,6 @@ jobs:
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.20.4
uses: anchore/sbom-action@v0.17.5
with:
format: cyclonedx-json

View File

@@ -1,51 +0,0 @@
name: For each PR
on:
pull_request:
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
strategy:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests

View File

@@ -1,16 +1,18 @@
name: For each commit
name: For each commit and PR
on:
push:
pull_request:
env:
GO_VERSION: "1.21"
jobs:
validation:
runs-on: ubuntu-latest
name: Checks and linters
steps:
- name: Init
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
run: sudo apt-get update && sudo apt-get install -y build-essential golint && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Install golangci-lint
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
@@ -35,14 +37,48 @@ jobs:
name: Integration tests
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
run: make integration-tests
e2e-tests:
runs-on: ubuntu-latest
name: E2E ARP tests
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run Control plane tests
run: make e2e-tests
- name: Run Control plane tests v1.29.0 onwards
run: make e2e-tests129
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: make integration-tests
run: DOCKERTAG=action make service-tests
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan
@@ -54,9 +90,10 @@ jobs:
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: "plndr/kube-vip:action"
format: "table"
exit-code: "1"
image-ref: 'plndr/kube-vip:action'
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: "os,library"
severity: "CRITICAL,HIGH"
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'

View File

@@ -27,12 +27,6 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Metadata
uses: docker/metadata-action@v5.7.0
id: metadata
with:
labels: |
org.opencontainers.image.documentation=https://kube-vip.io/docs/
- name: Build and push main branch
id: docker_build
uses: docker/build-push-action@v6
@@ -40,7 +34,6 @@ jobs:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
push: ${{ github.event_name != 'pull_request' }}
labels: ${{ steps.metadata.outputs.labels }}
tags: >-
plndr/kube-vip:${{ github.ref_name }},
plndr/kube-vip:latest,
@@ -54,7 +47,6 @@ jobs:
file: Dockerfile_iptables
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
push: ${{ github.event_name != 'pull_request' }}
labels: ${{ steps.metadata.outputs.labels }}
tags: >-
plndr/kube-vip-iptables:${{ github.ref_name }},
plndr/kube-vip-iptables:latest,

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.5-alpine3.22 as dev
FROM golang:1.23.2-alpine3.20 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.5-alpine3.22 as dev
FROM golang:1.23.2-alpine3.20 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.22.1
FROM alpine:3.20.3
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.0.0
VERSION := v0.8.6
BUILD := `git rev-parse HEAD`
@@ -115,7 +115,7 @@ manifests:
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@-rm ./kube-vip
manifest-test:
@@ -133,21 +133,13 @@ integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests:
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
e2e-tests129:
V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
service-tests:
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/e2e/services -Services
trivy: dockerx86ActionIPTables
docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
@@ -159,21 +151,3 @@ trivy: dockerx86ActionIPTables
--severity 'CRITICAL,HIGH' \
$(REPOSITORY)/$(TARGET):action
kind-quick:
echo "Standing up your cluster"
kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
mkdir -p bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin

View File

@@ -4,7 +4,7 @@ High Availability and Load-Balancing
![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
## Overview
Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
@@ -32,7 +32,6 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
- Service LoadBalancer address pools per namespace or global
- Service LoadBalancer address via (existing network DHCP)
- Service LoadBalancer address exposure to gateway via UPNP
- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod.
- ... manifest generation, vendor API integrations and many more...
## Why?

View File

@@ -4,9 +4,8 @@ import (
"fmt"
"os"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@@ -23,7 +22,7 @@ func init() {
var kubeKubeadm = &cobra.Command{
Use: "kubeadm",
Short: "Kubeadm functions",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
_ = cmd.Help()
// TODO - A load of text detailing what's actually happening
},
@@ -33,35 +32,32 @@ var kubeKubeadmInit = &cobra.Command{
Use: "init",
Short: "kube-vip init",
Long: "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalf("Error parsing environment from config: %v", err)
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
_ = cmd.Help()
log.Error("No interface is specified for kube-vip to bind to")
return
log.Fatalln("No interface is specified for kube-vip to bind to")
}
if initConfig.VIP == "" && initConfig.Address == "" {
_ = cmd.Help()
log.Error("No address is specified for kube-vip to expose services on")
return
log.Fatalln("No address is specified for kube-vip to expose services on")
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
log.Fatalln(err)
}
}
@@ -73,40 +69,37 @@ var kubeKubeadmInit = &cobra.Command{
var kubeKubeadmJoin = &cobra.Command{
Use: "join",
Short: "kube-vip join",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalf("Error parsing environment from config: %v", err)
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
_ = cmd.Help()
log.Error("No interface is specified for kube-vip to bind to")
return
log.Fatalln("No interface is specified for kube-vip to bind to")
}
if initConfig.VIP == "" && initConfig.Address == "" {
_ = cmd.Help()
log.Error("No address is specified for kube-vip to expose services on")
return
log.Fatalln("No address is specified for kube-vip to expose services on")
}
if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
log.Error("kubeConfig not found", "Path", kubeConfigPath)
return
log.Fatalf("Unable to find file [%s]", kubeConfigPath)
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
log.Fatalln(err)
}
}

View File

@@ -3,25 +3,23 @@ package cmd
import (
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
// manifests will eventually deprecate the kubeadm set of subcommands
// manifests will be used to generate:
// - Pod spec manifest, mainly used for a static pod (kubeadm)
// - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
// - RBAC manifest, used to generate the RBAC permissions for kube-vip
var taint, role, rolebinding bool
// var inCluster bool
var taint bool
func init() {
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
kubeManifest.AddCommand(kubeManifestPod)
kubeManifest.AddCommand(kubeManifestDaemon)
@@ -31,7 +29,7 @@ func init() {
var kubeManifest = &cobra.Command{
Use: "manifest",
Short: "Manifest functions",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
_ = cmd.Help()
// TODO - A load of text detailing what's actually happening
},
@@ -40,29 +38,28 @@ var kubeManifest = &cobra.Command{
var kubeManifestPod = &cobra.Command{
Use: "pod",
Short: "Generate a Pod Manifest",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
var err error
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalf("Error parsing environment from config: %v", err)
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("No address is specified for kube-vip to expose services on")
return
log.Fatalln("No address is specified for kube-vip to expose services on")
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
log.Fatalln(err)
}
}
@@ -74,28 +71,30 @@ var kubeManifestPod = &cobra.Command{
var kubeManifestDaemon = &cobra.Command{
Use: "daemonset",
Short: "Generate a Daemonset Manifest",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
var err error
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalf("error parsing environment config: %v", err)
}
// TODO - check for certain things VIP/interfaces
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("No address is specified for kube-vip to expose services on")
return
log.Fatalln("No address is specified for kube-vip to expose services on")
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
log.Fatalln(err)
}
}
@@ -107,44 +106,33 @@ var kubeManifestDaemon = &cobra.Command{
var kubeManifestRbac = &cobra.Command{
Use: "rbac",
Short: "Generate an RBAC Manifest",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
var err error
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalf("Error parsing environment from config: %v", err)
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("No address is specified for kube-vip to expose services on")
return
log.Fatalln("No address is specified for kube-vip to expose services on")
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
log.Fatalln(err)
}
}
saCfg := kubevip.GenerateSA(&initConfig)
roleCfg := kubevip.GenerateRole(&initConfig, role)
if role {
rolebinding = true
}
roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
// Output the YAML manifests to stdout
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
cfg := kubevip.GenerateSA()
b, _ := yaml.Marshal(cfg)
fmt.Println(string(b)) // output manifest to stdout
},
}

View File

@@ -9,14 +9,13 @@ import (
"strings"
"time"
log "log/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/manager"
"github.com/kube-vip/kube-vip/pkg/vip"
@@ -28,6 +27,12 @@ var inCluster bool
// ConfigMap name within a Kubernetes cluster
var configMap string
// Configure the level of logging
var logLevel uint32
// Provider Config
var providerConfig string
// Points to a kubernetes configuration file
var kubeConfigPath string
@@ -53,19 +58,20 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc..")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
// VIP flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address. Default to 32 for IPv4 and 128 for IPv6") // todo: deprecate
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "Port for the VIP")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
@@ -78,6 +84,13 @@ func init() {
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
// Equinix Metal flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
// BGP flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
@@ -93,23 +106,18 @@ func init() {
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
// Manage logging
kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
// Service flags
kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
// Routing Table flags
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", 0, "The type of route that will be added to the routing table")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
@@ -128,9 +136,8 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
// Prometheus HTTP Server
kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
@@ -142,6 +149,7 @@ func init() {
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
// Kubernetes client specific flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
kubeVipCmd.AddCommand(kubeKubeadm)
@@ -150,9 +158,6 @@ func init() {
kubeVipCmd.AddCommand(kubeVipSample)
kubeVipCmd.AddCommand(kubeVipService)
kubeVipCmd.AddCommand(kubeVipVersion)
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
}
// Execute - starts the command parsing process
@@ -166,7 +171,7 @@ func Execute() {
var kubeVipVersion = &cobra.Command{
Use: "version",
Short: "Version and Release information about the Kubernetes Virtual IP Server",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("Kube-VIP Release Information\n")
fmt.Printf("Version: %s\n", Release.Version)
fmt.Printf("Build: %s\n", Release.Build)
@@ -176,7 +181,7 @@ var kubeVipVersion = &cobra.Command{
var kubeVipSample = &cobra.Command{
Use: "sample",
Short: "Generate a Sample configuration",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
_ = cmd.Help()
},
}
@@ -184,26 +189,18 @@ var kubeVipSample = &cobra.Command{
var kubeVipService = &cobra.Command{
Use: "service",
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing env", "err", err)
return
log.Fatalln(err)
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
initConfig.RoutingProtocol = unix.RTN_UNICAST
}
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
if err := initConfig.CheckInterface(); err != nil {
log.Error("checking interface", "err", err)
return
log.Fatalln(err)
}
// User Environment variables as an option to make manifest clearer
@@ -213,26 +210,23 @@ var kubeVipService = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating CIDR", "err", err)
return
log.Fatalln(err)
}
}
// Define the new service manager
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
log.Fatalf("%v", err)
}
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start()
if err != nil {
log.Error("manager start", "err", err)
return
log.Fatalf("%v", err)
}
},
}
@@ -240,33 +234,27 @@ var kubeVipService = &cobra.Command{
var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
Run: func(cmd *cobra.Command, args []string) {
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
log.Fatalln(err)
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
initConfig.RoutingProtocol = unix.RTN_UNICAST
}
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("No interface is specified for kube-vip to bind to")
return
log.Fatalln(err)
}
}
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(initConfig.Logging))
// Welome messages
log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
log.Infof("Starting kube-vip.io [%s]", Release.Version)
log.Debugf("Build kube-vip.io [%s]", Release.Build)
// start prometheus server
if initConfig.PrometheusHTTPServer != "" {
@@ -294,22 +282,19 @@ var kubeVipManager = &cobra.Command{
}
// Provide configuration to output/logging
log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
log.Infof("namespace [%s], Mode: [%s], Features(s): Control Plane:[%t], Services:[%t]", initConfig.Namespace, mode, initConfig.EnableControlPlane, initConfig.EnableServices)
// End if nothing is enabled
if !initConfig.EnableServices && !initConfig.EnableControlPlane {
log.Error("no features are enabled")
return
log.Fatalln("no features are enabled")
}
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
log.Error("auto subnet discovery cannot be used outside ARP mode")
return
if !initConfig.EnableARP && strings.Contains(initConfig.VIPCIDR, kubevip.Auto) {
log.Fatalln("auto subnet discovery cannot be used outside ARP mode")
}
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
log.Error("auto subnet discovery cannot be used if VIP address was provided")
return
if strings.Contains(initConfig.VIPCIDR, kubevip.Auto) && initConfig.Address != "" {
log.Fatalln("auto subnet discovery cannot be used if VIP address was provided")
}
// If we're using wireguard then all traffic goes through the wg0 interface
@@ -319,55 +304,48 @@ var kubeVipManager = &cobra.Command{
initConfig.Interface = "wg0"
}
log.Info("configuring Wireguard networking")
log.Infof("configuring Wireguard networking")
l, err := netlink.LinkByName(initConfig.Interface)
if err != nil {
if strings.Contains(err.Error(), "Link not found") {
log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
log.Warnf("interface \"%s\" doesn't exist, attempting to create wireguard interface", initConfig.Interface)
err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
if err != nil {
log.Error("adding link", "err", err)
return
log.Fatalln(err)
}
l, err = netlink.LinkByName(initConfig.Interface)
if err != nil {
log.Error("finding link", "err", err)
return
log.Fatalln(err)
}
}
}
err = netlink.LinkSetUp(l)
if err != nil {
log.Error("setting link UP", "err", err)
return
log.Fatalln(err)
}
} else { // if we're not using Wireguard then we'll need to use an actual interface
// Check if the interface needs auto-detecting
if initConfig.Interface == "" {
log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
log.Infof("No interface is specified for VIP in config, auto-detecting default Interface")
defaultIF, err := vip.GetDefaultGatewayInterface()
if err != nil {
_ = cmd.Help()
log.Error("detecting interface", "err", err)
return
log.Fatalf("unable to detect default interface -> [%v]", err)
}
initConfig.Interface = defaultIF.Name
log.Info("kube-vip bind", "interface", initConfig.Interface)
log.Infof("kube-vip will bind to interface [%s]", initConfig.Interface)
go func() {
if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
log.Error("interface monitor", "err", err)
return
log.Fatalf("crash: %s", err.Error())
}
}()
}
}
// Perform a check on the state of the interface
// Perform a check on th state of the interface
if err := initConfig.CheckInterface(); err != nil {
log.Error("checking interface", "err", err)
return
log.Fatalln(err)
}
// User Environment variables as an option to make manifest clearer
@@ -376,11 +354,22 @@ var kubeVipManager = &cobra.Command{
configMap = envConfigMap
}
// If Equinix Metal is enabled and there is a provider configuration passed
if initConfig.EnableMetal {
if providerConfig != "" {
providerAPI, providerProject, err := equinixmetal.GetPacketConfig(providerConfig)
if err != nil {
log.Fatalf("%v", err)
}
initConfig.MetalAPIKey = providerAPI
initConfig.MetalProject = providerProject
}
}
// Define the new service manager
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
log.Fatalf("configuring new Manager error -> %v", err)
}
prometheus.MustRegister(mgr.PrometheusCollector()...)
@@ -388,8 +377,7 @@ var kubeVipManager = &cobra.Command{
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start()
if err != nil {
log.Error("start manager", "err", err)
return
log.Fatalf("starting new Manager error -> %v", err)
}
},
}
@@ -404,7 +392,7 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
var err error
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`<html>
<head><title>kube-vip</title></head>
<body>
@@ -422,16 +410,15 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
go func() {
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("prometheus HTTP server", "err", err)
return
log.Fatalf("listen:%+s\n", err)
}
}()
log.Info("prometheus HTTP server started")
log.Printf("prometheus HTTP server started")
<-ctx.Done()
log.Info("prometheus HTTP server stopped")
log.Printf("prometheus HTTP server stopped")
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
@@ -439,8 +426,7 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
}()
if err = srv.Shutdown(ctxShutDown); err != nil {
log.Error("shutting down prometheus HTTP server", "err", err)
return
log.Fatalf("server Shutdown Failed:%+s", err)
}
if err == http.ErrServerClosed {
@@ -454,12 +440,9 @@ func GenerateCidrRange(address string) (string, error) {
addresses := strings.Split(address, ",")
for _, a := range addresses {
ip := net.ParseIP(a)
if ip == nil {
ips, err := net.LookupIP(a)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
ip = ips[0]
return "", fmt.Errorf("invalid IP address: %s from [%s]", a, address)
}
if ip.To4() != nil {

View File

@@ -60,7 +60,7 @@ func main() {
fmt.Println("error: ", err)
}
ServerConn.WriteTo(buf[0:n], addr)
ServerConn.WriteTo(buf[0:n])
}
}
}

220
go.mod
View File

@@ -1,165 +1,133 @@
module github.com/kube-vip/kube-vip
go 1.24.4
go 1.21
toolchain go1.21.3
require (
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.3.3+incompatible
github.com/cloudflare/ipvs v0.10.2
github.com/davecgh/go-spew v1.1.1
github.com/florianl/go-conntrack v0.4.0
github.com/google/go-cmp v0.7.0
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.5.8
github.com/golang/protobuf v1.5.4
github.com/google/go-cmp v0.6.0
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/insomniacslk/dhcp v0.0.0-20230731140434-0f9eb93a696c
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/mdlayher/ndp v1.0.1
github.com/onsi/ginkgo/v2 v2.19.0
github.com/onsi/gomega v1.33.1
github.com/osrg/gobgp/v3 v3.27.0
github.com/packethost/packngo v0.31.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.22.0
github.com/prometheus/client_golang v1.19.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
github.com/vishvananda/netlink v1.2.1-beta.2
go.etcd.io/etcd/api/v3 v3.5.13
go.etcd.io/etcd/client/pkg/v3 v3.5.13
go.etcd.io/etcd/client/v3 v3.5.13
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8
golang.org/x/sync v0.7.0
golang.org/x/sys v0.20.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.29.1
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.1
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/kind v0.22.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k-sone/critbitgo v1.4.0 // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/netlink v1.7.2 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mdlayher/socket v0.4.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.16.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/tj/go-spin v1.1.0 // indirect
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect
github.com/vishvananda/netns v0.0.4 // indirect
github.com/xlab/c-for-go v0.0.0-20230906092656-a1822f0a09c1 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.21.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/opt v0.1.4 // indirect
modernc.org/sortutil v1.2.1 // indirect
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
modernc.org/cc/v4 v4.1.0 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
)

884
go.sum

File diff suppressed because it is too large Load Diff

View File

@@ -1,149 +0,0 @@
package arp
import (
"context"
log "log/slog"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type Manager struct {
instances map[string]*Instance
config *kubevip.Config
}
type Instance struct {
network vip.Network
ndp *vip.NdpResponder
mu sync.Mutex
counter int
}
func NewManager(config *kubevip.Config) *Manager {
return &Manager{
instances: make(map[string]*Instance),
config: config,
}
}
func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
return &Instance{
ndp: ndp,
network: network,
counter: 1,
}
}
func (i *Instance) Name() string {
return i.network.ARPName()
}
func (m *Manager) Insert(instance *Instance) {
i, ok := m.instances[instance.Name()]
if !ok {
log.Info("inserting ARP/NDP instance", "name", instance.Name())
m.instances[instance.Name()] = instance
} else {
i.mu.Lock()
defer i.mu.Unlock()
i.counter++
}
}
func (m *Manager) Remove(instance *Instance) {
if i, ok := m.instances[instance.Name()]; ok {
i.mu.Lock()
defer i.mu.Unlock()
if i.counter > 1 {
i.counter--
} else {
log.Info("removing ARP/NDP instance", "name", instance.Name())
delete(m.instances, instance.Name())
}
}
}
func (m *Manager) Count(name string) int {
if i, ok := m.instances[name]; ok {
i.mu.Lock()
defer i.mu.Unlock()
return i.counter
}
return 0
}
func (m *Manager) StartAdvertisement(ctx context.Context) {
log.Info("Starting ARP/NDP advertisement")
for {
select {
case <-ctx.Done(): // if cancel() execute
return
default:
for _, instance := range m.instances {
if instance.counter > 0 {
ensureIPAndSendGratuitous(instance)
}
}
}
if m.config.ArpBroadcastRate < 500 {
log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
m.config.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
}
}
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
func ensureIPAndSendGratuitous(instance *Instance) {
iface := instance.network.Interface()
ipString := instance.network.IP()
// Check if IP is dadfailed
if instance.network.IsDADFAIL() {
log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
deleted, err := instance.network.DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
// if _, err := instance.network.AddIP(false); err != nil {
// log.Error("failed to recreate address", "IP", ipString, "interface", iface)
// }
}
}
// Ensure the address exists on the interface before attempting to ARP
// if instance.network.HasEndpoints() {
if added, err := instance.network.AddIP(true); err != nil {
log.Warn(err.Error())
} else if added {
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
}
// }
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if instance.ndp == nil {
log.Error("NDP responder was not created")
} else {
err := instance.ndp.SendGratuitous(ipString)
if err != nil {
log.Warn(err.Error())
}
}
} else {
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
err := vip.ARPSendGratuitous(ipString, iface)
if err != nil {
log.Warn(err.Error())
}
}
}

View File

@@ -4,19 +4,17 @@ import (
"fmt"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
log "github.com/sirupsen/logrus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
type Entry struct {
Addr string
Port uint16
IsLocal bool
Addr string
Port int
}
type Map map[Entry]bool
@@ -40,27 +38,28 @@ func (e *Entry) Check() bool {
switch {
case utils.FileExists(adminConfigPath):
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
log.Errorf("could not create k8s REST config for external file: %q: %v", adminConfigPath, err)
return false
}
default:
config, err = k8s.NewRestConfig("", true, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "err", err)
log.Errorf("could not create k8s REST config %v", err)
return false
}
}
client, err = k8s.NewClientset(config)
if err != nil {
log.Error("create k8s client", "err", err)
log.Errorf("failed to create k8s client: %v", err)
return false
}
_, err = client.DiscoveryClient.ServerVersion()
if err != nil {
log.Error("discover k8s version", "err", err)
log.Errorf("failed check k8s server version: %s", err)
return false
}
return true

View File

@@ -7,11 +7,9 @@ import (
"strconv"
"strings"
//nolint
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/golang/protobuf/ptypes" //nolint
"github.com/golang/protobuf/ptypes/any"
api "github.com/osrg/gobgp/v3/api"
"github.com/osrg/gobgp/v3/pkg/server"
"google.golang.org/protobuf/types/known/anypb"
)
// AddPeer will add peers to the BGP configuration
@@ -44,94 +42,36 @@ func (b *Server) AddPeer(peer Peer) (err error) {
},
}
if b.c.MpbgpNexthop != "" {
p.AfiSafis = []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
},
},
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
},
},
}
peer.setMpbgpOptions(b.c)
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
if err != nil {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
mask := "128"
address := ipv4Address
family := api.Family_AFI_IP
if vip.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
address = ipv6Address
family = api.Family_AFI_IP6
}
err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
DefinedSet: &api.DefinedSet{
DefinedType: api.DefinedType_NEIGHBOR,
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
List: []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
},
})
if err != nil {
return fmt.Errorf("failed to add defined set: %v", err)
}
if address != "" {
if err := insertPolicy(b.s, address, p, family); err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
}
} else {
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
return fmt.Errorf("failed to add peer: %v", err)
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
return nil
return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
Peer: p,
})
}
func (b *Server) getPath(ip net.IP) (path *api.Path) {
isV6 := ip.To4() == nil
//nolint
originAttr, _ := anypb.New(&api.OriginAttribute{
originAttr, _ := ptypes.MarshalAny(&api.OriginAttribute{
Origin: 0,
})
if !isV6 {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 32,
})
//nolint
nhAttr, _ := anypb.New(&api.NextHopAttribute{
nhAttr, _ := ptypes.MarshalAny(&api.NextHopAttribute{
NextHop: "0.0.0.0", // gobgp will fill this
})
@@ -141,11 +81,11 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
Safi: api.Family_SAFI_UNICAST,
},
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, nhAttr},
Pattrs: []*any.Any{originAttr, nhAttr},
}
} else {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 128,
})
@@ -156,16 +96,16 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
}
//nolint
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
mpAttr, _ := ptypes.MarshalAny(&api.MpReachNLRIAttribute{
Family: v6Family,
NextHops: []string{"::"}, // gobgp will fill this
Nlris: []*anypb.Any{nlri},
Nlris: []*any.Any{nlri},
})
path = &api.Path{
Family: v6Family,
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, mpAttr},
Pattrs: []*any.Any{originAttr, mpAttr},
}
}
return
@@ -175,13 +115,11 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
return nil, fmt.Errorf("No BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
@@ -224,118 +162,14 @@ func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := Peer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
Address: address,
AS: uint32(ASNumber),
Password: password,
MultiHop: multiHop,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
familyType := "v4"
if family == api.Family_AFI_IP6 {
familyType = "v6"
}
setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
policyName := fmt.Sprintf("%s-%s", setName, familyType)
policy := &api.Policy{
Name: policyName,
Statements: []*api.Statement{
{
Conditions: &api.Conditions{
AfiSafiIn: []*api.Family{
{
Afi: family,
Safi: api.Family_SAFI_UNICAST,
},
},
NeighborSet: &api.MatchSet{
Type: api.MatchSet_ANY,
Name: setName,
},
},
Actions: &api.Actions{
RouteAction: api.RouteAction_ACCEPT,
Nexthop: &api.NexthopAction{
Address: address,
},
},
},
{
Conditions: &api.Conditions{
NeighborSet: &api.MatchSet{
Type: api.MatchSet_ANY,
Name: setName,
},
},
Actions: &api.Actions{
RouteAction: api.RouteAction_ACCEPT,
},
},
},
}
err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
Policy: policy,
})
if err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
Assignment: &api.PolicyAssignment{
Name: "global",
Direction: api.PolicyDirection_EXPORT,
Policies: []*api.Policy{
{
Name: policy.Name,
},
},
},
})
if err != nil {
return fmt.Errorf("failed to add policy assignment: %v", err)
}
return nil
}

View File

@@ -1,54 +0,0 @@
package bgp
import (
"reflect"
"testing"
)
func TestParseBGPPeerConfig(t *testing.T) {
type args struct {
config string
}
tests := []struct {
name string
args args
wantBgpPeers []Peer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
},
},
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
},
},
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []Peer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
}
})
}
}

View File

@@ -5,49 +5,35 @@ import (
"fmt"
"time"
log "log/slog"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c *Config) (b *Server, err error) {
func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
return nil, fmt.Errorf("You need to provide AS")
}
if c.SourceIP != "" && c.SourceIF != "" {
return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
}
if len(c.Peers) == 0 {
return nil, fmt.Errorf("you need to provide at least one peer")
return nil, fmt.Errorf("You need to provide at least one peer")
}
b = &Server{
s: gobgp.NewBgpServer(),
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
}
return
}
// Start starts the BGP server
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: b.c.AS,
RouterId: b.c.RouterID,
Asn: c.AS,
RouterId: c.RouterID,
ListenPort: -1,
},
}); err != nil {
@@ -56,7 +42,7 @@ func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_Peer
if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
log.Info("[BGP]", "peer", p.String())
log.Infof("[BGP] %s", p.String())
if peerStateChangeCallback != nil {
peerStateChangeCallback(p)
}
@@ -65,23 +51,12 @@ func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_Peer
return
}
for _, p := range b.c.Peers {
for _, p := range c.Peers {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
}); err != nil {
log.Error(err.Error())
return
}
}
return
}

View File

@@ -1,145 +1,30 @@
package bgp
import (
"fmt"
"net"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
import gobgp "github.com/osrg/gobgp/v3/pkg/server"
// Peer defines a BGP Peer
type Peer struct {
Address string
Port uint16
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
func (p *Peer) setMpbgpOptions(server *Config) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := vip.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if vip.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
Address string
AS uint32
Password string
MultiHop bool
}
// Config defines the BGP server configuration
type Config struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
AS uint32
RouterID string
SourceIP string
SourceIF string
HoldTime uint64
KeepaliveInterval uint64
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -3,11 +3,9 @@ package cluster
import (
"sync"
log "log/slog"
log "github.com/sirupsen/logrus"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -17,17 +15,16 @@ type Cluster struct {
completed chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
var networks []vip.Network
var err error
if !disableVIP {
// Start the Virtual IP Networking configuration
networks, err = startNetworking(c, intfMgr)
networks, err = startNetworking(c)
if err != nil {
return nil, err
}
@@ -35,15 +32,14 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
// Initialise the Cluster structure
newCluster := &Cluster{
Network: networks,
arpMgr: arpMgr,
}
log.Debug("service security", "enabled", c.EnableServiceSecurity)
log.Debugf("init enable service security: %t", c.EnableServiceSecurity)
return newCluster, nil
}
func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
func startNetworking(c *kubevip.Config) ([]vip.Network, error) {
address := c.VIP
if c.Address != "" {
@@ -54,9 +50,7 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend)
if err != nil {
return nil, err
}

View File

@@ -11,13 +11,15 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
log "log/slog"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -94,7 +96,7 @@ func NewManager(path string, inCluster bool, port int) (*Manager, error) {
func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
var err error
log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, c.LeaseName, c.NodeName)
// use a Go context so we can tell the leaderelection code when we
// want to step down
@@ -132,12 +134,9 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
// (attempt to) Remove the virtual IP, in case it already exists
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
err = cluster.Network[i].DeleteIP()
if err != nil {
log.Error("could not delete virtualIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Errorf("could not delete virtualIP: %v", err)
}
}
@@ -148,15 +147,41 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
}
}()
// If Equinix Metal is enabled then we can begin our preparation work
var packetClient *packngo.Client
if c.EnableMetal {
if c.ProviderConfig != "" {
key, project, err := equinixmetal.GetPacketConfig(c.ProviderConfig)
if err != nil {
log.Error(err)
} else {
// Set the environment variable with the key for the project
os.Setenv("PACKET_AUTH_TOKEN", key)
// Update the configuration with the project key
c.MetalProjectID = project
}
}
packetClient, err = packngo.NewClient()
if err != nil {
log.Error(err)
}
// We're using Equinix Metal with BGP, populate the Peer information from the API
if c.EnableBGP {
log.Infoln("Looking up the BGP configuration from Equinix Metal")
err = equinixmetal.BGPLookup(packetClient, c)
if err != nil {
log.Error(err)
}
}
}
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
log.Error(err)
}
}
@@ -164,11 +189,11 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
config: c,
leaseID: c.NodeName,
sm: sm,
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
onStartedLeading: func(ctx context.Context) {
// As we're leading lets start the vip service
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient)
if err != nil {
log.Error("starting VIP service on leader", "err", err)
log.Errorf("Error starting the VIP service on the leader [%s]", err)
}
},
onStoppedLeading: func() {
@@ -184,26 +209,22 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
if bgpServer != nil {
err := bgpServer.Close()
if err != nil {
log.Warn("close BGP server", "err", err)
log.Warnf("%v", err)
}
}
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Warnf("%v", err)
}
}
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO - we could also return here
log.Fatal("lost leadership, restarting kube-vip")
},
onNewLeader: func(identity string) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
log.Infof("Node [%s] is assuming leadership of the cluster", identity)
},
}
@@ -284,16 +305,16 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
})
}
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
log.Infof("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
@@ -322,16 +343,9 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("add IPVS backend", "err", err)
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("remove IPVS backend", "err", err)
}
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Errorf("add IPVS backend [%v]", err)
}
}
}
@@ -346,12 +360,12 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("Del IPVS backend", "err", err)
log.Errorf("Del IPVS backend [%v]", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
log.Infof("Node [%s] has been deleted", node.Name)
case watch.Bookmark:
// Un-used
@@ -362,29 +376,15 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
log.Errorf("%v", status)
default:
}
}
log.Info("Exiting Node watcher")
log.Infoln("Exiting Node watcher")
return nil
}
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
return false
}

View File

@@ -8,25 +8,24 @@ import (
"net"
"os"
"os/signal"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, packetClient *packngo.Client) error {
var err error
// listen for interrupts or the Linux SIGTERM signal and cancel
@@ -39,74 +38,117 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
if network.IsDDNS() {
if cluster.Network[i].IsDDNS() {
if err := cluster.StartDDNS(ctxDNS); err != nil {
log.Error(err.Error())
log.Error(err)
}
}
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("starting the DNS updater", "address", network.DNSName())
ipUpdater := vip.NewIPUpdater(network)
if cluster.Network[i].IsDNS() {
log.Infof("starting the DNS updater for the address %s", cluster.Network[i].DNSName())
ipUpdater := vip.NewIPUpdater(cluster.Network[i])
ipUpdater.Run(ctxDNS)
}
if !c.EnableRoutingTable {
if _, err = network.AddIP(false); err != nil {
log.Error(err.Error())
if c.EnableARP {
subnets := vip.Split(c.VIPCIDR)
subnet := ""
if len(subnets) > 0 {
subnet = subnets[0]
}
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
subnet = subnets[1]
}
if subnet == "" {
log.Fatalf("no subnet provided for IP %s", cluster.Network[i].IP())
}
if err = cluster.Network[i].SetMask(subnet); err != nil {
log.Fatalf("failed to set mask %s: %s", subnet, err.Error())
}
}
if err = cluster.Network[i].AddIP(false); err != nil {
log.Fatalf("%v", err)
}
}
if c.EnableMetal {
// We're not using Equinix Metal with BGP
if !c.EnableBGP {
// Attempt to attach the EIP in the standard manner
log.Debugf("Attaching the Equinix Metal EIP through the API to this host")
err = equinixmetal.AttachEIP(packetClient, c, c.NodeName)
if err != nil {
log.Error(err)
}
}
}
if c.EnableBGP {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(network.CIDR())
cidrVip := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), c.VIPCIDR)
log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
err = bgpServer.AddHost(cidrVip)
if err != nil {
log.Error(err.Error())
log.Error(err)
}
}
if c.EnableLoadBalancer {
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
log.Infof("Starting IPVS LoadBalancer")
lb, err := loadbalancer.NewIPVSLB(cluster.Network[i].IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval)
if err != nil {
log.Error("Error creating IPVS LoadBalancer", "err", err)
log.Errorf("Error creating IPVS LoadBalancer [%s]", err)
}
go func() {
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
err = sm.NodeWatcher(lb, c.Port)
if err != nil {
log.Error("Error watching node labels", "err", err)
log.Errorf("Error watching node labels [%s]", err)
}
}()
loadbalancers = append(loadbalancers, lb)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-signalChan
err = lb.RemoveIPVSLB()
if err != nil {
log.Errorf("Error stopping IPVS LoadBalancer [%s]", err)
}
log.Info("Stopping IPVS LoadBalancer")
}()
}
if c.EnableARP {
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
}
go func(ctx context.Context, i int) {
ipString := cluster.Network[i].IP()
isIPv6 := vip.IsIPv6(ipString)
if c.EnableLoadBalancer {
// Shutdown function that will wait on this signal, unless we call it ourselves
<-signalChan
for _, lb := range loadbalancers {
err = lb.RemoveIPVSLB()
if err != nil {
log.Error("Error stopping IPVS LoadBalancer", "err", err)
}
var ndp *vip.NdpResponder
if isIPv6 {
ndp, err = vip.NewNDPResponder(cluster.Network[i].Interface())
if err != nil {
log.Fatalf("failed to create new NDP Responder")
}
}
if ndp != nil {
defer ndp.Close()
}
log.Infof("Gratuitous Arp broadcast will repeat every 3 seconds for [%s/%s]", ipString, cluster.Network[i].Interface())
for {
select {
case <-ctx.Done(): // if cancel() execute
return
default:
cluster.ensureIPAndSendGratuitous(i, ndp)
}
time.Sleep(3 * time.Second)
}
}(ctxArp, i)
}
}
@@ -125,7 +167,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
ips := []string{}
if nodename != "" {
if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane nod", "err", err)
log.Error("failed to get IP of control-plane node: %w", err)
}
}
@@ -140,7 +182,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
ips = append(ips, "::1")
}
log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
log.Infof("no IP address found for node - will fallback to use localhost address: %v", ips)
}
for _, ip := range ips {
@@ -166,8 +208,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
backend.Watch(func() {
for i := range cluster.Network {
network := cluster.Network[i]
networkIP := network.IP()
networkIP := cluster.Network[i].IP()
isNetworkV6, err := isV6(networkIP)
if err != nil {
log.Error("failed to check IP type", "IP", networkIP, "error", err)
@@ -181,19 +222,19 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
for entry := range *backendMap {
if entry.Check() {
_, err = network.AddIP(true)
err = cluster.Network[i].AddIP(true)
if err != nil {
log.Error("error adding address", "err", err)
log.Fatalf("error adding IP: %v", err)
}
if !(*backendMap)[entry] {
log.Info("added backend", "ip", network.IP())
log.Infof("added IP: %s", cluster.Network[i].IP())
}
err = network.AddRoute(true)
err = cluster.Network[i].AddRoute(true)
if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn(err.Error())
log.Warnf("%v", err)
} else if err == nil && !(*backendMap)[entry] {
log.Info("added route", "route", network.PrepareRoute().String())
log.Infof("added route: %s", cluster.Network[i].PrepareRoute().String())
}
(*backendMap)[entry] = true
@@ -211,20 +252,23 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
if deleteAddress {
err = network.DeleteRoute()
err = cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn("deleting route", "err", err)
log.Warnf("error while deleting route: %v", err)
} else if err == nil {
log.Info("deleted route", "route", network.PrepareRoute().String())
log.Infof("deleted route: %s", cluster.Network[i].PrepareRoute().String())
}
deleted, err := network.DeleteIP()
isSet, err := cluster.Network[i].IsSet()
if err != nil {
log.Error("error deleting IP", "err", err)
panic("")
log.Error("failed to check IP address", "error", err)
}
if deleted {
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
if isSet {
err = cluster.Network[i].DeleteIP()
if err != nil {
log.Fatalf("error deleting IP: %v", err)
}
log.Infof("deleted IP: %s", cluster.Network[i].IP())
}
}
}
@@ -257,50 +301,88 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
ctxArp, cancelArp := context.WithCancel(ctx)
ctxArp, cancelArp := context.WithCancel(context.Background())
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
_, err := network.DeleteIP()
if err != nil {
log.Warn("attempted to clean existing VIP", "err", err)
}
err := network.DeleteIP()
if err != nil {
log.Warnf("Attempted to clean existing VIP => %v", err)
}
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
err = network.AddRoute(false)
if err != nil {
log.Warn(err.Error())
log.Warnf("%v", err)
}
} else if !c.EnableRoutingTable {
if _, err = network.AddIP(false); err != nil {
log.Warn(err.Error())
if c.EnableARP {
subnets := vip.Split(c.VIPCIDR)
subnet := ""
if len(subnets) > 0 {
subnet = subnets[0]
}
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
subnet = subnets[1]
}
if subnet == "" {
log.Fatalf("no subnet provided for IP %s", cluster.Network[i].IP())
}
if err = network.SetMask(subnet); err != nil {
log.Fatalf("failed to set mask %s: %s", subnet, err.Error())
}
}
if err = network.AddIP(false); err != nil {
log.Warnf("%v", err)
}
}
if c.EnableARP {
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
ipString := network.IP()
var ndp *vip.NdpResponder
if vip.IsIPv6(ipString) {
ndp, err = vip.NewNDPResponder(network.Interface())
if err != nil {
log.Fatalf("failed to create new NDP Responder")
}
}
go func(ctx context.Context) {
if ndp != nil {
defer ndp.Close()
}
log.Debugf("(svcs) broadcasting ARP update for %s via %s, every %dms", ipString, network.Interface(), c.ArpBroadcastRate)
for {
select {
case <-ctx.Done(): // if cancel() execute
log.Debugf("(svcs) ending ARP update for %s via %s, every %dms", ipString, network.Interface(), c.ArpBroadcastRate)
return
default:
cluster.ensureIPAndSendGratuitous(i, ndp)
}
if c.ArpBroadcastRate < 500 {
log.Errorf("arp broadcast rate is [%d], this shouldn't be lower that 300ms (defaulting to 3000)", c.ArpBroadcastRate)
c.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(c.ArpBroadcastRate) * time.Millisecond)
}
}(ctxArp)
}
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
err = bgp.AddHost(network.CIDR())
cidrVip := fmt.Sprintf("%s/%s", network.IP(), c.VIPCIDR)
log.Debugf("(svcs) attempting to advertise the address [%s] over BGP", cidrVip)
err = bgp.AddHost(cidrVip)
if err != nil {
log.Error(err.Error())
log.Error(err)
}
}
}
@@ -310,19 +392,13 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
// Stop the Arp context if it is running
cancelArp()
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
log.Info("[LOADBALANCER] Stopping load balancers")
if c.EnableRoutingTable {
for i := range cluster.Network {
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
}
log.Infof("[VIP] Deleting Route for Virtual IP [%s]", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warnf("%v", err)
}
}
@@ -330,16 +406,9 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
return
}
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
continue
}
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Infof("[VIP] Releasing the Virtual IP [%s]", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteIP(); err != nil {
log.Warnf("%v", err)
}
}
@@ -347,35 +416,52 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
}()
}
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if vip.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
func (cluster *Cluster) ensureIPAndSendGratuitous(index int, ndp *vip.NdpResponder) {
iface := cluster.Network[index].Interface()
ipString := cluster.Network[index].IP()
} else {
ndp, err = vip.NewNDPResponder(network.Interface())
if err != nil {
log.Error("failed to create new NDP Responder", "error", err)
} else {
if ndp != nil {
defer ndp.Close()
}
}
// Check if IP is dadfailed
if cluster.Network[index].IsDADFAIL() {
log.Warnf("IP address is in dadfailed state, removing [%s] from interface [%s]", ipString, iface)
err := cluster.Network[index].DeleteIP()
if err != nil {
log.Warnf("%v", err)
}
}
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
// Ensure the address exists on the interface before attempting to ARP
set, err := cluster.Network[index].IsSet()
if err != nil {
log.Warnf("%v", err)
}
if !set {
log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, iface)
err = cluster.Network[index].AddIP(false)
if err != nil {
log.Warnf("%v", err)
}
}
arpInstance := arp.NewInstance(network, ndp)
cluster.arpMgr.Insert(arpInstance)
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if ndp == nil {
log.Error("NDP responder was not created")
} else {
err := ndp.SendGratuitous(ipString)
if err != nil {
log.Warnf("%v", err)
}
}
} else {
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
err := vip.ARPSendGratuitous(ipString, iface)
if err != nil {
log.Warnf("%v", err)
}
}
<-ctx.Done() // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.Remove(arpInstance)
}

View File

@@ -3,7 +3,8 @@ package cluster
import (
"context"
log "log/slog"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
@@ -15,7 +16,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
// Start kube-vip as a single node server
// TODO - Split all this code out as a separate function
log.Info("Starting kube-vip as a single node cluster")
log.Infoln("Starting kube-vip as a single node cluster")
log.Info("This node is assuming leadership of the cluster")
@@ -24,17 +25,14 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
for i := range cluster.Network {
if !disableVIP {
deleted, err := cluster.Network[i].DeleteIP()
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Warnf("Attempted to clean existing VIP => %v", err)
}
_, err = cluster.Network[i].AddIP(false)
err = cluster.Network[i].AddIP(false)
if err != nil {
log.Warn(err.Error())
log.Warnf("%v", err)
}
}
@@ -43,7 +41,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
// Gratuitous ARP, will broadcast to new MAC <-> IP
err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
if err != nil {
log.Warn(err.Error())
log.Warnf("%v", err)
}
}
}
@@ -53,23 +51,20 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
if !disableVIP {
for i := range cluster.Network {
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
log.Infof("[VIP] Releasing the Virtual IP [%s]", cluster.Network[i].IP())
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Warnf("%v", err)
}
}
}
close(cluster.completed)
}()
log.Info("Started Load Balancer and Virtual IP")
log.Infoln("Started Load Balancer and Virtual IP")
return nil
}
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server, packetClient *packngo.Client) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
@@ -80,5 +75,5 @@ func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, packetClient)
}

View File

@@ -1,99 +0,0 @@
package egress
import (
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
IPv6 := false
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" {
return nftables.DeleteSNAT(IPv6, serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,165 +0,0 @@
package endpoints
import (
"context"
"fmt"
"net"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
)
type Processor struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
// There are local endpoints available on the node
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
// There are no local endpoints
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = endpoints[0]
return
}
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == *lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
activeEndpointAnnotation := kubevip.ActiveEndpoint
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}

View File

@@ -1,125 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type BGP struct {
generic
bgpServer *bgp.Server
}
func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
return &BGP{
generic: generic,
bgpServer: bgpServer,
}
}
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
}
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
}
}
}
}

View File

@@ -1,118 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableRoutingTable {
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
}
return &generic
}
type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
} else {
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
}
return endpoints, nil
}
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -1,178 +0,0 @@
package endpoints
import (
"context"
"errors"
"fmt"
"net"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
}
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
}
}
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
return rt.getAllEndpoints(service, id)
}
func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -1,137 +0,0 @@
package providers
import (
"context"
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
log "log/slog"
)
type Endpoints struct {
label string
//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
endpoints *v1.Endpoints
}
func NewEndpoints() Provider {
return &Endpoints{
label: "endpoints",
}
}
func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
}
ep.endpoints = eps
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -1,22 +0,0 @@
package providers
import (
"context"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
watchtools "k8s.io/client-go/tools/watch"
)
type Provider interface {
CreateRetryWatcher(context.Context, *kubernetes.Clientset,
*v1.Service) (*watchtools.RetryWatcher, error)
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
GetProtocol() string
}

63
pkg/equinixmetal/bgp.go Normal file
View File

@@ -0,0 +1,63 @@
package equinixmetal
import (
"fmt"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
)
// BGPLookup will use the Equinix Metal API functions to populate the BGP information
func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
var thisDevice *packngo.Device
if k.MetalProjectID == "" {
proj := findProject(k.MetalProject, c)
if proj == nil {
return fmt.Errorf("Unable to find Project [%s]", k.MetalProject)
}
thisDevice = findSelf(c, proj.ID)
} else {
thisDevice = findSelf(c, k.MetalProjectID)
}
if thisDevice == nil {
return fmt.Errorf("Unable to find local/this device in Equinix Metal API")
}
log.Infof("Querying BGP settings for [%s]", thisDevice.Hostname)
neighbours, _, err := c.Devices.ListBGPNeighbors(thisDevice.ID, &packngo.ListOptions{})
if err != nil {
return err
}
// Ensure neighbours exist (and it's enabled)
if len(neighbours) == 0 {
return fmt.Errorf("The server [%s]/[%s] has no BGP neighbours, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
}
// Add a warning (TODO)
if len(neighbours) > 1 {
log.Warnf("There are [%d] neighbours, only designed to manage one", len(neighbours))
}
// Ensure a peer exists
if len(neighbours[0].PeerIps) == 0 {
return fmt.Errorf("The server [%s]/[%s] has no BGP peers, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
}
k.BGPConfig.RouterID = neighbours[0].CustomerIP
k.BGPConfig.AS = uint32(neighbours[0].CustomerAs)
// Add the peer(s)
for x := range neighbours[0].PeerIps {
peer := bgp.Peer{
Address: neighbours[0].PeerIps[x],
AS: uint32(neighbours[0].PeerAs),
MultiHop: neighbours[0].Multihop,
Password: neighbours[0].Md5Password,
}
k.BGPConfig.Peers = append(k.BGPConfig.Peers, peer)
}
return nil
}

65
pkg/equinixmetal/eip.go Normal file
View File

@@ -0,0 +1,65 @@
package equinixmetal
import (
"fmt"
"path"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
)
// AttachEIP will use the Equinix Metal APIs to move an EIP and attach to a host
func AttachEIP(c *packngo.Client, k *kubevip.Config, _ string) error {
// Use MetalProjectID if it is defined
projID := k.MetalProjectID
if projID == "" {
// Fallback to attempting to find the project by name
proj := findProject(k.MetalProject, c)
if proj == nil {
return fmt.Errorf("unable to find Project [%s]", k.MetalProject)
}
projID = proj.ID
}
// Prefer Address over VIP
vip := k.Address
if vip == "" {
vip = k.VIP
}
ips, _, _ := c.ProjectIPs.List(projID, &packngo.ListOptions{})
for _, ip := range ips {
// Find the device id for our EIP
if ip.Address == vip {
log.Infof("Found EIP ->%s ID -> %s\n", ip.Address, ip.ID)
// If attachments already exist then remove them
if len(ip.Assignments) != 0 {
hrefID := path.Base(ip.Assignments[0].Href)
_, err := c.DeviceIPs.Unassign(hrefID)
if err != nil {
return fmt.Errorf("unable to unassign deviceIP %q: %v", hrefID, err)
}
}
}
}
// Lookup this server through the Equinix Metal API
thisDevice := findSelf(c, projID)
if thisDevice == nil {
return fmt.Errorf("unable to find local/this device in Equinix Metal API")
}
// Assign the EIP to this device
log.Infof("Assigning EIP to -> %s\n", thisDevice.Hostname)
_, _, err := c.DeviceIPs.Assign(thisDevice.ID, &packngo.AddressStruct{
Address: vip,
})
if err != nil {
return err
}
return nil
}

59
pkg/equinixmetal/utils.go Normal file
View File

@@ -0,0 +1,59 @@
package equinixmetal
import (
"encoding/json"
"fmt"
"os"
"github.com/packethost/packngo"
log "github.com/sirupsen/logrus"
)
func findProject(project string, c *packngo.Client) *packngo.Project {
l := &packngo.ListOptions{Includes: []string{project}}
ps, _, err := c.Projects.List(l)
if err != nil {
log.Error(err)
}
for _, p := range ps {
// Find our project
if p.Name == project {
return &p
}
}
return nil
}
func findSelf(c *packngo.Client, projectID string) *packngo.Device {
// Go through devices
dev, _, _ := c.Devices.List(projectID, &packngo.ListOptions{})
for _, d := range dev {
// TODO do we need to replace os.Hostname with config.NodeName here?
me, _ := os.Hostname()
if me == d.Hostname {
return &d
}
}
return nil
}
// GetPacketConfig will lookup the configuration from a file path
func GetPacketConfig(providerConfig string) (string, string, error) {
var config struct {
AuthToken string `json:"apiKey"`
ProjectID string `json:"projectId"`
}
// get our token and project
if providerConfig != "" {
configBytes, err := os.ReadFile(providerConfig)
if err != nil {
return "", "", fmt.Errorf("failed to get read configuration file at path %s: %v", providerConfig, err)
}
err = json.Unmarshal(configBytes, &config)
if err != nil {
return "", "", fmt.Errorf("failed to process json of configuration file at path %s: %v", providerConfig, err)
}
}
return config.AuthToken, config.ProjectID, nil
}

View File

@@ -5,9 +5,8 @@ import (
"hash/fnv"
"time"
log "log/slog"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
pb "go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
@@ -32,7 +31,7 @@ type LeaderElectionConfig struct {
// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
// from string to int using hashing.
MemberUniqueID *uint64
MemberUniqueID *int64
// LeaseDurationSeconds is the duration that non-leader candidates will
// wait to force acquire leadership.
@@ -74,7 +73,7 @@ func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
// RunElection blocks until leader election loop is
// stopped by ctx or it has stopped holding the leader lease.
func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
var memberID uint64
var memberID int64
if config.MemberUniqueID != nil {
memberID = *config.MemberUniqueID
} else {
@@ -82,11 +81,11 @@ func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
return err
}
memberID = h.Sum64()
memberID = int64(h.Sum64())
}
ttl := config.LeaseDurationSeconds
r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
r := &pb.LeaseGrantRequest{TTL: ttl, ID: memberID}
lease, err := clientv3.RetryLeaseClient(
config.EtcdConfig.Client,
).LeaseGrant(ctx, r)
@@ -148,9 +147,9 @@ watcher:
m.isLeader = true
m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
log.Debugf("[%s] Marking self as leader with key %s\n", m.memberID, m.key)
case response := <-changes:
log.Debug("Leader Changes", "id", m.memberID, "response", response)
log.Debugf("[%s] Leader Changes: %+v\n", m.memberID, response)
if len(response.Kvs) == 0 {
// There is a race condition where just after we stop being the leader
// if there are no more leaders, we might get a response with no key-values
@@ -182,15 +181,15 @@ watcher:
m.callbacks.OnStoppedLeading()
}
log.Debug("Exiting watcher", "id", m.memberID)
log.Debugf("[%s] Exiting watcher\n", m.memberID)
}
func (m *member) tryToBeLeader(ctx context.Context) {
if err := m.election.Campaign(ctx, m.memberID); err != nil {
log.Error("Failed trying to become the leader", "err", err)
log.Errorf("Failed trying to become the leader: %s", err)
// Resign just in case we acquired leadership just before failing
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
log.Warnf("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader: %s", err)
}
return
// TODO: what to do here?
@@ -210,7 +209,7 @@ func (m *member) tryToBeLeader(ctx context.Context) {
// the previous leader to detect the new leadership (if there was one) and
// stop its processes
// TODO: is this too cautious?
log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
log.Debugf("[%s] Waiting %d seconds before running OnStartedLeading", m.memberID, m.leaseTTL)
time.Sleep(time.Second * time.Duration(m.leaseTTL))
// We are the leader, execute our code
@@ -222,6 +221,6 @@ func (m *member) tryToBeLeader(ctx context.Context) {
func (m *member) resignOnCancel(ctx context.Context) {
<-ctx.Done()
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Error("Failed to resign after the context was canceled", "err", err)
log.Errorf("Failed to resign after the context was canceled: %s", err)
}
}

View File

@@ -91,12 +91,12 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
config1 := configBase
config1.EtcdConfig.Client = cliMember1
config1.MemberID = "my-host"
uniqueID := rand.Uint64()
uniqueID := rand.Int63()
config1.MemberUniqueID = &uniqueID
config1.Callbacks = baseCallbacksForName(config1.MemberID)
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-host, the new leader!!!!")
log.Println("Losing the leadership on purpose by stopping renewing the lease")
log.Println("Loosing the leadership on purpose by stopping renewing the lease")
g.Expect(cliMember1.Lease.Close()).To(Succeed())
log.Println("Member1 leases closed")
}

View File

@@ -6,8 +6,7 @@ import (
"net"
"time"
log "log/slog"
log "github.com/sirupsen/logrus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@@ -100,21 +99,21 @@ func FindWorkingKubernetesAddress(configPath string, inCluster bool) (*rest.Conf
return nil, err
}
for x := range ips {
log.Debug("[k8s client] testing", "address", ips[x].String())
log.Debugf("[k8s client] checking with IP address [%s]", ips[x].String())
c, err := NewRestConfig(configPath, inCluster, net.JoinHostPort(ips[x].String(), "6443"))
if err != nil {
log.Error("failed to create k8s REST config", "err", err)
log.Errorf("failed to create k8s REST config: %v", err)
}
c.Timeout = 2 * time.Second
k, err := NewClientset(c)
if err != nil {
log.Error("failed to create k8s clientset", "err", err)
log.Errorf("failed to create k8s clientset: %v", err)
}
_, err = k.DiscoveryClient.ServerVersion()
if err == nil {
log.Info("[k8s client] working", "address", ips[x].String())
log.Infof("[k8s client] working with IP address [%s]", ips[x].String())
c.Timeout = defaultTimeout
return c, nil
}

View File

@@ -1,49 +0,0 @@
package kubevip
const (
// Hardware address of the host that has the VIP
HwAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
// The host that has the VIP
VipHost = "kube-vip.io/vipHost"
// Enable Egress on a service
Egress = "kube-vip.io/egress"
// Enable internal Egress
EgressInternal = "kube-vip.io/egress-internal"
// Egress should be IPv6
EgressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
EgressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
ActiveEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
)

View File

@@ -7,7 +7,6 @@ import (
"math/bits"
"os"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
@@ -20,13 +19,17 @@ func ParseEnvironment(c *Config) error {
}
// Ensure that logging is set through the environment variables
env := os.Getenv(vipLogLevel)
// Set default value
if env == "" {
env = "4"
}
if env != "" {
logLevel, err := strconv.ParseInt(env, 10, 32)
logLevel, err := strconv.ParseUint(env, 10, 32)
if err != nil {
panic("Unable to parse environment variable [vip_loglevel], should be int")
}
c.Logging = int32(logLevel)
c.Logging = int(logLevel)
}
// Find interface
@@ -50,6 +53,12 @@ func ParseEnvironment(c *Config) error {
c.ServicesInterface = env
}
// Find provider configuration
env = os.Getenv(providerConfig)
if env != "" {
c.ProviderConfig = env
}
// Find Kubernetes Leader Election configuration
env = os.Getenv(vipLeaderElection)
if env != "" {
@@ -127,11 +136,11 @@ func ParseEnvironment(c *Config) error {
// Find vip port
env = os.Getenv(port)
if env != "" {
i, err := strconv.ParseUint(env, 10, 16)
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
c.Port = uint16(i)
c.Port = int(i)
}
// Find vipDdns
@@ -233,6 +242,12 @@ func ParseEnvironment(c *Config) error {
}
}
// Find vip address cidr range
env = os.Getenv(vipCidr)
if env != "" {
c.VIPCIDR = env
}
// Find vip address subnet
env = os.Getenv(vipSubnet)
if env != "" {
@@ -429,24 +444,6 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.Peers = peers
}
// MPBGP mode
env = os.Getenv(mpbgpNexthop)
if env != "" {
c.BGPConfig.MpbgpNexthop = env
}
// MPBGP fixed IPv4
env = os.Getenv(mpbgpIPv4)
if env != "" {
c.BGPConfig.MpbgpIPv4 = env
}
// MPBGP fixed IPv6
env = os.Getenv(mpbgpIPv6)
if env != "" {
c.BGPConfig.MpbgpIPv6 = env
}
// BGP Peer mutlihop
env = os.Getenv(bgpMultiHop)
if env != "" {
@@ -501,32 +498,28 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
env = os.Getenv(zebraEnable)
// Enable the Equinix Metal API calls
env = os.Getenv(vipPacket)
if env != "" {
result, err := strconv.ParseBool(env)
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.BGPConfig.Zebra.Enabled = result
c.EnableMetal = b
}
env = os.Getenv(zebraURL)
// Find the Equinix Metal project name
env = os.Getenv(vipPacketProject)
if env != "" {
c.BGPConfig.Zebra.URL = env
// TODO - parse address net.Host()
c.MetalProject = env
}
env = os.Getenv(zebraVersion)
// Find the Equinix Metal project ID
env = os.Getenv(vipPacketProjectID)
if env != "" {
u64, err := strconv.ParseUint(env, 10, 32)
if err != nil {
return err
}
c.BGPConfig.Zebra.Version = uint32(u64)
}
env = os.Getenv(zebraSoftwareName)
if env != "" {
c.BGPConfig.Zebra.SoftwareName = env
// TODO - parse address net.Host()
c.MetalProjectID = env
}
// Enable the load-balancer
@@ -542,11 +535,11 @@ func ParseEnvironment(c *Config) error {
// Find loadbalancer port
env = os.Getenv(lbPort)
if env != "" {
i, err := strconv.ParseUint(env, 10, 16)
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
c.LoadBalancerPort = uint16(i)
c.LoadBalancerPort = int(i)
}
// Find loadbalancer forwarding method
@@ -607,13 +600,13 @@ func ParseEnvironment(c *Config) error {
c.K8sConfigFile = env
}
env = os.Getenv(enableEndpoints)
env = os.Getenv(enableEndpointSlices)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableEndpoints = b
c.EnableEndpointSlices = b
}
env = os.Getenv(mirrorDestInterface)
@@ -635,37 +628,5 @@ func ParseEnvironment(c *Config) error {
c.BackendHealthCheckInterval = int(i)
}
env = os.Getenv(healthCheckPort)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
if i < 1024 {
return fmt.Errorf("health check port should be > 1024")
}
c.HealthCheckPort = int(i)
}
env = os.Getenv(enableUPNP)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableUPNP = b
}
if env = os.Getenv(egressClean); env == "" {
env = os.Getenv(strings.ToUpper(egressClean))
}
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EgressClean = b
}
return nil
}

View File

@@ -39,6 +39,9 @@ const (
// vipServicesInterface - defines the interface that the service vips should bind too
vipServicesInterface = "vip_servicesinterface"
// vipCidr - defines the cidr that the vip will use (for BGP)
vipCidr = "vip_cidr"
// vipSubnet - defines the subnet that the vip will use
vipSubnet = "vip_subnet"
@@ -83,6 +86,18 @@ const (
// vipStartLeader - will start this instance as the leader of the cluster
vipStartLeader = "vip_startleader"
// vipPacket defines that the packet API will be used for EIP
vipPacket = "vip_packet"
// vipPacketProject defines which project within Packet to use
vipPacketProject = "vip_packetproject"
// vipPacketProjectID defines which projectID within Packet to use
vipPacketProjectID = "vip_packetprojectid"
// providerConfig defines a path to a configuration that should be parsed
providerConfig = "provider_config"
// bgpEnable defines if BGP should be enabled
bgpEnable = "bgp_enable"
// bgpRouterID defines the routerID for the BGP server
@@ -110,22 +125,6 @@ const (
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
zebraURL = "zebra_url"
// zebraVersion specifies Zebra API Version
zebraVersion = "zebra_version"
// zebraSoftwareName specifies Software Name for Zebra
zebraSoftwareName = "zebra_software_name"
// mpbgpNexthop defines MPBGP mode
mpbgpNexthop = "mpbgp_nexthop"
// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
mpbgpIPv4 = "mpbgp_ipv4"
// mpbgpIPv6 defines fixed IPv6 to be used with MPBGP
mpbgpIPv6 = "mpbgp_ipv6"
// vipWireguard - defines if wireguard will be used for vips
vipWireguard = "vip_wireguard" //nolint
@@ -212,8 +211,8 @@ const (
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
// enableEndpoints enables use of Endpoints instead of EndpointSlices
enableEndpoints = "enable_endpoints"
// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
enableEndpointSlices = "enable_endpointslices"
// mirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
@@ -225,13 +224,4 @@ const (
// backendHealthCheckInterval Interval in seconds for checking backend health.
backendHealthCheckInterval = "backend_health_check_interval"
// healthCheckPort, if set to non-zero will be the port the health check will listen on
healthCheckPort = "health_check_port"
// enableUPNP enables UPNP functions
enableUPNP = "enable_upnp"
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
)

View File

@@ -2,14 +2,11 @@ package kubevip
import (
"fmt"
"log"
"strconv"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
@@ -17,43 +14,11 @@ import (
"sigs.k8s.io/yaml"
)
// TransformApplyObjectToManifest transforms an apply object into a normal Kubernetes manifest
func TransformApplyObjectToManifest(applyObject interface{}) string {
// Convert the apply object to an unstructured object
unstructuredObj := &unstructured.Unstructured{}
err := runtime.DefaultUnstructuredConverter.FromUnstructured(applyConfigToMap(applyObject), unstructuredObj)
if err != nil {
log.Fatalf("Error converting apply object to unstructured: %v", err)
}
// Marshal the unstructured object into YAML
yamlData, err := yaml.Marshal(unstructuredObj.Object)
if err != nil {
log.Fatalf("Error marshaling unstructured object to YAML: %v", err)
}
return string(yamlData)
}
// Helper function to convert apply configuration to a map
func applyConfigToMap(applyConfig interface{}) map[string]interface{} {
data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(applyConfig)
if err != nil {
log.Fatalf("Error converting apply configuration to map: %v", err)
}
return data
}
// GenerateSA will create the service account for kube-vip
func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
func GenerateSA() *applyCoreV1.ServiceAccountApplyConfiguration {
kind := "ServiceAccount"
name := "kube-vip"
var namespace string
if c.ServiceNamespace != "" {
namespace = c.ServiceNamespace
} else {
namespace = metav1.NamespaceSystem
}
namespace := "kube-system"
newManifest := &applyCoreV1.ServiceAccountApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &corev1.SchemeGroupVersion.Version, Kind: &kind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
@@ -65,31 +30,15 @@ func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
}
// GenerateCR will generate the Cluster role for kube-vip
func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
var kind, name string
var namespace *string
if role {
kind = "Role"
name = "kube-vip"
if c.ServiceNamespace != "" {
namespace = &c.ServiceNamespace
} else {
// If the namespace is empty then we need to set it to the system namespace
copiedNamespace := metav1.NamespaceSystem
namespace = &copiedNamespace
}
} else {
kind = "ClusterRole"
name = "system:kube-vip-role"
}
func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
name := "system:kube-vip-role"
roleRefKind := "ClusterRole"
apiVersion := "rbac.authorization.k8s.io/v1"
newManifest := &applyRbacV1.RoleApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
newManifest := &applyRbacV1.ClusterRoleApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &roleRefKind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
Name: &name,
Namespace: namespace,
Name: &name,
},
Rules: []applyRbacV1.PolicyRuleApplyConfiguration{
{
@@ -100,7 +49,7 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
{
APIGroups: []string{""},
Resources: []string{"services", "endpoints"},
Verbs: []string{"list", "get", "watch", "update"},
Verbs: []string{"list", "get", "watch", "endoints"},
},
{
APIGroups: []string{""},
@@ -112,54 +61,38 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
Resources: []string{"leases"},
Verbs: []string{"list", "get", "watch", "update", "create"},
},
{
APIGroups: []string{"discovery.k8s.io"},
Resources: []string{"endpointslices"},
Verbs: []string{"list", "get", "watch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"pods"},
Verbs: []string{"list"},
},
},
}
return newManifest
}
// GenerateCRB will generate the clusterRoleBinding or rolebinding
func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountApplyConfiguration, crCfg *applyRbacV1.RoleApplyConfiguration) *applyRbacV1.RoleBindingApplyConfiguration {
// GenerateCRB will generate the clusterRoleBinding
func GenerateCRB() *applyRbacV1.ClusterRoleBindingApplyConfiguration {
kind := "ClusterRoleBinding"
apiVersion := "rbac.authorization.k8s.io/v1"
subjectKind := "ServiceAccount"
apiGroup := "rbac.authorization.k8s.io"
var kind, bindName string
var namespace, objectNamespace *string
if rolebinding {
kind = "RoleBinding"
bindName = "kube-vip"
namespace = nil
objectNamespace = saCfg.Namespace
} else {
kind = "ClusterRoleBinding"
bindName = "system:kube-vip-binding"
namespace = saCfg.Namespace
objectNamespace = nil
}
newManifest := &applyRbacV1.RoleBindingApplyConfiguration{
roleRefKind := "ClusterRole"
roleRefName := "system:kube-vip-role"
name := "kube-vip"
bindName := "system:kube-vip-role-binding"
namespace := "kube-system"
newManifest := &applyRbacV1.ClusterRoleBindingApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
Name: &bindName,
Namespace: objectNamespace,
Name: &bindName,
},
RoleRef: &applyRbacV1.RoleRefApplyConfiguration{
APIGroup: &apiGroup,
Kind: crCfg.Kind,
Name: crCfg.Name,
Kind: &roleRefKind,
Name: &roleRefName,
},
Subjects: []applyRbacV1.SubjectApplyConfiguration{
{
Kind: saCfg.Kind,
Name: saCfg.Name,
Namespace: namespace,
Kind: &subjectKind,
Name: &name,
Namespace: &namespace,
},
},
}
@@ -228,6 +161,18 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
newEnvironment = append(newEnvironment, svcInterface...)
}
// If a CIDR is used add it to the manifest
if c.VIPCIDR != "" {
// build environment variables
cidr := []corev1.EnvVar{
{
Name: vipCidr,
Value: c.VIPCIDR,
},
}
newEnvironment = append(newEnvironment, cidr...)
}
// If a subnet is required for the VIP
if c.VIPSubnet != "" {
// build environment variables
@@ -386,6 +331,40 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
// If we're specifying a configuration
if c.ProviderConfig != "" {
provider := []corev1.EnvVar{
{
Name: providerConfig,
Value: c.ProviderConfig,
},
}
newEnvironment = append(newEnvironment, provider...)
}
// If Equinix Metal is enabled then add it to the manifest
if c.EnableMetal {
packet := []corev1.EnvVar{
{
Name: vipPacket,
Value: strconv.FormatBool(c.EnableMetal),
},
{
Name: vipPacketProject,
Value: c.MetalProject,
},
{
Name: vipPacketProjectID,
Value: c.MetalProjectID,
},
{
Name: "PACKET_AUTH_TOKEN",
Value: c.MetalAPIKey,
},
}
newEnvironment = append(newEnvironment, packet...)
}
// Detect and enable wireguard mode
if c.EnableWireguard {
wireguard := []corev1.EnvVar{
@@ -407,7 +386,8 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, routingtable...)
}
// If BGP
// If BGP, but we're not using Equinix Metal
if c.EnableBGP {
bgp := []corev1.EnvVar{
{
@@ -417,9 +397,8 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, bgp...)
}
// If BGP
if c.EnableBGP {
// If BGP, but we're not using Equinix Metal
if c.EnableBGP && !c.EnableMetal {
bgpConfig := []corev1.EnvVar{
{
Name: bgpRouterID,
@@ -521,10 +500,10 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, prometheus...)
if c.EnableEndpoints {
if c.EnableEndpointSlices {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: enableEndpoints,
Value: strconv.FormatBool(c.EnableEndpoints),
Name: enableEndpointSlices,
Value: strconv.FormatBool(c.EnableEndpointSlices),
})
}
@@ -549,16 +528,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
newEnvironment = append(newEnvironment, mdif...)
}
if c.HealthCheckPort != 0 {
healthPort := []corev1.EnvVar{
{
Name: healthCheckPort,
Value: fmt.Sprintf("%d", c.HealthCheckPort),
},
}
newEnvironment = append(newEnvironment, healthPort...)
}
var securityContext *corev1.SecurityContext
if c.LoadBalancerForwardingMethod == "masquerade" {
var privileged = true
@@ -572,9 +541,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
"NET_ADMIN",
"NET_RAW",
},
Drop: []corev1.Capability{
"ALL",
},
},
}
}
@@ -633,6 +599,26 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
}
if c.ProviderConfig != "" {
providerConfigMount := corev1.VolumeMount{
Name: "cloud-sa-volume",
MountPath: "/etc/cloud-sa",
ReadOnly: true,
}
newManifest.Spec.Containers[0].VolumeMounts = append(newManifest.Spec.Containers[0].VolumeMounts, providerConfigMount)
providerConfigVolume := corev1.Volume{
Name: "cloud-sa-volume",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "metal-cloud-config",
},
},
}
newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, providerConfigVolume)
}
return newManifest
}

View File

@@ -3,8 +3,7 @@ package kubevip
import (
"fmt"
log "log/slog"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
@@ -45,8 +44,8 @@ func isValidInterface(iface string) error {
// userspace has set operational state. Interface must be considered for user
// data as setting operational state has not been implemented in every driver."
if attrs.OperState == netlink.OperUnknown {
log.Warn(
"the status of the interface is unknown. Ensure your interface is ready to accept traffic, if so you can safely ignore this message", "interface",
log.Warningf(
"the status of the interface %s is unknown. Ensure your interface is ready to accept traffic, if so you can safely ignore this message",
iface,
)
} else if attrs.OperState != netlink.OperUp {

View File

@@ -7,7 +7,7 @@ import (
// Config defines all of the settings for the Kube-Vip Pod
type Config struct {
// Logging, settings
Logging int32 `yaml:"logging"`
Logging int `yaml:"logging"`
// EnableARP, will use ARP to advertise the VIP address
EnableARP bool `yaml:"enableARP"`
@@ -76,11 +76,14 @@ type Config struct {
// VipSubnet is the Subnet that is applied to the VIP
VIPSubnet string `yaml:"vipSubnet"`
// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
VIPCIDR string `yaml:"vipCidr"`
// Address is the IP or DNS Name to use as a VirtualIP
Address string `yaml:"address"`
// Listen port for the VirtualIP
Port uint16 `yaml:"port"`
Port int `yaml:"port"`
// Namespace will define which namespace the control plane pods will run in
Namespace string `yaml:"namespace"`
@@ -110,7 +113,7 @@ type Config struct {
EnableLoadBalancer bool `yaml:"enableLoadBalancer"`
// Listen port for the IPVS Service
LoadBalancerPort uint16 `yaml:"lbPort"`
LoadBalancerPort int `yaml:"lbPort"`
// Forwarding method for the IPVS Service
LoadBalancerForwardingMethod string `yaml:"lbForwardingMethod"`
@@ -132,6 +135,21 @@ type Config struct {
BGPPeerConfig bgp.Peer
BGPPeers []string
// EnableMetal, will use the metal API to update the EIP <-> VIP (if BGP is enabled then BGP will be used)
EnableMetal bool `yaml:"enableMetal"`
// MetalAPIKey, is the API token used to authenticate to the API
MetalAPIKey string
// MetalProject, is the name of a particular defined project
MetalProject string
// MetalProjectID, is the name of a particular defined project
MetalProjectID string
// ProviderConfig, is the path to a provider configuration file
ProviderConfig string
// LoadBalancers are the various services we can load balance over
LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
@@ -161,8 +179,8 @@ type Config struct {
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
EnableEndpoints bool `yaml:"enableEndpoints"`
// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
// MirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
@@ -177,15 +195,6 @@ type Config struct {
// LoInterfaceGlobalScope, if true will set global scope when using the lo interface, otherwise a host scope will be used
LoInterfaceGlobalScope bool `yaml:"loInterfaceGlobalScope"`
// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
HealthCheckPort int `yaml:"healthCheckPort"`
// EnableUPNP, enables UPNP functions
EnableUPNP bool `yaml:"enableUPNP"`
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

View File

@@ -1,23 +1,17 @@
package loadbalancer
import (
"context"
"errors"
"fmt"
"net"
"net/netip"
"os"
"strings"
"sync"
log "log/slog"
"github.com/cloudflare/ipvs"
"github.com/cloudflare/ipvs/netmask"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
log "github.com/sirupsen/logrus"
)
/*
@@ -45,51 +39,44 @@ const (
type IPVSLoadBalancer struct {
client ipvs.Client
loadBalancerService ipvs.Service
Port uint16
Port int
forwardingMethod ipvs.ForwardType
backendMap backend.Map
interval int
lock sync.Mutex
stop chan struct{}
networkInterface string
leaderCancel context.CancelFunc
signal chan os.Signal
address string
family ipvs.AddressFamily
}
func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
log.Info("Starting IPVS LoadBalancer", "address", address)
func NewIPVSLB(address string, port int, forwardingMethod string, backendHealthCheckInterval int) (*IPVSLoadBalancer, error) {
// Create IPVS client
c, err := ipvs.New()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("Error starting IPVS", "err", err)
panic("")
log.Errorf("ensure IPVS kernel modules are loaded")
log.Fatalf("Error starting IPVS [%v]", err)
}
i, err := c.Info()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("Error retrieving IPVS info", "err", err)
if errors.Is(err, os.ErrPermission) {
log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
}
panic("")
log.Errorf("ensure IPVS kernel modules are loaded")
log.Fatalf("Error getting IPVS version [%v]", err)
}
log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
ip, family := ipAndFamily(address)
log.Infof("IPVS Loadbalancer enabled for %d.%d.%d", i.Version[0], i.Version[1], i.Version[2])
if strings.ToLower(forwardingMethod) == "masquerade" {
enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
if family == ipvs.INET6 {
enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
} else {
enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/vs/conntrack", "1")
if err != nil {
log.Fatalf("Error ensuring net.ipv4.vs.conntrack enabled [%v]", err)
}
log.Infof("sysctl set net.ipv4.vs.conntrack to 1")
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/ip_forward", "1")
if err != nil {
log.Fatalf("Error ensuring net.ipv4.ip_forward enabled [%v]", err)
}
log.Infof("sysctl set net.ipv4.ip_forward to 1")
}
ip, family := ipAndFamily(address)
netMask := netmask.MaskFrom(31, 32) // For ipv4
if family == ipvs.INET6 {
netMask = netmask.MaskFrom(128, 128) // For ipv6
@@ -100,7 +87,7 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
Netmask: netMask,
Family: family,
Protocol: ipvs.TCP,
Port: port,
Port: uint16(port),
Address: ip,
Scheduler: ROUNDROBIN,
}
@@ -119,7 +106,7 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
m = ipvs.Bypass
default:
m = ipvs.Local
log.Warn("unknown forwarding method. Defaulting to Local")
log.Warnf("unknown forwarding method. Defaulting to Local")
}
if backendHealthCheckInterval <= 0 {
@@ -133,33 +120,17 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
forwardingMethod: m,
interval: backendHealthCheckInterval,
backendMap: make(backend.Map),
stop: make(chan struct{}),
networkInterface: networkInterface,
leaderCancel: leaderCancel,
signal: signal,
address: address,
family: family,
}
go lb.healthCheck()
if strings.ToLower(forwardingMethod) == "masquerade" {
go lb.healthCheck()
}
// Return our created load-balancer
return lb, nil
}
func enableProcSys(path, name string) {
isSet, err := sysctl.EnableProcSys(path)
if err != nil {
log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
panic("")
}
if isSet {
log.Info(fmt.Sprintf("sysctl set %s to 1", name))
}
}
func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
log.Info("Stopping IPVS LoadBalancer", "address", lb.address)
close(lb.stop)
err := lb.client.RemoveService(lb.loadBalancerService)
if err != nil {
@@ -168,31 +139,15 @@ func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
return nil
}
func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
isLocal := false
var err error
// Discard backend if it is of different IP family than LB address.
if _, family := ipAndFamily(address); family != lb.family {
return nil
}
if lb.forwardingMethod == ipvs.Local {
log.Info("checking if backend is local", "addr", address)
isLocal, err = lb.isLocal(address)
if err != nil {
log.Error("checking if backend is local", "err", err)
}
}
backend := backend.Entry{Addr: address, Port: port, IsLocal: isLocal}
func (lb *IPVSLoadBalancer) AddBackend(address string, port int) error {
backend := backend.Entry{Addr: address, Port: port}
lb.lock.Lock()
defer lb.lock.Unlock()
if _, ok := lb.backendMap[backend]; !ok {
isHealth := backend.Check()
if isHealth {
err := lb.addBackend(address, port)
err := lb.addBackend(backend)
if err != nil {
return err
}
@@ -202,19 +157,18 @@ func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
return nil
}
func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
backend := backend.Entry{Addr: address, Port: port}
func (lb *IPVSLoadBalancer) addBackend(backend backend.Entry) error {
// Check if this is the first backend
backends, err := lb.client.Destinations(lb.loadBalancerService)
if err != nil && strings.Contains(err.Error(), "file does not exist") {
log.Error("querying backends", "err", err)
log.Errorf("Error querying backends %s", err)
}
// If this is our first backend, then we can create the load-balancer service and add a backend
if len(backends) == 0 {
err = lb.client.CreateService(lb.loadBalancerService)
// If we've an error it could be that the IPVS lb instance has been left from a previous leadership
if err != nil && strings.Contains(err.Error(), "file exists") {
log.Warn("load balancer for API server already exists, attempting to remove and re-create")
log.Warnf("load balancer for API server already exists, attempting to remove and re-create")
err = lb.client.RemoveService(lb.loadBalancerService)
if err != nil {
return fmt.Errorf("error re-creating IPVS service: %v", err)
@@ -225,12 +179,10 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
}
} else if err != nil {
// Fatal error at this point as IPVS is probably not working
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Error("IPVS service", "err", err)
panic("")
log.Errorf("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Fatalf("IPVS service error: %v", err)
}
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
log.Infof("Created Load-Balancer services on [%s:%d]", lb.addrString(), lb.Port)
}
ip, family := ipAndFamily(backend.Addr)
@@ -243,7 +195,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
dst := ipvs.Destination{
Address: ip,
Port: backend.Port,
Port: uint16(backend.Port),
Family: family,
Weight: 1,
FwdMethod: lb.forwardingMethod,
@@ -259,12 +211,12 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
// file exists is fine, we will just return at this point
return nil
}
log.Info("backend added", "src addr", lb.addrString(), "src port", lb.Port, "dst addr", backend.Addr, "dst port", backend.Port)
log.Infof("Added backend for [%s:%d] on [%s:%d]", lb.addrString(), lb.Port, backend.Addr, backend.Port)
return nil
}
func (lb *IPVSLoadBalancer) RemoveBackend(address string, port uint16) error {
func (lb *IPVSLoadBalancer) RemoveBackend(address string, port int) error {
backend := backend.Entry{Addr: address, Port: port}
lb.lock.Lock()
@@ -281,7 +233,7 @@ func (lb *IPVSLoadBalancer) RemoveBackend(address string, port uint16) error {
return nil
}
func (lb *IPVSLoadBalancer) removeBackend(address string, port uint16) error {
func (lb *IPVSLoadBalancer) removeBackend(address string, port int) error {
ip, family := ipAndFamily(address)
if family != lb.loadBalancerService.Family {
return nil
@@ -289,12 +241,12 @@ func (lb *IPVSLoadBalancer) removeBackend(address string, port uint16) error {
dst := ipvs.Destination{
Address: ip,
Port: port,
Port: uint16(port),
Family: family,
Weight: 1,
}
err := lb.client.RemoveDestination(lb.loadBalancerService, dst)
if err != nil && !errors.Is(err, os.ErrNotExist) {
if err != nil {
return fmt.Errorf("error removing backend: %v", err)
}
return nil
@@ -315,77 +267,29 @@ func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
func (lb *IPVSLoadBalancer) healthCheck() {
backend.Watch(func() {
lb.lock.Lock()
defer lb.lock.Unlock()
for backend, oldStatus := range lb.backendMap {
newStatus := backend.Check()
if newStatus {
// old status -> health
if !oldStatus {
err := lb.addBackend(backend.Addr, backend.Port)
err := lb.AddBackend(backend.Addr, backend.Port)
if err != nil {
log.Error("add backend", "err", err)
log.Errorf("failed to add backend: %s", err)
}
lb.backendMap[backend] = newStatus
}
} else {
// old status -> not health
if oldStatus {
log.Info("healthCheck failed - removing backend", "address", backend.Addr, "port", backend.Port)
log.Infof("healthCheck failed for backend %s:%d, attempting to remove from load balancer", backend.Addr, backend.Port)
err := lb.removeBackend(backend.Addr, backend.Port)
if err != nil {
log.Error("failed to remove backend", "address", backend.Addr, "port", backend.Port, "err", err)
log.Errorf("failed to remove backend %s:%d: %s", backend.Addr, backend.Port, err)
}
lb.backendMap[backend] = newStatus
}
if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
if lb.signal != nil {
close(lb.signal)
}
if lb.leaderCancel != nil {
lb.leaderCancel()
}
}
}
}
lb.lock.Unlock()
}, lb.interval, lb.stop)
}
func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
link, err := netlink.LinkByName(lb.networkInterface)
if err != nil {
return false, fmt.Errorf("getting link '%s': %w", lb.networkInterface, err)
}
family := netlink.FAMILY_V6
if vip.IsIPv4(address) {
family = netlink.FAMILY_V4
}
target := net.ParseIP(address)
if target == nil {
return false, fmt.Errorf("address '%s' is not a valid IP address", address)
}
addrs, err := netlink.AddrList(link, family)
if err != nil {
return false, fmt.Errorf("listing addresses for link '%s': %w", lb.networkInterface, err)
}
for _, addr := range addrs {
if addr.IP.Equal(target) {
return true, nil
}
}
return false, nil
}
func (lb *IPVSLoadBalancer) localBackendExists() bool {
for backend, isHealthy := range lb.backendMap {
if backend.IsLocal && isHealthy {
return true
}
}
return false
}

View File

@@ -1,56 +1,54 @@
package instance
package manager
import (
"fmt"
"net"
"strconv"
"strings"
"log/slog"
log "log/slog"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Instance defines an instance of everything needed to manage vips
type Instance struct {
// Virtual IP / Load Balancer configuration
VIPConfigs []*kubevip.Config
vipConfigs []*kubevip.Config
// cluster instances
Clusters []*cluster.Cluster
clusters []*cluster.Cluster
// Service uses DHCP
IsDHCP bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPHostname string
DHCPClient *vip.DHCPClient
isDHCP bool
dhcpInterface string
dhcpInterfaceHwaddr string
dhcpInterfaceIP string
dhcpHostname string
dhcpClient *vip.DHCPClient
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
upnpGatewayIPs []string
// Kubernetes service mapping
ServiceSnapshot *v1.Service
VIPs []string
UID string
ExternalPorts []Port
serviceSnapshot *v1.Service
}
type Port struct {
Port uint16
Port int32
Type string
}
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses := FetchServiceAddresses(svc)
//instanceUID := string(svc.UID)
func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
instanceAddresses := fetchServiceAddresses(svc)
instanceUID := string(svc.UID)
var newVips []*kubevip.Config
var link netlink.Link
@@ -59,14 +57,14 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
svcInterface = svc.Annotations[serviceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
log.Errorf("failed to automatically discover network interface for annotated IP address [%s] with error: %s", address, err.Error())
} else {
if link == nil {
log.Error("automatically discover network interface for annotated IP address", "address", address)
log.Errorf("failed to automatically discover network interface for annotated IP address [%s]", address)
}
}
if link == nil {
@@ -81,9 +79,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
case kubevip.Auto:
link, err = autoFindInterface(address)
if err != nil {
log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
log.Errorf("failed to automatically discover network interface for IP address [%s] with error: %s - defaulting to: %s", address, err.Error(), config.Interface)
} else if link == nil {
log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
log.Errorf("failed to automatically discover network interface for IP address [%s] - defaulting to: %s", address, config.Interface)
}
svcInterface = getAutoInterfaceName(link, config.Interface)
case "":
@@ -102,7 +100,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
cidrs := vip.Split(config.VIPSubnet)
cidrs := vip.Split(config.VIPCIDR)
ipv4AutoSubnet := false
ipv6AutoSubnet := false
@@ -127,7 +125,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
if cidrs[0] != "" {
subnet = cidrs[0]
} else {
subnet = "32"
@@ -140,7 +138,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
if len(cidrs) > 1 {
subnet = cidrs[1]
} else {
subnet = "128"
@@ -148,6 +146,8 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
//log.Info("new instance", "svc", *svc, "interface", svcInterface)
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
@@ -155,7 +155,8 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: subnet,
VIPCIDR: subnet,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
@@ -173,27 +174,27 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
// Create new service
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
UID: instanceUID,
VIPs: instanceAddresses,
serviceSnapshot: svc,
}
for _, port := range svc.Spec.Ports {
instance.ExternalPorts = append(instance.ExternalPorts, Port{
Port: port.Port,
Type: string(port.Protocol),
})
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
// Port: uint16(port.Port), //nolint
// Type: string(port.Protocol),
// })
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
instance.dhcpInterfaceHwaddr = svc.Annotations[hwAddrKey]
instance.dhcpInterfaceIP = svc.Annotations[requestedIP]
instance.dhcpHostname = svc.Annotations[loadbalancerHostname]
}
configPorts := make([]kubevip.Port, 0)
for _, p := range svc.Spec.Ports {
for _, p := range instance.ExternalPorts {
configPorts = append(configPorts, kubevip.Port{
Type: string(p.Protocol),
Type: p.Type,
Port: int(p.Port),
})
}
@@ -208,7 +209,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
vip.LoadBalancers = append(vip.LoadBalancers, newLB)
}
// Create Add configuration to the new service
instance.VIPConfigs = newVips
instance.vipConfigs = newVips
// If this was purposely created with the address 0.0.0.0,
// we will create a macvlan on the main interface and a DHCP client
@@ -219,20 +220,20 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
case err := <-instance.dhcpClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
instance.serviceSnapshot.Namespace, instance.serviceSnapshot.Name, err)
case ip := <-instance.dhcpClient.IPChannel():
instance.vipConfigs[0].Interface = instance.dhcpInterface
instance.vipConfigs[0].VIP = ip
instance.dhcpInterfaceIP = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
for _, vipConfig := range instance.vipConfigs {
c, err := cluster.InitCluster(vipConfig, false)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
log.Errorf("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
}
@@ -240,8 +241,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
c.Network[i].SetServicePorts(svc)
}
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
instance.clusters = append(instance.clusters, c)
log.Infof("(svcs) adding VIP [%s] via %s for [%s/%s]", vipConfig.VIP, vipConfig.Interface, svc.Namespace, svc.Name)
}
return instance, nil
@@ -305,24 +307,24 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
}
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
if len(i.vipConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.vipConfigs))
}
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
parent, err := netlink.LinkByName(i.vipConfigs[0].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
// Generate name from UID
interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
interfaceName := fmt.Sprintf("vip-%s", i.UID[0:8])
// Check if the interface doesn't exist first
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
log.Infof("Creating new macvlan interface for DHCP [%s]", interfaceName)
hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
if i.DHCPInterfaceHwaddr != "" && err != nil {
hwaddr, err := net.ParseMAC(i.dhcpInterfaceHwaddr)
if i.dhcpInterfaceHwaddr != "" && err != nil {
return err
} else if hwaddr == nil {
hwaddr, err = net.ParseMAC(vip.GenerateMac())
@@ -331,7 +333,7 @@ func (i *Instance) startDHCP() error {
}
}
log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
log.Infof("New interface [%s] mac is %s", interfaceName, hwaddr)
mac := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: interfaceName,
@@ -356,127 +358,31 @@ func (i *Instance) startDHCP() error {
return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
}
} else {
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
log.Infof("Using existing macvlan interface for DHCP [%s]", interfaceName)
}
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
if i.dhcpInterfaceIP != "" {
initRebootFlag = true
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
client := vip.NewDHCPClient(iface, initRebootFlag, i.dhcpInterfaceIP)
// Add hostname to dhcp client if annotated
if i.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
client.WithHostName(i.DHCPHostname)
if i.dhcpHostname != "" {
log.Infof("Hostname specified for dhcp lease: [%s] - [%s]", interfaceName, i.dhcpHostname)
client.WithHostName(i.dhcpHostname)
}
go client.Start()
// Set that DHCP is enabled
i.IsDHCP = true
i.isDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
i.dhcpInterface = interfaceName
i.dhcpInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
i.dhcpClient = client
return nil
}
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
// If the service has no status, return empty
lbStatusAddresses := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
// TODO: Handle hostname if needed
}
return lbStatusAddresses
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs
}
}
lbStatusAddresses := []string{}
if !annotationAvailable {
if len(s.Status.LoadBalancer.Ingress) > 0 {
for _, ingress := range s.Status.LoadBalancer.Ingress {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
}
}
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
if len(lbStatusAddresses) > 0 {
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}
}
}
return lbStatusAddresses
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}
}
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
return nil
}

View File

@@ -3,26 +3,24 @@ package manager
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -39,6 +37,11 @@ type Manager struct {
// Manager services
// service bool
// Keeps track of all running instances
serviceInstances []*Instance
// UPNP functionality
upnp bool
// BGP Manager, this is a singleton that manages all BGP advertisements
bgpServer *bgp.Server
@@ -48,8 +51,6 @@ type Manager struct {
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
@@ -60,12 +61,6 @@ type Manager struct {
// This mutex is to protect calls from various goroutines
mutex sync.Mutex
// This tracks used network interfaces and guards them with mutex for concurrent changes.
intfMgr *networkinterface.Manager
// This tracks VIPs and performs ARP/NDP advertisement.
arpMgr *arp.Manager
}
// New will create a new managing object
@@ -77,14 +72,14 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// we set it to hostname as a fallback.
// This mimics legacy behavior and should work on old kube-vip installations.
if config.NodeName == "" {
log.Warn("Node name is missing from the config, fall back to hostname")
log.Warning("Node name is missing from the config, fall back to hostname")
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("could not get hostname: %v", err)
}
config.NodeName = hostname
}
log.Info("using node name", "name", config.NodeName)
log.Infof("Using node name [%v]", config.NodeName)
adminConfigPath := "/etc/kubernetes/admin.conf"
homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
@@ -98,7 +93,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// Do nothing, we don't construct a k8s client for etcd leader election
case utils.FileExists(adminConfigPath):
if config.KubernetesAddr != "" {
log.Info("k8s address", "address", config.KubernetesAddr)
log.Infof("k8s address [%s]", config.KubernetesAddr)
clientConfig, err = k8s.NewRestConfig(adminConfigPath, false, config.KubernetesAddr)
} else if config.EnableControlPlane {
// If this is a control plane host it will likely have started as a static pod or won't have the
@@ -119,7 +114,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
if clientset, err = k8s.NewClientset(clientConfig); err != nil {
return nil, fmt.Errorf("could not create k8s clientset: %w", err)
}
log.Debug("Using external Kubernetes configuration from file", "path", adminConfigPath)
log.Debugf("Using external Kubernetes configuration from file: %q", adminConfigPath)
case utils.FileExists(homeConfigPath):
clientConfig, err = k8s.NewRestConfig(homeConfigPath, false, "")
if err != nil {
@@ -129,7 +124,7 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
if err != nil {
return nil, fmt.Errorf("could not create k8s clientset from external file: %q: %w", homeConfigPath, err)
}
log.Debug("Using external Kubernetes configuration from file", "path", adminConfigPath)
log.Debugf("Using external Kubernetes configuration from file [%s]", homeConfigPath)
default:
clientConfig, err = k8s.NewRestConfig("", true, "")
if err != nil {
@@ -171,32 +166,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// }
// }
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
var bgpServer *bgp.Server
if config.EnableBGP {
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
@@ -214,12 +183,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
@@ -238,26 +201,6 @@ func (sm *Manager) Start() error {
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
sm.shutdownChan = make(chan struct{})
// HealthCheck
if sm.config.HealthCheckPort != 0 {
if sm.config.HealthCheckPort < 1024 {
return fmt.Errorf("healthcheck port is using a port that is less than 1024 [%d]", sm.config.HealthCheckPort)
}
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "OK")
})
go func() {
server := &http.Server{
Addr: fmt.Sprintf(":%d", sm.config.HealthCheckPort),
ReadHeaderTimeout: 3 * time.Second,
}
err := server.ListenAndServe()
if err != nil {
log.Error("healthcheck", "unable to start", err)
}
}()
}
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
@@ -267,48 +210,49 @@ func (sm *Manager) Start() error {
return err
}
log.Info("Starting Kube-vip Manager with the BGP engine")
log.Infoln("Starting Kube-vip Manager with the BGP engine")
return sm.startBGP()
}
if sm.config.EnableARP || sm.config.EnableWireguard {
if sm.config.EnableUPNP {
// Before starting the leader Election enable any additional functionality
upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
if upnpEnabled {
sm.upnp = true
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
log.Errorf("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
sm.config.EnableUPNP = false
sm.upnp = false
} else {
for _, c := range clients {
ip, err := c.GetExternalIPAddress()
if err != nil {
log.Error("unable to find IGD2 Gateway address", "err", err)
}
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
log.Infof("Found UPNP IGD2 Gateway address[%s] error: [%s]", ip, err)
}
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.svcProcessor.RefreshUPNPForwards()
go sm.refreshUPNPForwards()
}
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
if sm.config.EnableARP {
log.Info("Starting Kube-vip Manager with the ARP engine")
log.Infoln("Starting Kube-vip Manager with the ARP engine")
return sm.startARP(sm.config.NodeName)
}
if sm.config.EnableWireguard {
log.Info("Starting Kube-vip Manager with the Wireguard engine")
log.Infoln("Starting Kube-vip Manager with the Wireguard engine")
return sm.startWireguard(sm.config.NodeName)
}
if sm.config.EnableRoutingTable {
log.Info("Starting Kube-vip Manager with the Routing Table engine")
log.Infoln("Starting Kube-vip Manager with the Routing Table engine")
return sm.startTableMode(sm.config.NodeName)
}
log.Error("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
log.Errorln("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
return nil
}
@@ -324,7 +268,7 @@ func returnNameSpace() (string, error) {
func (sm *Manager) parseAnnotations() error {
if sm.config.Annotations == "" {
log.Debug("No Node annotations to parse")
log.Debugf("No Node annotations to parse")
return nil
}
@@ -334,3 +278,65 @@ func (sm *Manager) parseAnnotations() error {
}
return nil
}
func (sm *Manager) serviceInterface() string {
svcIf := sm.config.Interface
if sm.config.ServicesInterface != "" {
svcIf = sm.config.ServicesInterface
}
return svcIf
}
func (sm *Manager) startTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Infof("mirroring traffic from interface %s to interface %s", svcIf, sm.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) stopTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Infof("clean up qdisc config on interface %s", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) findServiceInstance(svc *v1.Service) *Instance {
svcUID := string(svc.UID)
log.Debugf("service UID: %s", svcUID)
for i := range sm.serviceInstances {
log.Debugf("saved service instance %d UID: %s", i, sm.serviceInstances[i].UID)
if sm.serviceInstances[i].UID == svcUID {
return sm.serviceInstances[i]
}
}
return nil
}
// Refresh UPNP Port Forwards for all Service Instances registered in the SM
func (sm *Manager) refreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Infof("[UPNP] Refreshing %d Instances", len(sm.serviceInstances))
for i := range sm.serviceInstances {
sm.upnpMap(context.TODO(), sm.serviceInstances[i])
if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
log.Warnf("[UPNP] Error updating service IPs %s [%s]", sm.serviceInstances[i].serviceSnapshot.Name, err.Error())
}
}
}
}

View File

@@ -2,11 +2,11 @@ package manager
import (
"context"
"os"
"syscall"
"time"
log "log/slog"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -27,9 +27,6 @@ func (sm *Manager) startARP(id string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("Start ARP/NDP advertisement")
go sm.arpMgr.StartAdvertisement(ctx)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
@@ -44,7 +41,7 @@ func (sm *Manager) startARP(id string) error {
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return err
}
@@ -57,16 +54,17 @@ func (sm *Manager) startARP(id string) error {
go func() {
err := cpCluster.StartCluster(sm.config, clusterManager, nil)
if err != nil {
log.Error("starting control plane", "err", err)
log.Errorf("Control Plane Error [%v]", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.shutdownChan
log.Info("Shutting down Kube-Vip")
<-sm.signalChan
log.Infof("Shutting down Kube-Vip")
return nil
}
@@ -76,27 +74,27 @@ func (sm *Manager) startARP(id string) error {
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", sm.config.Namespace)
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}
}
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
if os.Getenv("EGRESS_CLEAN") != "" {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
log.Infof("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", sm.config.ServicesLeaseName, "id", id)
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, sm.config.ServicesLeaseName, id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
@@ -125,21 +123,21 @@ func (sm *Manager) startARP(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
log.Fatal(err)
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
sm.svcProcessor.Stop()
log.Infof("leader lost: %s", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here
log.Fatal("lost leadership, restarting kube-vip")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
@@ -150,7 +148,7 @@ func (sm *Manager) startARP(id string) error {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
log.Infof("new leader elected: %s", identity)
},
},
})

View File

@@ -3,14 +3,16 @@ package manager
import (
"context"
"fmt"
"os"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
api "github.com/osrg/gobgp/v3/api"
"github.com/packethost/packngo"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
// Start will begin the Manager, which will start services and watch the configmap
@@ -19,15 +21,37 @@ func (sm *Manager) startBGP() error {
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
// If Equinix Metal is enabled then we can begin our preparation work
var packetClient *packngo.Client
if sm.config.EnableMetal {
if sm.config.ProviderConfig != "" {
key, project, err := equinixmetal.GetPacketConfig(sm.config.ProviderConfig)
if err != nil {
return err
}
// Set the environment variable with the key for the project
os.Setenv("PACKET_AUTH_TOKEN", key)
// Update the configuration with the project key
sm.config.MetalProjectID = project
}
packetClient, err = packngo.NewClient()
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
return err
}
// We're using Equinix Metal with BGP, populate the Peer information from the API
if sm.config.EnableBGP {
log.Infoln("Looking up the BGP configuration from Equinix Metal")
err = equinixmetal.BGPLookup(packetClient, sm.config)
if err != nil {
return err
}
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
@@ -43,8 +67,9 @@ func (sm *Manager) startBGP() error {
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
})
if err != nil {
return err
}
// use a Go context so we can tell the leaderelection code when we
@@ -73,7 +98,7 @@ func (sm *Manager) startBGP() error {
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return err
}
@@ -87,10 +112,10 @@ func (sm *Manager) startBGP() error {
if sm.config.EnableLeaderElection {
err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
} else {
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer, packetClient)
}
if err != nil {
log.Error("Control Plane", "err", err)
log.Errorf("Control Plane Error [%v]", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
@@ -99,18 +124,18 @@ func (sm *Manager) startBGP() error {
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.signalChan
log.Info("Shutting down Kube-Vip")
log.Infof("Shutting down Kube-Vip")
return nil
}
}
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
return err
}
log.Info("Shutting down Kube-Vip")
log.Infof("Shutting down Kube-Vip")
return nil
}

View File

@@ -3,15 +3,15 @@ package manager
import (
"context"
"fmt"
"os"
"strconv"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
@@ -27,22 +27,29 @@ func (sm *Manager) startTableMode(id string) error {
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("destination for routes", "table", sm.config.RoutingTableID, "protocol", sm.config.RoutingProtocol)
log.Infof("all routing table entries will exist in table [%d] with protocol [%d]", sm.config.RoutingTableID, sm.config.RoutingProtocol)
if sm.config.CleanRoutingTable {
go func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
if err := sm.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
log.Errorf("error checking for old routes: %v", err)
}
}()
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
egressCleanEnv := os.Getenv("EGRESS_CLEAN")
if egressCleanEnv != "" {
egressClean, err := strconv.ParseBool(egressCleanEnv)
if err != nil {
log.Warnf("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
}
if egressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPTables rules cleaned on startup")
}
}
// Shutdown function that will wait on this signal, unless we call it ourselves
@@ -58,7 +65,7 @@ func (sm *Manager) startTableMode(id string) error {
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
@@ -68,29 +75,29 @@ func (sm *Manager) startTableMode(id string) error {
return fmt.Errorf("cluster manager initialization error: %w", err)
}
if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
log.Error("Control Plane", "err", err)
if err := cpCluster.StartVipService(sm.config, clusterManager, nil, nil); err != nil {
log.Errorf("Control Plane Error [%v]", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
} else {
ns, err := returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
log.Infof("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else if sm.config.EnableLeaderElection {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, plunderLock, id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
@@ -118,21 +125,21 @@ func (sm *Manager) startTableMode(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error(err.Error())
panic("")
log.Fatal(err)
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Infof("leader lost: %s", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("")
log.Fatal("lost leadership, restarting kube-vip")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
@@ -140,15 +147,15 @@ func (sm *Manager) startTableMode(id string) error {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
log.Infof("new leader elected: %s", identity)
},
},
})
} else {
log.Info("beginning watching services without leader election")
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
log.Infof("beginning watching services without leader election")
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
log.Errorf("Cannot watch services, %v", err)
}
}
}
@@ -157,8 +164,6 @@ func (sm *Manager) startTableMode(id string) error {
}
func (sm *Manager) cleanRoutes() error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
@@ -169,17 +174,41 @@ func (sm *Manager) cleanRoutes() error {
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
for n := range cluster.Network {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == routes[i].Dst.String() {
found = true
}
}
}
}
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] deletion", "route", routes[i], "err", err)
log.Errorf("[route] error deleting route: %v", routes[i])
}
log.Debug("[route] deletion", "route", routes[i])
log.Debugf("[route] deleted route: %v", routes[i])
}
}
return nil
}
func (sm *Manager) countRouteReferences(route *netlink.Route) int {
cnt := 0
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
for n := range cluster.Network {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
return cnt
}

View File

@@ -4,9 +4,8 @@ import (
"context"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/wireguard"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -21,7 +20,7 @@ func (sm *Manager) startWireguard(id string) error {
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("reading wireguard peer configuration from Kubernetes secret")
log.Infoln("reading wireguard peer configuration from Kubernetes secret")
s, err := sm.clientSet.CoreV1().Secrets(sm.config.Namespace).Get(ctx, "wireguard", metav1.GetOptions{})
if err != nil {
return err
@@ -48,21 +47,21 @@ func (sm *Manager) startWireguard(id string) error {
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
log.Infof("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, plunderLock, id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
@@ -91,21 +90,21 @@ func (sm *Manager) startWireguard(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error(err.Error())
panic("")
log.Fatal(err)
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Infof("leader lost: %s", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("")
log.Fatal("lost leadership, restarting kube-vip")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
@@ -113,7 +112,7 @@ func (sm *Manager) startWireguard(id string) error {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
log.Infof("new leader elected: %s", identity)
},
},
})

View File

@@ -5,8 +5,7 @@ import (
"encoding/json"
"fmt"
log "log/slog"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
@@ -29,23 +28,24 @@ func applyNodeLabel(clientSet *kubernetes.Clientset, address, id, identity strin
ctx := context.Background()
node, err := clientSet.CoreV1().Nodes().Get(ctx, id, metav1.GetOptions{})
if err != nil {
log.Error("can't query node labels", "node", id, "err", err)
log.Errorf("can't query node %s labels. error: %v", id, err)
return
}
log.Debug(fmt.Sprintf("node %s labels: %+v", id, node.Labels))
log.Debugf("node %s labels: %+v", id, node.Labels)
value, ok := node.Labels[nodeLabelIndex]
path := fmt.Sprintf("/metadata/labels/%s", nodeLabelJSONPath)
log.Debug(fmt.Sprintf("Received identity: %s - id: %s", identity, id))
if ok && value == address {
log.Debug(fmt.Sprintf("removing node label `has-ip=%s` on %s", address, id))
if (!ok || value != address) && id == identity {
log.Debugf("setting node label `has-ip=%s` on %s", address, id)
// Append label
applyPatchLabels(ctx, clientSet, id, "add", path, address)
} else if ok && value == address {
log.Debugf("removing node label `has-ip=%s` on %s", address, id)
// Remove label
applyPatchLabels(ctx, clientSet, id, "remove", path, address)
} else {
log.Debug(fmt.Sprintf("setting node label `has-ip=%s` on %s", address, id))
// Append label
applyPatchLabels(ctx, clientSet, id, "add", path, address)
log.Debugf("no node label change needed")
}
}
@@ -59,15 +59,15 @@ func applyPatchLabels(ctx context.Context, clientSet *kubernetes.Clientset,
}}
patchData, err := json.Marshal(patchLabels)
if err != nil {
log.Error("node patch marshaling failed", "err", err)
log.Errorf("node patch marshaling failed. error: %v", err)
return
}
// patch node
node, err := clientSet.CoreV1().Nodes().Patch(ctx,
name, types.JSONPatchType, patchData, metav1.PatchOptions{})
if err != nil {
log.Error("node patching failed", "err", err)
log.Errorf("can't patch node %s. error: %v", name, err)
return
}
log.Debug("updated", "node", name, "labels", node.Labels)
log.Debugf("updated node %s labels: %+v", name, node.Labels)
}

View File

@@ -4,12 +4,5 @@ import "github.com/prometheus/client_golang/prometheus"
// PrometheusCollector defines a service watch event counter.
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
collectors := []prometheus.Collector{}
if sm.svcProcessor != nil {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
}

View File

@@ -1,4 +1,4 @@
package services
package manager
import (
"bufio"
@@ -8,12 +8,9 @@ import (
"os"
"strings"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
log "github.com/sirupsen/logrus"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -23,7 +20,7 @@ const (
defaultServiceCIDR = "10.96.0.0/12"
)
func (p *Processor) iptablesCheck() error {
func (sm *Manager) iptablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
@@ -50,31 +47,6 @@ func (p *Processor) iptablesCheck() error {
return nil
}
func (p *Processor) nftablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
var queue, ct bool
for scanner.Scan() {
line := strings.Fields(scanner.Text())
switch line[0] {
case "nft_queue":
queue = true
case "nft_ct":
ct = true
}
}
if !queue || !ct {
return fmt.Errorf("missing nftables modules -> nft_ct [%t] -> ntf_queue [%t]", ct, queue)
}
return nil
}
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
cidrs := strings.Split(sourceCidrs, ",")
isV6 := vip.IsIPv6(ip)
@@ -86,7 +58,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
matchingFamily = append(matchingFamily, cidr)
selectedCIDR, err := checkCIDR(ip, cidr)
if err != nil {
log.Warn("IPv6 CIDR check ", "err", err)
log.Warnf("IPv6 CIDR check failed: %s", err.Error())
continue
}
if selectedCIDR != "" {
@@ -98,7 +70,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
matchingFamily = append(matchingFamily, cidr)
selectedCidr, err := checkCIDR(ip, cidr)
if err != nil {
log.Warn("IPv4 CIDR check ", "err", err)
log.Warnf("IPv4 CIDR check failed: %s", err.Error())
continue
}
if selectedCidr != "" {
@@ -134,27 +106,21 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
func (sm *Manager) configureEgress(vipIP, podIP, destinationPorts, namespace string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
}
if discoverErr != nil {
log.Warn("autodiscover CIDR", "err", discoverErr)
log.Warn(discoverErr)
}
if p.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
if sm.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
} else {
if discoverErr == nil {
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
@@ -164,13 +130,13 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
if podCidr == "" {
// There's no default IPv6 pod CIDR, therefore we silently back off if CIDR s not specified.
if !vip.IsIPv4(podIP) {
return fmt.Errorf("error with the CIDR [%s]", podIP)
return nil
}
podCidr = defaultPodCIDR
}
if p.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
if sm.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
} else {
if discoverErr == nil {
serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
@@ -185,21 +151,21 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
serviceCidr = defaultServiceCIDR
}
log.Info("[Egress]", "podCIDR", podCidr, "serviceCIDR", serviceCidr, "vip", serviceCidr, "pod", podIP)
log.Infof("[Egress] pod CIDR [%s], service CIDR [%s] for vip [%s] / pod [%s]", podCidr, serviceCidr, vipIP, podIP)
// checking if all addresses are of the same IP family
if vip.IsIPv4(podIP) != vip.IsIPv4CIDR(podCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "podCIDR", podCidr)
log.Errorf("[Egress] pod's IP [%s] and Pod CIDR [%s] family is not matching. Backing off...", podIP, podCidr)
return nil
}
if vip.IsIPv4(vipIP) != vip.IsIPv4CIDR(serviceCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "serviceCIDR", serviceCidr)
log.Errorf("[Egress] VIP's IP [%s] and Service CIDR [%s] family is not matching. Backing off...", podIP, podCidr)
return nil
}
if vip.IsIPv4(vipIP) != vip.IsIPv4(podIP) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "vipIP", vipIP)
log.Errorf("[Egress] VIP's IP [%s] and Pod's IP [%s] family is not matching. Backing off...", podIP, podCidr)
return nil
}
@@ -208,30 +174,7 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
protocol = iptables.ProtocolIPv6
}
// Use the internal egress implementation
if internalEgress != "" {
// Create an array of CIDRs that we wont SNAT to.
ignoreCIDRs := []string{
podCidr,
serviceCidr,
}
// Add any specifically denied networks
if deniedNetworks != "" {
networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
ignoreCIDRs = append(ignoreCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
return nil
}
i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
@@ -258,35 +201,16 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.AppendReturnRulesForDestinationSubnet(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error adding rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
mask := "/32"
if !vip.IsIPv4(podIP) {
mask = "/128"
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.AppendReturnRulesForMarkingForNetwork(vip.MangleChainName, podIP+mask, networks[x])
if err != nil {
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
err = i.AppendReturnRulesForMarking(vip.MangleChainName, podIP+mask)
if err != nil {
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
err = i.AppendReturnRulesForMarking(vip.MangleChainName, podIP+mask)
if err != nil {
return fmt.Errorf("error adding marking rules to mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
err = i.InsertMangeTableIntoPrerouting(vip.MangleChainName)
if err != nil {
return fmt.Errorf("error adding prerouting mangle chain [%s], error [%s]", vip.MangleChainName, err)
@@ -330,12 +254,12 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
return nil
}
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debugf("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}
@@ -358,3 +282,57 @@ func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
return
}
func (sm *Manager) TeardownEgress(podIP, vipIP, destinationPorts, namespace string) error {
protocol := iptables.ProtocolIPv4
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,4 +1,4 @@
package services
package manager
import "testing"

495
pkg/manager/services.go Normal file
View File

@@ -0,0 +1,495 @@
package manager
import (
"context"
"fmt"
"slices"
"strings"
"sync"
"time"
"github.com/google/go-cmp/cmp"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/vip"
)
const (
hwAddrKey = "kube-vip.io/hwaddr"
requestedIP = "kube-vip.io/requestedIP"
vipHost = "kube-vip.io/vipHost"
egress = "kube-vip.io/egress"
egressIPv6 = "kube-vip.io/egress-ipv6"
egressDestinationPorts = "kube-vip.io/egress-destination-ports"
egressSourcePorts = "kube-vip.io/egress-source-ports"
activeEndpoint = "kube-vip.io/active-endpoint"
activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
flushContrack = "kube-vip.io/flush-conntrack"
loadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
loadbalancerHostname = "kube-vip.io/loadbalancerHostname"
serviceInterface = "kube-vip.io/serviceInterface"
upnpEnabled = "kube-vip.io/forwardUPNP"
)
func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service, wg *sync.WaitGroup) error {
defer wg.Done()
log.Debugf("[STARTING] Service Sync")
// Iterate through the synchronising services
foundInstance := false
newServiceAddresses := fetchServiceAddresses(svc)
newServiceUID := string(svc.UID)
ingressIPs := []string{}
for _, ingress := range svc.Status.LoadBalancer.Ingress {
ingressIPs = append(ingressIPs, ingress.IP)
}
shouldBreake := false
for x := range sm.serviceInstances {
if shouldBreake {
break
}
for _, newServiceAddress := range newServiceAddresses {
log.Debugf("isDHCP: %t, newServiceAddress: %s", sm.serviceInstances[x].isDHCP, newServiceAddress)
if sm.serviceInstances[x].UID == newServiceUID {
// If the found instance's DHCP configuration doesn't match the new service, delete it.
if (sm.serviceInstances[x].isDHCP && newServiceAddress != "0.0.0.0") ||
(!sm.serviceInstances[x].isDHCP && newServiceAddress == "0.0.0.0") ||
(!sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, newServiceAddress)) ||
(len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc)) ||
(sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, sm.serviceInstances[x].dhcpInterfaceIP)) {
if err := sm.deleteService(newServiceUID); err != nil {
return err
}
shouldBreake = true
break
}
foundInstance = true
}
}
}
// This instance wasn't found, we need to add it to the manager
if !foundInstance && len(newServiceAddresses) > 0 {
if err := sm.addService(ctx, svc); err != nil {
return err
}
}
return nil
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
if len(portsStatus) != len(svc.Spec.Ports) {
return false
}
for i, portSpec := range svc.Spec.Ports {
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
return false
}
}
return true
}
func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
startTime := time.Now()
newService, err := NewInstance(svc, sm.config)
if err != nil {
return err
}
for x := range newService.vipConfigs {
newService.clusters[x].StartLoadBalancerService(newService.vipConfigs[x], sm.bgpServer)
}
sm.upnpMap(ctx, newService)
if newService.isDHCP && len(newService.vipConfigs) == 1 {
go func() {
for ip := range newService.dhcpClient.IPChannel() {
log.Debugf("IP %s may have changed", ip)
newService.vipConfigs[0].VIP = ip
newService.dhcpInterfaceIP = ip
if !sm.config.DisableServiceUpdates {
if err := sm.updateStatus(newService); err != nil {
log.Warnf("error updating svc: %s", err)
}
}
}
log.Debugf("IP update channel closed, stopping")
}()
}
sm.serviceInstances = append(sm.serviceInstances, newService)
if !sm.config.DisableServiceUpdates {
log.Debugf("(svcs) will update [%s/%s]", newService.serviceSnapshot.Namespace, newService.serviceSnapshot.Name)
if err := sm.updateStatus(newService); err != nil {
// delete service to collect garbage
if deleteErr := sm.deleteService(newService.UID); deleteErr != nil {
return deleteErr
}
return err
}
}
serviceIPs := fetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[flushContrack] == "true" {
log.Debugf("Flushing conntrack rules for service [%s]", svc.Name)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
if err != nil {
log.Errorf("Error flushing any remaining egress connections [%s]", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
if err != nil {
log.Errorf("Error flushing any remaining ingress connections [%s]", err)
}
}
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
log.Debugf("Enabling egress for the service [%s]", svc.Name)
// We will need to modify the iptables rules
err = sm.iptablesCheck()
if err != nil {
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
}
var podIP string
errList := []error{}
// Should egress be IPv6
if svc.Annotations[egressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[activeEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[activeEndpointIPv6]
err = sm.configureEgress(serviceIP, podIP, svc.Annotations[egressDestinationPorts], svc.Namespace)
if err != nil {
errList = append(errList, err)
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
}
}
}
}
} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[activeEndpoint]
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[activeEndpointIPv6]
}
err = sm.configureEgress(serviceIP, podIPs, svc.Annotations[egressDestinationPorts], svc.Namespace)
if err != nil {
errList = append(errList, err)
log.Errorf("Error configuring egress for loadbalancer [%s]", err)
}
}
}
if len(errList) == 0 {
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
if err != nil {
log.Errorf("error configuring egress annotation for loadbalancer [%s]", err)
}
}
}
finishTime := time.Since(startTime)
log.Infof("[service] synchronised in %dms", finishTime.Milliseconds())
return nil
}
func (sm *Manager) deleteService(uid string) error {
// protect multiple calls
sm.mutex.Lock()
defer sm.mutex.Unlock()
var updatedInstances []*Instance
var serviceInstance *Instance
found := false
for x := range sm.serviceInstances {
log.Debugf("Looking for [%s], found [%s]", uid, sm.serviceInstances[x].UID)
// Add the running services to the new array
if sm.serviceInstances[x].UID != uid {
updatedInstances = append(updatedInstances, sm.serviceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = sm.serviceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
return nil
}
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
for _, vip := range updatedInstances[x].VIPs {
vipSet[vip] = nil
}
}
for _, vip := range serviceInstance.VIPs {
if _, found := vipSet[vip]; found {
shared = true
}
}
if !shared {
for x := range serviceInstance.clusters {
serviceInstance.clusters[x].Stop()
}
if serviceInstance.isDHCP {
serviceInstance.dhcpClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.dhcpInterface)
if err != nil {
return fmt.Errorf("error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("error deleting DHCP Link : %v", err)
}
}
// TODO: Implement dual-stack loadbalancer support if BGP is enabled
for i := range serviceInstance.vipConfigs {
if serviceInstance.vipConfigs[i].EnableBGP {
cidrVip := fmt.Sprintf("%s/%s", serviceInstance.vipConfigs[i].VIP, serviceInstance.vipConfigs[i].VIPCIDR)
err := sm.bgpServer.DelHost(cidrVip)
if err != nil {
return fmt.Errorf("[BGP] error deleting BGP host: %v", err)
}
log.Debugf("[BGP] deleted host: %s", cidrVip)
}
}
// We will need to tear down the egress
if serviceInstance.serviceSnapshot.Annotations[egress] == "true" {
if serviceInstance.serviceSnapshot.Annotations[activeEndpoint] != "" {
log.Infof("service [%s] has an egress re-write enabled", serviceInstance.serviceSnapshot.Name)
err := sm.TeardownEgress(serviceInstance.serviceSnapshot.Annotations[activeEndpoint], serviceInstance.serviceSnapshot.Spec.LoadBalancerIP, serviceInstance.serviceSnapshot.Annotations[egressDestinationPorts], serviceInstance.serviceSnapshot.Namespace)
if err != nil {
log.Errorf("%v", err)
}
}
}
}
// Update the service array
sm.serviceInstances = updatedInstances
log.Infof("Removed [%s] from manager, [%d] advertised services remain", uid, len(sm.serviceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (sm *Manager) upnpMap(ctx context.Context, s *Instance) {
if !isUPNPEnabled(s.serviceSnapshot) {
// Skip services missing the annotation
return
}
if !sm.upnp {
log.Warnf("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding service %s", s.serviceSnapshot.Name)
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
gateways := upnp.GetGatewayClients(ctx)
// Reset Gateway IPs to remove stale addresses
s.upnpGatewayIPs = make([]string, 0)
for _, vip := range s.VIPs {
for _, port := range s.ExternalPorts {
for _, gw := range gateways {
log.Infof("[UPNP] Adding map to [%s:%d - %s] on gateway %s", vip, port.Port, s.serviceSnapshot.Name, gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(port.Type), 3600)
if pinholeErr == nil {
forwardSucessful = true
log.Infof("[UPNP] Service should be accessible externally on port [%d]; PinholeID is [%d]", port.Port, pinholeID)
} else {
//TODO: Cleanup
log.Errorf("[UPNP] Unable to map port to gateway using Pinhole API[%s]", pinholeErr.Error())
}
}
// Fallback to PortForward
if !forwardSucessful {
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(port.Type), uint16(port.Port), vip, true, s.serviceSnapshot.Name, 3600)
if portMappingErr == nil {
log.Infof("[UPNP] Service should be accessible externally on port [%d]", port.Port)
forwardSucessful = true
} else {
//TODO: Cleanup
log.Errorf("[UPNP] Unable to map port to gateway using PortForward API[%s]", portMappingErr.Error())
}
}
if forwardSucessful {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err == nil {
s.upnpGatewayIPs = append(s.upnpGatewayIPs, ip)
}
}
}
}
}
// Remove duplicate IPs
slices.Sort(s.upnpGatewayIPs)
s.upnpGatewayIPs = slices.Compact(s.upnpGatewayIPs)
}
func (sm *Manager) updateStatus(i *Instance) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(i.serviceSnapshot.Namespace).Get(context.TODO(), i.serviceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if sm.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
}
if i.dhcpInterfaceHwaddr != "" || i.dhcpInterfaceIP != "" {
currentServiceCopy.Annotations[hwAddrKey] = i.dhcpInterfaceHwaddr
currentServiceCopy.Annotations[requestedIP] = i.dhcpInterfaceIP
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Errorf("Error updating Service Spec [%s] : %v", i.serviceSnapshot.Name, err)
return err
}
}
ports := make([]v1.PortStatus, 0, len(i.serviceSnapshot.Spec.Ports))
for _, port := range i.serviceSnapshot.Spec.Ports {
ports = append(ports, v1.PortStatus{
Port: port.Port,
Protocol: port.Protocol,
})
}
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.vipConfigs {
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
if err != nil {
return err
}
for _, ip := range ips {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
} else {
i := v1.LoadBalancerIngress{
IP: c.VIP,
Ports: ports,
}
ingresses = append(ingresses, i)
}
if isUPNPEnabled(currentService) {
for _, ip := range i.upnpGatewayIPs {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
}
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Errorf("Error updating Service %s/%s Status: %v", i.serviceSnapshot.Namespace, i.serviceSnapshot.Name, err)
return err
}
}
return nil
})
if retryErr != nil {
log.Errorf("Failed to set Services: %v", retryErr)
return retryErr
}
return nil
}
// fetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func fetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[loadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs
}
}
if !annotationAvailable {
if len(s.Status.LoadBalancer.Ingress) > 0 {
addresses := []string{}
for _, ingress := range s.Status.LoadBalancer.Ingress {
addresses = append(addresses, ingress.IP)
}
return addresses
}
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}
}
return []string{}
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
}

View File

@@ -0,0 +1,102 @@
package manager
import (
"context"
"fmt"
"sync"
"time"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
if err != nil {
return err
}
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
log.Infof("Shutting down kube-Vip")
return nil
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service, wg *sync.WaitGroup) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Infof("(svc election) service [%s], namespace [%s], lock name [%s], host id [%s]", service.Name, service.Namespace, serviceLease, sm.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: sm.config.NodeName,
},
}
activeService[string(service.UID)] = true
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
wg.Add(1)
go func() {
if err := sm.syncServices(ctx, service, wg); err != nil {
log.Errorln(err)
}
}()
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Infof("(svc election) service [%s] leader lost: [%s]", service.Name, sm.config.NodeName)
if activeService[string(service.UID)] {
if err := sm.deleteService(string(service.UID)); err != nil {
log.Errorln(err)
}
}
// Mark this service is inactive
activeService[string(service.UID)] = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == sm.config.NodeName {
// I just got the lock
return
}
log.Infof("(svc election) new leader elected: %s", identity)
},
},
})
log.Infof("(svc election) for service [%s] stopping", service.Name)
return nil
}

View File

@@ -8,9 +8,8 @@ import (
"strconv"
"strings"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
log "github.com/sirupsen/logrus"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
@@ -27,7 +26,7 @@ import (
// present
func (sm *Manager) annotationsWatcher() error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", sm.config.Annotations)
log.Infof("Kube-Vip is waiting for annotation prefix [%s] to be present on this node", sm.config.Annotations)
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": sm.config.NodeName}}
listOptions := metav1.ListOptions{
@@ -55,11 +54,10 @@ func (sm *Manager) annotationsWatcher() error {
// We got an error with the annotations, falling back to the watch until
// they're as needed
log.Warn(err.Error())
log.Warn(err)
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
@@ -96,7 +94,7 @@ func (sm *Manager) annotationsWatcher() error {
bgpConfig, bgpPeer, err := parseBgpAnnotations(sm.config.BGPConfig, node, sm.config.Annotations)
if err != nil {
log.Error(err.Error())
log.Error(err)
continue
}
@@ -110,7 +108,7 @@ func (sm *Manager) annotationsWatcher() error {
return fmt.Errorf("unable to parse Kubernetes Node from Kubernetes watcher")
}
log.Info("Node has been deleted", "name", node.Name)
log.Infof("Node [%s] has been deleted", node.Name)
case watch.Bookmark:
// Un-used
@@ -121,17 +119,17 @@ func (sm *Manager) annotationsWatcher() error {
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error(status.String())
log.Errorf("%v", status)
default:
}
}
close(exitFunction)
log.Info("Exiting Annotations watcher")
log.Infoln("Exiting Annotations watcher")
return nil
}

View File

@@ -0,0 +1,481 @@
package manager
import (
"context"
"errors"
"fmt"
"net"
"strings"
"sync"
"syscall"
"github.com/kube-vip/kube-vip/pkg/kubevip"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type epProvider interface {
createRetryWatcher(context.Context, *Manager,
*v1.Service) (*watchtools.RetryWatcher, error)
getAllEndpoints() ([]string, error)
getLocalEndpoints(string, *kubevip.Config) ([]string, error)
getLabel() string
updateServiceAnnotation(string, string, *v1.Service, *Manager) error
loadObject(runtime.Object, context.CancelFunc) error
getProtocol() string
}
type endpointsProvider struct {
label string
endpoints *v1.Endpoints
}
func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
}
ep.endpoints = eps
return nil
}
func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debugf("[%s] processing endpoint [%s]", ep.label, address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debugf("[%s] found local endpoint - address: %s, hostname: %s, node: %s", ep.label, address.IP, address.Hostname, *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debugf("[%s] found local endpoint - address: %s, hostname: %s", ep.label, address.IP, address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Errorf("[%s] error updating Service Spec [%s] : %v", ep.getLabel(), currentServiceCopy.Name, err)
return err
}
return nil
})
if retryErr != nil {
log.Errorf("[%s] failed to set Services: %v", ep.getLabel(), retryErr)
return retryErr
}
return nil
}
func (ep *endpointsProvider) getLabel() string {
return ep.label
}
func (ep *endpointsProvider) getProtocol() string {
return ""
}
func (sm *Manager) watchEndpoint(ctx context.Context, id string, service *v1.Service, wg *sync.WaitGroup, provider epProvider) error {
log.Infof("[%s] watching for service [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderContext, cancel := context.WithCancel(ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.createRetryWatcher(leaderContext, sm, service)
if err != nil {
cancel()
return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-ctx.Done():
log.Debugf("[%s] context cancelled", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-sm.shutdownChan:
log.Debugf("[%s] shutdown called", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debugf("[%s] function ending", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
var lastKnownGoodEndpoint string
for event := range ch {
activeEndpointAnnotation := activeEndpoint
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err = provider.loadObject(event.Object, cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
}
if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = activeEndpointIPv6
}
// Build endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
}
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
continue
}
// if we haven't populated one, then do so
if lastKnownGoodEndpoint != "" {
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
if sm.config.EnableRoutingTable {
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Annotations[egressDestinationPorts], service.Namespace); err != nil {
log.Warnf("error removing redundant egress rules: %s", err.Error())
}
}
if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
log.Warnf("[%s] existing [%s] has been removed, restarting leaderElection", provider.getLabel(), lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
leaderElectionActive = false
}
// Set our active endpoint to an existing one
lastKnownGoodEndpoint = endpoints[0]
}
} else {
lastKnownGoodEndpoint = endpoints[0]
}
if !leaderElectionActive && sm.config.EnableServicesElection {
go func() {
leaderContext, cancel = context.WithCancel(ctx)
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if leaderContext.Err() != context.Canceled {
leaderElectionActive = true
err := sm.StartServicesLeaderElection(leaderContext, service, wg)
if err != nil {
log.Error(err)
}
leaderElectionActive = false
} else {
leaderElectionActive = false
break
}
}
}()
}
isRouteConfigured, err := isRouteConfigured(service.UID)
if err != nil {
return fmt.Errorf("[%s] error while checking if route is configured: %w", provider.getLabel(), err)
}
// There are local endpoints available on the node
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection && !isRouteConfigured {
// If routing table mode is enabled - routes should be added per node
if sm.config.EnableRoutingTable {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
}
if isUpdated {
log.Debugf("[%s] updated route: %s", provider.getLabel(), cluster.Network[i].IP())
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
}
} else {
log.Infof("[%s] added route: %s, service: %s/%s, interface: %s, table: %d",
provider.getLabel(), cluster.Network[i].IP(), service.Namespace, service.Name, cluster.Network[i].Interface(), sm.config.RoutingTableID)
configuredLocalRoutes.Store(string(service.UID), true)
leaderElectionActive = true
}
}
}
}
}
// If BGP mode is enabled - hosts should be added per node
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
log.Debugf("[%s] attempting to advertise BGP service: %s", provider.getLabel(), address)
err := sm.bgpServer.AddHost(address)
if err != nil {
log.Errorf("[%s] error adding BGP host %s\n", err.Error(), provider.getLabel())
} else {
log.Infof("[%s] added BGP host: %s, service: %s/%s",
provider.getLabel(), address, service.Namespace, service.Name)
configuredLocalRoutes.Store(string(service.UID), true)
leaderElectionActive = true
}
}
}
}
}
}
} else {
// There are no local endpoints
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be deleted
if sm.config.EnableRoutingTable {
if errs := sm.clearRoutes(service); len(errs) == 0 {
configuredLocalRoutes.Store(string(service.UID), false)
}
}
// If BGP mode is enabled - routes should be deleted
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
err := sm.bgpServer.DelHost(address)
if err != nil {
log.Errorf("[%s] error deleting BGP host%s: %s\n", provider.getLabel(), address, err.Error())
} else {
log.Infof("[%s] deleted BGP host: %s, service: %s/%s",
provider.getLabel(), address, service.Namespace, service.Name)
configuredLocalRoutes.Store(string(service.UID), false)
leaderElectionActive = false
}
}
}
}
}
}
// If there are no local endpoints, and we had one then remove it and stop the leaderElection
if lastKnownGoodEndpoint != "" && sm.config.EnableRoutingTable {
log.Warnf("[%s] existing [%s] has been removed, no remaining endpoints for leaderElection", provider.getLabel(), lastKnownGoodEndpoint)
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Annotations[egressDestinationPorts], service.Namespace); err != nil {
log.Errorf("error removing redundant egress rules: %s", err.Error())
}
lastKnownGoodEndpoint = "" // reset endpoint
if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
cancel() // stop services watcher
}
leaderElectionActive = false
}
}
// Set the service accordingly
if service.Annotations[egress] == "true" {
service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
}
log.Debugf("[%s watcher] service %s/%s: local endpoint(s) [%d], known good [%s], active election [%t]",
provider.getLabel(), service.Namespace, service.Name, len(endpoints), lastKnownGoodEndpoint, leaderElectionActive)
case watch.Deleted:
// When no-leader-elecition mode
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// find all existing local endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
// Delete all routes in routing table mode
if sm.config.EnableRoutingTable {
sm.clearRoutes(service)
}
// Delete all hosts in BGP mode
if sm.config.EnableBGP {
sm.clearBGPHosts(service)
}
}
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Infof("[%s] deleted stopping watching for [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Errorf("[%s] -> %v", provider.getLabel(), statusErr)
}
}
close(exitFunction)
log.Infof("[%s] stopping watching for [%s] in namespace [%s]", provider.getLabel(), service.Name, service.Namespace)
return nil //nolint:govet
}
func (sm *Manager) clearRoutes(service *v1.Service) []error {
errs := []error{}
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if sm.countRouteReferences(route) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Errorf("failed to delete route for %s: %s", cluster.Network[i].IP(), err.Error())
errs = append(errs, err)
}
log.Debugf("deleted route: %s, service: %s/%s, interface: %s, table: %d",
cluster.Network[i].IP(), service.Namespace, service.Name, cluster.Network[i].Interface(), sm.config.RoutingTableID)
}
}
}
}
return errs
}
func (sm *Manager) clearBGPHosts(service *v1.Service) {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
err := sm.bgpServer.DelHost(address)
if err != nil {
log.Errorf("[endpoint] error deleting BGP host %s\n", err.Error())
} else {
log.Debugf("[endpoint] deleted BGP host: %s, service: %s/%s",
address, service.Namespace, service.Name)
}
}
}
}
}

View File

@@ -1,36 +1,28 @@
package providers
package manager
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type Endpointslices struct {
type endpointslicesProvider struct {
label string
endpoints *discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
}
}
func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
@@ -38,9 +30,9 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
@@ -50,7 +42,7 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
return rw, nil
}
func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
cancel()
@@ -60,7 +52,7 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
return nil
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
@@ -68,21 +60,21 @@ func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
log.Debug("processing endpoint", "provider", ep.label, "ip", address)
log.Debugf("[%s] processing endpoint [%s]", ep.label, address)
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
log.Debugf("[%s] found endpoint - address: %s, hostname: %s, node: %s", ep.label, address, *endpoint.Hostname, *endpoint.NodeName)
} else {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
log.Debugf("[%s] found endpoint - address: %s, node: %s", ep.label, address, *endpoint.NodeName)
}
localEndpoints = append(localEndpoints, address)
continue
@@ -90,7 +82,7 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
if endpoint.Hostname != nil && id == *endpoint.Hostname {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
log.Debugf("[%s] found endpoint - address: %s, hostname: %s", ep.label, address, *endpoint.Hostname)
localEndpoints = append(localEndpoints, address)
}
}
@@ -98,11 +90,11 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
return localEndpoints, nil
}
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -112,28 +104,28 @@ func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string,
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
currentServiceCopy.Annotations[activeEndpoint] = endpoint
currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
log.Errorf("[%s] error updating Service Spec [%s] : %v", ep.label, currentServiceCopy.Name, err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
log.Errorf("[%s] failed to set Services: %v", ep.label, retryErr)
return retryErr
}
return nil
}
func (ep *Endpointslices) GetLabel() string {
func (ep *endpointslicesProvider) getLabel() string {
return ep.label
}
func (ep *Endpointslices) GetProtocol() string {
func (ep *endpointslicesProvider) getProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -0,0 +1,380 @@
package manager
import (
"context"
"fmt"
"sync"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// TODO: Fix the naming of these contexts
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancer map[string]context.Context
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancerCancel map[string]func()
// activeService keeps track of services that already have a leaderElection in place
var activeService map[string]bool
// watchedService keeps track of services that are already being watched
var watchedService map[string]bool
// watchedService keeps track of routes that has been configured on the node
var configuredLocalRoutes sync.Map
func init() {
// Set up the caches for monitoring existing active or watched services
activeServiceLoadBalancerCancel = make(map[string]func())
activeServiceLoadBalancer = make(map[string]context.Context)
activeService = make(map[string]bool)
watchedService = make(map[string]bool)
}
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service, *sync.WaitGroup) error) error {
// Watch function
var wg sync.WaitGroup
// first start port mirroring if enabled
if err := sm.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := sm.stopTrafficMirroringIfEnabled()
if err != nil {
log.Fatal(err)
}
}()
if sm.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
sm.config.ServiceNamespace = v1.NamespaceAll
log.Infof("(svcs) starting services watcher for all namespaces")
} else {
log.Infof("(svcs) starting services watcher for services in namespace [%s]", sm.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Infof("(svcs) [%s] has an ignore annotation for kube-vip", svc.Name)
break
}
// Select loadbalancer class filtering function
lbClassFilterFunc := sm.lbClassFilter
if sm.config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = sm.lbClassFilterLegacy
}
// Check the loadBalancer class
if lbClassFilterFunc(svc) {
break
}
svcAddresses := fetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
break
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(sm.config.Interface, addr)
if err != nil {
log.Errorf("(svcs) cleaning existing address error: [%s]", err.Error())
}
if f {
log.Warnf("(svcs) already found existing address [%s] on adapter [%s]", addr, sm.config.Interface)
}
}
}
// Scenarios:
// 1.
if !activeService[string(svc.UID)] {
log.Debugf("(svcs) [%s] has been added/modified with addresses [%s]", svc.Name, fetchServiceAddresses(svc))
wg.Add(1)
activeServiceLoadBalancer[string(svc.UID)], activeServiceLoadBalancerCancel[string(svc.UID)] = context.WithCancel(ctx)
// Background the services election
// EnableServicesElection enabled
// watchEndpoint will do a ServicesElection by Service and understands local endpoints
//
// EnableRoutingTable enabled and EnableLeaderElection disabled
// watchEndpoint will also not do a leaderElection by service.
if sm.config.EnableServicesElection ||
((sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !watchedService[string(svc.UID)] {
// background the endpoint watcher
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
wg.Add(1)
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, &wg, provider); err != nil {
log.Error(err)
}
wg.Done()
}
}()
if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
wg.Add(1)
go func() {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
if err != nil {
log.Error(err)
}
wg.Done()
}()
}
// We're now watching this service
watchedService[string(svc.UID)] = true
}
} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
wg.Add(1)
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, &wg, provider); err != nil {
log.Error(err)
}
wg.Done()
}
}()
wg.Add(1)
go func() {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
if err != nil {
log.Error(err)
}
wg.Done()
}()
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
wg.Add(1)
go func() {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
if err != nil {
log.Error(err)
}
wg.Done()
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
wg.Add(1)
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc, &wg)
if err != nil {
log.Error(err)
}
wg.Done()
}
activeService[string(svc.UID)] = true
}
case watch.Deleted:
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
if activeService[string(svc.UID)] {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Infof("(svcs) [%s] has an ignore annotation for kube-vip", svc.Name)
break
}
isRouteConfigured, err := isRouteConfigured(svc.UID)
if err != nil {
return fmt.Errorf("error while checkig if route is configured: %w", err)
}
// If no leader election is enabled, delete routes here
if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
sm.config.EnableRoutingTable && isRouteConfigured {
if errs := sm.clearRoutes(svc); len(errs) == 0 {
configuredLocalRoutes.Store(string(svc.UID), false)
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = sm.deleteService(string(svc.UID))
if err != nil {
log.Error(err)
}
// Calls the cancel function of the context
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
activeServiceLoadBalancerCancel[string(svc.UID)]()
}
activeService[string(svc.UID)] = false
watchedService[string(svc.UID)] = false
}
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
if sm.config.EnableBGP {
instance := sm.findServiceInstance(svc)
for _, vip := range instance.vipConfigs {
vipCidr := fmt.Sprintf("%s/%s", vip.VIP, vip.VIPCIDR)
err = sm.bgpServer.DelHost(vipCidr)
if err != nil {
log.Errorf("error deleting host %s: %s", vipCidr, err.Error())
}
}
} else {
sm.clearRoutes(svc)
}
}
log.Infof("(svcs) [%s/%s] has been deleted", svc.Namespace, svc.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Errorf("services -> %v", status)
default:
}
}
close(exitFunction)
log.Warnln("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
if svc == nil {
log.Infof("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Infof("(svcs) [%s] specified the loadBalancer class [%s], ignoring", svc.Name, *svc.Spec.LoadBalancerClass)
return true
}
} else if sm.config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Infof("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service [%s] didn't specify any loadBalancer class, ignoring", svc.Name)
return true
}
return false
}
func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
if svc == nil {
log.Infof("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
log.Infof("(svcs) [%s] specified no loadBalancer class, expected [%s], ignoring", svc.Name, sm.config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Infof("(svcs) [%s] specified loadBalancer class [%s], expected [%s], ignoring", svc.Name, *svc.Spec.LoadBalancerClass, sm.config.LoadBalancerClassName)
return true
}
return false
}
func isRouteConfigured(serviceUID types.UID) (bool, error) {
isConfigured := false
value, ok := configuredLocalRoutes.Load(string(serviceUID))
if ok {
isConfigured, ok = value.(bool)
if !ok {
return false, fmt.Errorf("error converting configuredLocalRoute item to boolean value")
}
}
return isConfigured, nil
}

View File

@@ -1,34 +0,0 @@
package networkinterface
import (
"sync"
"github.com/vishvananda/netlink"
)
type Manager struct {
interfaces map[string]*Link
}
type Link struct {
Lock sync.Mutex
Intf netlink.Link
}
func NewManager() *Manager {
return &Manager{
interfaces: make(map[string]*Link),
}
}
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
return l
}
result := &Link{
Intf: intf,
}
m.interfaces[intf.Attrs().Name] = result
return result
}

View File

@@ -1,458 +0,0 @@
package nftables
import (
"errors"
"fmt"
"log/slog"
"net"
"strconv"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"golang.org/x/sys/unix"
)
const (
NatTable = "kube_vip_%s"
SNatChain = "kube_vip_snat_%s"
)
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
conn, err := nftables.New()
if err != nil {
return err
}
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Look up the table
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
if t == nil {
// If it doesn't exist then create it
slog.Debug("[egress]", "Creating Table", tableName)
conn.AddTable(GetTable(IPv6))
}
}
slog.Debug("[egress]", "Creating Chain for service", service, "IPv6", IPv6)
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddChain(GetSNatChain(IPv6, service))
conn.Flush()
// Create our nftables rule
rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
if err != nil {
return err
}
slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
conn.AddRule(rule) // Add the rule
err = conn.Flush() // Commit the rule to nftables
if err != nil {
return err
}
return conn.CloseLasting() // Close out any remaining netlink communication
}
func DeleteSNAT(IPv6 bool, service string) error {
conn, err := nftables.New()
if err != nil {
return err
}
var chainName = fmt.Sprintf(SNatChain, service)
slog.Info("[egress]", "Looking for", chainName)
chain, err := conn.ListChain(GetTable(IPv6), chainName)
if err != nil {
return err
}
if chain != nil {
slog.Info("[egress]", "Deleting chain", chainName)
conn.DelChain(chain)
return conn.Flush()
}
return fmt.Errorf("unable to find chain [%s]", chainName)
}
func GetTable(IPv6 bool) *nftables.Table {
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Default to IPv4
table := &nftables.Table{
Family: nftables.TableFamilyIPv4,
Name: tableName,
}
// Move to IPv6 if needed
if IPv6 {
table.Family = nftables.TableFamilyIPv6
}
return table
}
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
var chainName = fmt.Sprintf(SNatChain, service)
policy := nftables.ChainPolicyAccept
return &nftables.Chain{
Name: chainName,
Table: GetTable(IPv6),
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityNATSource,
Policy: &policy,
}
}
func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
if IPv6 {
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
}
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
}
// ClearTable will remove the original tables and create new empty ones
func ClearTable(conn *nftables.Conn) error {
tableName := fmt.Sprintf(NatTable, "v6")
if t, err := FilterTable(conn, tableName, false); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(true))
tableName = fmt.Sprintf(NatTable, "v4")
if t, err := FilterTable(conn, tableName, true); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(false))
return nil
}
// Create our nftables rule
func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
// Validate pod IP
if net.ParseIP(podIP) == nil {
return nil, errors.New("ip is invalid")
}
// Validate vip IP
if net.ParseIP(vipIP) == nil {
return nil, errors.New("output_ip is not a valid ip")
}
// Get the kube-vip table
table := GetTable(IPv6)
// Create our rule
rule := &nftables.Rule{
Table: table,
Exprs: []expr.Any{},
}
// Set the correct chain
rule.Chain = GetSNatChain(IPv6, service)
// Create a set for our original/source address
set := &nftables.Set{
Table: table,
Anonymous: true,
Constant: true,
KeyType: nftables.TypeIPAddr,
Interval: false,
}
if IPv6 {
set.KeyType = nftables.TypeIP6Addr
} else {
set.KeyType = nftables.TypeIPAddr
}
// Create an element using our pod IP
elements := []nftables.SetElement{}
if IPv6 {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
} else {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
}
// Add the elements to the set
err := conn.AddSet(set, elements)
if err != nil {
return nil, err
}
// Create the expression using the set
expression := []expr.Any{}
payload := &expr.Payload{
OperationType: expr.PayloadLoad,
Base: expr.PayloadBaseNetworkHeader,
DestRegister: 1,
SourceRegister: 0,
}
// Set the length of the data based upon the type of IP version being used
if IPv6 {
payload.Offset = 8
payload.Len = 16
} else {
payload.Offset = 12
payload.Len = 4
}
lookup := &expr.Lookup{
SourceRegister: 1,
DestRegister: 0,
SetID: set.ID,
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, lookup)
// Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
// If we filter on ports protocols then parse them
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
// Create an element using our pod IP
tcpElements := []nftables.SetElement{}
udpElements := []nftables.SetElement{}
sctpElements := []nftables.SetElement{}
tcpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
udpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
sctpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
for _, fixedPort := range fixedPorts {
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 2 { // Ensure we have two elements { proto:port }
// parse the port to a number
port, err := strconv.Atoi(data[1])
if err != nil {
slog.Error("[egress]", "unable to process port", data[1])
continue
}
// Ensure the port is within the valid range for uint16
if port < 0 || port > 65535 {
slog.Error("[egress]", "port out of range for uint16", data[1])
continue
}
switch data[0] {
case "tcp":
//nolint:gosec
tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "udp":
//nolint:gosec
udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "sctp":
//nolint:gosec
sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
default:
slog.Error("[egress]", "unknown protocol", data[0])
}
}
}
// Add TCP Ports
if len(tcpElements) != 0 {
err = conn.AddSet(tcpSet, tcpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_TCP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: tcpSet.Name,
SetID: tcpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add UDP ports
if len(udpElements) != 0 {
err = conn.AddSet(udpSet, udpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_UDP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: udpSet.Name,
SetID: udpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add SCTP Ports
if len(sctpElements) != 0 {
err = conn.AddSet(sctpSet, sctpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_SCTP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: sctpSet.Name,
SetID: sctpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
}
// Parse which CIDRs we will not SNAT for
for _, cidr := range ignoreCIDR {
start, end, err := nftables.NetFirstAndLastIP(cidr)
if err != nil {
return nil, err
}
expression = []expr.Any{}
payload := &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
}
notEqualRange := &expr.Range{
Op: expr.CmpOpNeq,
Register: 1,
}
if IPv6 {
payload.Len = 16
payload.Offset = 24
notEqualRange.FromData = start.To16()
notEqualRange.ToData = end.To16()
} else {
payload.Offset = 16
payload.Len = 4
notEqualRange.FromData = start.To4()
notEqualRange.ToData = end.To4()
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, notEqualRange)
// // Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
}
// Final expression to the rule is the SNAT to the VIP address
expression = []expr.Any{}
immediate := &expr.Immediate{
Register: 1,
}
nat := &expr.NAT{
Type: expr.NATTypeSourceNAT,
RegAddrMin: 1,
RegAddrMax: 1,
RegProtoMin: 0,
RegProtoMax: 0,
Random: false,
FullyRandom: false,
Persistent: false,
Prefix: false,
}
if IPv6 {
immediate.Data = net.ParseIP(vipIP).To16()
nat.Family = unix.NFPROTO_IPV6
} else {
immediate.Data = net.ParseIP(vipIP).To4()
nat.Family = unix.NFPROTO_IPV4
}
// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
// Add expressions
expression = append(expression, immediate)
expression = append(expression, nat)
rule.Exprs = append(rule.Exprs, expression...)
return rule, nil
}

View File

@@ -1,36 +0,0 @@
package servicecontext
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func New(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -1,36 +0,0 @@
package services
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func NewContext(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -1,127 +0,0 @@
package services
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
var (
svcLocks map[string]*sync.Mutex
)
func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
if err != nil {
return err
}
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
log.Info("Shutting down kube-Vip")
return nil
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: p.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
defer childCancel()
if _, ok := svcLocks[serviceLease]; !ok {
svcLocks[serviceLease] = new(sync.Mutex)
}
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
if svcCtx == nil {
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
svcCtx.IsActive = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity)
},
},
})
log.Info("stopping leader election", "service", service.Name)
return nil
}

View File

@@ -1,344 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"reflect"
"sync"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
config *kubevip.Config
lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
svcMap sync.Map
// Keeps track of all running instances
ServiceInstances []*instance.Instance
mutex sync.Mutex
bgpServer *bgp.Server
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
}
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
}
return &Processor{
config: config,
lbClassFilter: lbClassFilterFunc,
ServiceInstances: []*instance.Instance{},
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return true, nil
}
svcAddresses := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
return true, nil
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := instance.FindServiceInstance(svc, p.ServiceInstances)
var originalService []string
shouldGarbageCollect := true
if i != nil {
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
if i != nil {
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
newService := instance.FetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
err = p.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
p.svcMap.Delete(svc.UID)
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
if svcCtx == nil {
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.IsActive = true
}
return false, nil
}
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil && svcCtx.IsActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
}
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
if p.config.EnableBGP {
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
} else if p.config.EnableRoutingTable {
endpoints.ClearRoutes(svc, &p.ServiceInstances)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return true, nil
}
func (p *Processor) Stop() {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
}
func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
svcCtx, ok := p.svcMap.Load(uid)
if !ok {
return nil, nil
}
ctx, ok := svcCtx.(*servicecontext.Context)
if !ok {
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
}
return ctx, nil
}
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
}

View File

@@ -1,516 +0,0 @@
package services
import (
"context"
"fmt"
"slices"
"strings"
"time"
log "log/slog"
"github.com/google/go-cmp/cmp"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type ServiceInstanceAction string
const (
ActionDelete ServiceInstanceAction = "delete"
ActionAdd ServiceInstanceAction = "add"
ActionNone ServiceInstanceAction = "none"
)
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
return nil
}
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
addresses := instance.FetchServiceAddresses(svc)
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCP {
if address != "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
return ActionDelete
}
}
if !instance.IsDHCP {
if address == "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
return ActionDelete
}
}
if !comparePortsAndPortStatuses(svc) {
return ActionDelete
}
}
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
return ActionNone
}
}
if len(addresses) > 0 {
log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
return ActionAdd // If no matching instance is found, we need to add a new service instance
}
return ActionNone
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
if len(svc.Status.LoadBalancer.Ingress) == 0 {
return false
}
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
if len(portsStatus) != len(svc.Spec.Ports) {
return false
}
for i, portSpec := range svc.Spec.Ports {
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
return false
}
}
return true
}
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
p.upnpMap(ctx, newService)
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
for ip := range newService.DHCPClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IP update channel closed, stopping")
}()
}
p.ServiceInstances = append(p.ServiceInstances, newService)
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[kubevip.FlushContrack] == "true" {
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
if p.config.EgressWithNftables {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.nftablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.iptablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
var podIP string
errList := []error{}
// Should egress be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
}
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
if len(errList) == 0 {
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
finishTime := time.Since(startTime)
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
p.mutex.Lock()
defer p.mutex.Unlock()
var updatedInstances []*instance.Instance
var serviceInstance *instance.Instance
found := false
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = p.ServiceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
return nil
}
for _, c := range serviceInstance.Clusters {
for n := range c.Network {
c.Network[n].SetHasEndpoints(false)
}
}
// Determine if this this VIP is shared with other loadbalancers
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
for i := range serviceInstance.VIPConfigs {
if serviceInstance.VIPConfigs[i].EnableBGP {
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
if err != nil {
log.Error("[service] egress teardown", "err", err)
}
}
}
}
// Update the service array
p.ServiceInstances = updatedInstances
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
if !isUPNPEnabled(s.ServiceSnapshot) {
// Skip services missing the annotation
return
}
if !p.config.EnableUPNP {
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
return
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
gateways := upnp.GetGatewayClients(ctx)
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
if pinholeErr == nil {
forwardSucessful = true
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
}
}
// Fallback to PortForward
if !forwardSucessful {
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint TODO
if portMappingErr == nil {
log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
forwardSucessful = true
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
}
}
if forwardSucessful {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err == nil {
s.UPNPGatewayIPs = append(s.UPNPGatewayIPs, ip)
}
}
}
}
}
// Remove duplicate IPs
slices.Sort(s.UPNPGatewayIPs)
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
Duration: 10 * time.Millisecond,
Factor: 0,
Jitter: 0.1,
}
// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if p.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
log.Error("(Synthetic error ) updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return fmt.Errorf("(Synthetic) simulating api server errors")
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
}
}
ports := make([]v1.PortStatus, 0, len(i.ServiceSnapshot.Spec.Ports))
for _, port := range i.ServiceSnapshot.Spec.Ports {
ports = append(ports, v1.PortStatus{
Port: port.Port,
Protocol: port.Protocol,
})
}
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.VIPConfigs {
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
for _, ip := range ips {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
} else {
i := v1.LoadBalancerIngress{
IP: c.VIP,
Ports: ports,
}
ingresses = append(ingresses, i)
}
if isUPNPEnabled(currentService) {
for _, ip := range i.UPNPGatewayIPs {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
}
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
}
}
return nil
})
return err
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -1,94 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.Ctx.Done():
log.Debug("context cancelled", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-p.shutdownChan:
log.Debug("shutdown called", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
var lastKnownGoodEndpoint string
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
if restart {
continue
} else if err != nil {
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
}
case watch.Deleted:
if err := epProcessor.Delete(service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}

View File

@@ -1,183 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := p.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := p.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if p.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
p.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-p.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := p.AddOrModify(ctx, event, serviceFunc)
if restart {
break
}
if err != nil {
return fmt.Errorf("add/modify service error: %w", err)
}
case watch.Deleted:
restart, err := p.Delete(event)
if restart {
break
}
if err != nil {
return fmt.Errorf("delete service error: %w", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
return true
}
return false
}
func (p *Processor) serviceInterface() string {
svcIf := p.config.Interface
if p.config.ServicesInterface != "" {
svcIf = p.config.ServicesInterface
}
return svcIf
}
func (p *Processor) startTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (p *Processor) stopTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}

View File

@@ -4,7 +4,6 @@ import (
"fmt"
"io"
"os"
"strconv"
)
func WriteProcSys(path, value string) error {
@@ -28,47 +27,3 @@ func WriteProcSys(path, value string) error {
return nil
}
func CheckProcSys(path string) (bool, error) {
f, err := os.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
return false, fmt.Errorf("failed to open file: %w", err)
}
defer func() {
if cErr := f.Close(); cErr != nil && err == nil {
err = fmt.Errorf("failed to close file: %w", cErr)
}
}()
buffer := make([]byte, 1)
if n, err := f.Read(buffer); err != nil || n < len(buffer) {
return false, fmt.Errorf("failed to read file: %w", err)
}
var isEnabled bool
isEnabled, err = strconv.ParseBool(string(buffer))
if err != nil {
return false, fmt.Errorf("failed to parse value: %w", err)
}
return isEnabled, nil
}
func EnableProcSys(path string) (bool, error) {
isEnabled, err := CheckProcSys(path)
if err != nil {
return false, fmt.Errorf("failed to check '%s' status: %w", path, err)
}
if !isEnabled {
if err := WriteProcSys(path, "1"); err != nil {
if os.IsPermission(err) {
return false, fmt.Errorf("no permission to write to the file '%s' - please ensure that kube-vip is running with proper capabilities/privileged mode to write to sysfs: %w", path, err)
}
return false, fmt.Errorf("failed to enable '%s': %w", path, err)
}
return true, nil
}
return false, nil
}

View File

@@ -4,8 +4,7 @@ import (
"errors"
"fmt"
log "log/slog"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)
@@ -28,15 +27,16 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
return fmt.Errorf("failed to find nic %s: %v", toNICName, err)
}
toNICID := toNIC.Attrs().Index
log.Debug(fmt.Sprintf("interface %s has index %d", fromNICName, fromNICID))
log.Debug(fmt.Sprintf("interface %s has index %d", toNICName, toNICID))
log.Debug(fmt.Sprintf("clean up interface %s first in case it has stale qdsic", fromNICName))
log.Debugf("interface %s has index %d", fromNICName, fromNICID)
log.Debugf("interface %s has index %d", toNICName, toNICID)
log.Debugf("clean up interface %s first in case it has stale qdsic", fromNICName)
if err := CleanupQDSICFromNIC(fromNICName); err != nil {
return err
}
log.Debug(fmt.Sprintf("step 1: tc qdisc add dev %s ingress", fromNICName))
log.Debugf("step 1: tc qdisc add dev %s ingress", fromNICName)
qdisc1 := &netlink.Ingress{
QdiscAttrs: netlink.QdiscAttrs{
LinkIndex: fromNICID,
@@ -48,7 +48,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
return fmt.Errorf("failed to add qdisc for interface %s: %v", fromNICName, err)
}
log.Debug(fmt.Sprintf("step 2: tc filter add dev %s parent ffff: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, toNICName))
log.Debugf("step 2: tc filter add dev %s parent ffff: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, toNICName)
// add a filter to mirror traffic from index1 to index2
filter1 := &netlink.U32{
FilterAttrs: netlink.FilterAttrs{
@@ -71,7 +71,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
return fmt.Errorf("failed to add filter for interface %s: %v", fromNICName, err)
}
log.Debug(fmt.Sprintf("step 3: tc qdisc add dev %s ingress", fromNICName))
log.Debugf("step 3: tc qdisc add dev %s ingress", fromNICName)
qdiscTemp := netlink.NewPrio(netlink.QdiscAttrs{
LinkIndex: fromNICID,
Parent: netlink.HANDLE_ROOT,
@@ -90,12 +90,12 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
return err
}
log.Debug(fmt.Sprintf("step 4: tc filter add dev %s parent %d: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, qdiscID, toNICName))
log.Debugf("step 4: tc filter add dev %s parent %d: protocol ip u32 match u8 0 0 action mirred egress mirror dev %s", fromNICName, qdiscID, toNICName)
filter2 := &netlink.U32{
FilterAttrs: netlink.FilterAttrs{
LinkIndex: fromNICID,
Parent: netlink.MakeHandle(uint16(qdiscID), 0), //nolint
Parent: netlink.MakeHandle(uint16(qdiscID), 0),
Protocol: unix.ETH_P_ALL,
},
Actions: []netlink.Action{
@@ -113,7 +113,7 @@ func MirrorTrafficFromNIC(fromNICName, toNICName string) error {
return fmt.Errorf("failed to add filter for interface %s: %v", fromNICName, err)
}
log.Info("traffic mirroring has been set up", "src", toNICName, "dst", fromNICName)
log.Infof("traffic mirroring has been set up from interface %s to interface %s", toNICName, fromNICName)
return nil
}
@@ -126,7 +126,7 @@ func CleanupQDSICFromNIC(nicName string) error {
}
nicID := toNIC.Attrs().Index
log.Debug(fmt.Sprintf("interface %s has index %d", nicName, nicID))
log.Debugf("interface %s has index %d", nicName, nicID)
log.Debug("step 1: delete ingress qdisc")
if err := tryCleanupQdiscByType(nicID, nicName, "ingress"); err != nil {
@@ -138,7 +138,7 @@ func CleanupQDSICFromNIC(nicName string) error {
return err
}
log.Info("finished cleaning up all qdisc config", "interface", nicName)
log.Infof("finished cleaning up all qdisc config on interface %s", nicName)
return nil
}
@@ -154,7 +154,7 @@ func getQdiscFromInterfaceByType(nicID int, nicName string, qType string) (uint3
return q.Attrs().Handle, nil
}
}
log.Error("no qdisc", "interface", nicName, "is of type", qType)
log.Errorf("no qdisc under interface %s is %s type", nicName, qType)
return 0, errQdiscNotFound
}
@@ -178,7 +178,7 @@ func tryCleanupQdiscByType(nicID int, nicName, qType string) error {
_, err := getQdiscFromInterfaceByType(nicID, nicName, qType)
if err != nil {
if err == errQdiscNotFound {
log.Debug(fmt.Sprintf("%s type qdisc doesn't exist on interface %s, skip deleting", qType, nicName))
log.Debugf("%s type qdisc doesn't exist on interface %s, skip deleting", qType, nicName)
return nil
}
return err

View File

@@ -4,10 +4,9 @@ import (
"context"
"strings"
log "log/slog"
"github.com/huin/goupnp"
"github.com/huin/goupnp/dcps/internetgateway2"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
@@ -48,7 +47,7 @@ func GetGatewayClients(ctx context.Context) []Gateway {
if wanipv6clients, err := internetgateway2.NewWANIPv6FirewallControl1ClientsByURLCtx(ctx, gatewayURL); err == nil {
gatewayClients[i].WANIPv6FirewallControlClient = wanipv6clients[0]
} else {
log.Warn("[UPNP] Unable to find WANIPv6FirewallControl1Clients", "Gateway", gatewayURL, "err", err)
log.Warnf("[UPNP] Unable to find WANIPv6FirewallControl1Clients for Gateway %s [%s]", gatewayURL, err.Error())
}
}
return gatewayClients
@@ -85,7 +84,7 @@ func GetConnectionClients(ctx context.Context) []ConnectionClient {
var routers []ConnectionClient
if err := tasks.Wait(); err != nil {
log.Error("[UPNP] Could not finish querying UPNP connection clients", "err", err.Error())
log.Errorf("[UPNP] Could not finish querying UPNP connection clients [%s]", err.Error())
return routers
}
@@ -93,7 +92,7 @@ func GetConnectionClients(ctx context.Context) []ConnectionClient {
errors = append(errors, ppp1Error...)
for _, e := range errors {
log.Warn("[UPNP] UPNP Gateway responded with an error while querying WAN Connection Client", "err", e)
log.Warnf("[UPNP] UPNP Gateway responded with an error while querying WAN Connection Client [%s]", e.Error())
}
for _, c := range ip1Clients {

View File

@@ -1,10 +1,6 @@
package utils
import (
"fmt"
"net"
"os"
)
import "os"
func FileExists(filename string) bool {
info, err := os.Stat(filename)
@@ -13,15 +9,3 @@ func FileExists(filename string) bool {
}
return !info.IsDir()
}
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
// Check if the input is valid
_, _, err := net.ParseCIDR(addr)
if err != nil {
return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
}
return addr, nil
}

View File

@@ -2,24 +2,20 @@ package vip
import (
"fmt"
"math"
"net"
"slices"
"os"
"strconv"
"strings"
"sync"
log "log/slog"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netlink/nl"
"golang.org/x/sys/unix"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/utils"
)
const (
@@ -31,15 +27,13 @@ const (
// Network is an interface that enable managing operations for a given IP
type Network interface {
AddIP(precheck bool) (bool, error)
AddIP(precheck bool) error
AddRoute(precheck bool) error
DeleteIP() (bool, error)
DeleteIP() error
DeleteRoute() error
UpdateRoutes() (bool, error)
IsSet() (bool, error)
IP() string
CIDR() string
IPisLinkLocal() bool
PrepareRoute() *netlink.Route
SetIP(ip string) error
SetServicePorts(service *v1.Service)
@@ -50,9 +44,6 @@ type Network interface {
DDNSHostName() string
DNSName() string
SetMask(mask string) error
SetHasEndpoints(value bool)
HasEndpoints() bool
ARPName() string
}
// network - This allows network configuration
@@ -60,10 +51,9 @@ type network struct {
mu sync.Mutex
address *netlink.Addr
link *networkinterface.Link
link netlink.Link
ports []v1.ServicePort
serviceName string
enableSecurity bool
ignoreSecurity bool
dnsName string
@@ -75,16 +65,18 @@ type network struct {
routeTable int
routingTableType int
routingProtocol int
}
ipvsEnabled bool
hasEndpoints bool
func netlinkParse(addr string) (*netlink.Addr, error) {
mask, err := GetFullMask(addr)
if err != nil {
return nil, err
}
return netlink.ParseAddr(addr + mask)
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int, routingProtocol int, dnsMode, forwardMethod, iptablesBackend string) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -92,39 +84,28 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
return networks, errors.Wrapf(err, "could not get link for interface '%s'", iface)
}
networkLink := intfMgr.Get(link)
if IsIP(address) {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
ipvsEnabled: ipvsEnabled,
}
subnet, err = SelectSubnet(address, subnet)
if err != nil {
return networks, fmt.Errorf("unable to select subnet for IP %q from %q: %w", address, subnet, err)
}
// Check if the subnet needs overriding
cidr, err := utils.FormatIPWithSubnetMask(address, subnet)
if err != nil {
return networks, errors.Wrapf(err, "could not format address '%s' with subnetMask '%s'", address, subnet)
if subnet != "" {
result.address, err = netlink.ParseAddr(address + subnet)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
} else {
result.address, err = netlinkParse(address)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
}
result.address, err = netlink.ParseAddr(cidr)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
if iface == "lo" && !loGlobalScope {
// set host scope on loopback, otherwise global scope will be used by default
@@ -140,7 +121,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
// when leader starts, should do get IP from DHCP for the domain
if isDDNS {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
@@ -148,8 +129,6 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
networks = append(networks, result)
@@ -160,7 +139,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
for _, ip := range ips {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
@@ -168,21 +147,15 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
// we're able to resolve store this as the initial IP
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, subnet)); err != nil {
if result.address, err = netlinkParse(ip); err != nil {
return networks, err
}
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
networks = append(networks, result)
}
@@ -225,7 +198,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
route := &netlink.Route{
Scope: routeScope,
Dst: configurator.address.IPNet,
LinkIndex: configurator.link.Intf.Attrs().Index,
LinkIndex: configurator.link.Attrs().Index,
Table: configurator.routeTable,
Type: configurator.routingTableType,
Protocol: netlink.RouteProtocol(configurator.routingProtocol),
@@ -235,8 +208,6 @@ func (configurator *network) PrepareRoute() *netlink.Route {
// AddRoute - Add an IP address to a route table
func (configurator *network) AddRoute(precheck bool) error {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
route := configurator.PrepareRoute()
exists := false
@@ -258,7 +229,7 @@ func (configurator *network) AddRoute(precheck bool) error {
}
func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
routes, err := netlink.RouteList(configurator.link, netlink.FAMILY_ALL)
if err != nil {
return false, errors.Wrap(err, "failed to list routes")
}
@@ -308,41 +279,36 @@ func (configurator *network) UpdateRoutes() (bool, error) {
}
// AddIP - Add an IP address to the interface
func (configurator *network) AddIP(precheck bool) (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
func (configurator *network) AddIP(precheck bool) error {
exists := false
var err error
if precheck {
if exists, err = configurator.IsSet(); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
if exists, err = configurator.addressExists(); err != nil {
return errors.Wrap(err, "could not check if address exists")
}
}
if exists {
return false, nil
if !exists {
if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not add ip")
}
if err := configurator.configureIPTables(); err != nil {
return errors.Wrap(err, "could not configure IPTables")
}
}
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not add ip")
}
if err := configurator.configureIPTables(); err != nil {
return true, errors.Wrap(err, "could not configure IPTables")
}
return true, nil
return nil
}
func (configurator *network) configureIPTables() error {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
}
}
// It seems that masquerading is only reuired with IPv4 for IPVS to work.
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
if configurator.forwardMethod == "masquerade" {
if err := configurator.addIptablesRulesForMasquerade(); err != nil {
return errors.Wrap(err, "could not add iptables rules for masquerade")
}
@@ -351,6 +317,21 @@ func (configurator *network) configureIPTables() error {
return nil
}
func (configurator *network) addressExists() (bool, error) {
addrs, err := netlink.AddrList(configurator.link, netlink.FAMILY_ALL)
if err != nil {
return false, errors.Wrap(err, "could not list addresses")
}
for _, addr := range addrs {
if addr.Equal(*configurator.address) {
return true, nil
}
}
return false, nil
}
func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
ipt, err := iptables.New()
if err != nil {
@@ -362,7 +343,7 @@ func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
if err := insertCommonIPTablesRules(ipt, vip, comment); err != nil {
return fmt.Errorf("could not add common iptables rules: %w", err)
}
log.Debug("add iptables rules", "vip", vip, "ports", configurator.ports)
log.Debugf("add iptables rules, vip: %s, ports: %+v", vip, configurator.ports)
if err := configurator.insertIPTablesRulesForServicePorts(ipt, vip, comment); err != nil {
return fmt.Errorf("could not add iptables rules for service ports: %v", err)
}
@@ -467,7 +448,7 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
return fmt.Errorf("could not delete common iptables rules: %w", err)
}
log.Debug("remove iptables rules", "vip", vip, "ports", configurator.ports)
log.Debugf("remove iptables rules, vip: %s, ports: %+v", vip, configurator.ports)
for _, port := range configurator.ports {
// iptables -D INPUT -d <VIP> -p <protocol> --dport <port> -j ACCEPT
if err := ipt.DeleteIfExists(iptables.TableFilter, iptables.ChainInput, "-d", vip, "-p", string(port.Protocol),
@@ -480,37 +461,34 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
}
// DeleteIP - Remove an IP address from the interface
func (configurator *network) DeleteIP() (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
func (configurator *network) DeleteIP() error {
result, err := configurator.IsSet()
if err != nil {
return false, errors.Wrap(err, "ip check in DeleteIP failed")
return errors.Wrap(err, "ip check in DeleteIP failed")
}
// Nothing to delete
if !result {
return false, nil
return nil
}
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not delete ip")
if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not delete ip")
}
if configurator.enableSecurity && !configurator.ignoreSecurity {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
return errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
}
}
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
if configurator.forwardMethod == "masquerade" {
if err := configurator.removeIptablesRulesForMasquerade(); err != nil {
return true, errors.Wrap(err, "could not remove iptables masquerade rules ")
return errors.Wrap(err, "could not remove iptables masquerade rules ")
}
}
return true, nil
return nil
}
func (configurator *network) addIptablesRulesForMasquerade() error {
@@ -577,15 +555,12 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
func (configurator *network) IsDADFAIL() bool {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if configurator.address == nil || !IsIPv6(configurator.address.IP.String()) {
return false
}
// Get all the address
addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
addresses, err := netlink.AddrList(configurator.link, netlink.FAMILY_V6)
if err != nil {
return false
}
@@ -604,7 +579,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
return address.Flags&unix.IFA_F_DADFAILED != 0
}
// isSet - Check to see if VIP is set
// IsSet - Check to see if VIP is set
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
@@ -612,11 +587,11 @@ func (configurator *network) IsSet() (result bool, err error) {
return false, nil
}
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
addresses, err = netlink.AddrList(configurator.link, 0)
if err != nil {
err = errors.Wrap(err, "could not list addresses")
return false, err
return
}
for _, address := range addresses {
@@ -633,30 +608,13 @@ func (configurator *network) SetIP(ip string) error {
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if strings.Contains("/", ip) {
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
}
ones, _ := configurator.address.Mask.Size()
cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
if err != nil {
return fmt.Errorf("could not format address '%s' with subnetMask '%s'", ip, strconv.Itoa(ones))
}
addr, err := netlink.ParseAddr(cidr)
addr, err := netlinkParse(ip)
if err != nil {
return err
}
if configurator.address != nil && configurator.IsDNS() {
addr.ValidLft = defaultValidLft
} else {
addr.ValidLft = math.MaxInt
}
// set address as deprecated so it isn't used as source address according to RFC 3484
addr.PreferedLft = 0
configurator.address = addr
return nil
}
@@ -680,21 +638,6 @@ func (configurator *network) IP() string {
return configurator.address.IP.String()
}
func (configurator *network) CIDR() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
return configurator.address.IPNet.String()
}
// IP - return the IP Address
func (configurator *network) IPisLinkLocal() bool {
configurator.mu.Lock()
defer configurator.mu.Unlock()
return configurator.address.IP.IsLinkLocalUnicast()
}
// DNSName return the configured dnsName when use DNS
func (configurator *network) DNSName() string {
return configurator.dnsName
@@ -720,23 +663,19 @@ func (configurator *network) DDNSHostName() string {
// Interface - return the Interface name
func (configurator *network) Interface() string {
return configurator.link.Intf.Attrs().Name
return configurator.link.Attrs().Name
}
func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
func GarbageCollect(adapter, address string) (found bool, err error) {
// Get adapter
link, err := netlink.LinkByName(adapter)
if err != nil {
return true, errors.Wrapf(err, "could not get link for interface '%s'", adapter)
}
l := intfMgr.Get(link)
l.Lock.Lock()
defer l.Lock.Unlock()
// Get addresses on adapter
addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
return false, err
}
@@ -748,7 +687,7 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
found = true
// linting issue
existing := existing
if err = netlink.AddrDel(l.Intf, &existing); err != nil {
if err = netlink.AddrDel(link, &existing); err != nil {
return true, errors.Wrap(err, "could not delete ip")
}
}
@@ -757,83 +696,14 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
}
func (configurator *network) SetMask(mask string) error {
selectedMask, err := SelectSubnet(configurator.IP(), mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
m, err := strconv.Atoi(selectedMask)
m, err := strconv.Atoi(mask)
if err != nil {
return err
}
size := 32
family := "IPv4"
if IsIPv6(configurator.IP()) {
if IsIPv6(configurator.address.IP.String()) {
size = 128
family = "IPv6"
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
toSet := net.CIDRMask(m, size)
if toSet == nil {
return fmt.Errorf("failed to create mask /%d", m)
}
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.address.Mask = toSet
configurator.address.IPNet.Mask = net.CIDRMask(m, size)
return nil
}
func (configurator *network) SetHasEndpoints(value bool) {
log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
configurator.hasEndpoints = value
}
func (configurator *network) HasEndpoints() bool {
log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
return configurator.hasEndpoints
}
func (configurator *network) ARPName() string {
return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
}
// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {
// Split the SubnetMasks input into DualStack or SingleStack
// If the input is "32,128", it will be split into ["32", "128"]
subnetMasksParts := strings.Split(subnetMasks, ",")
if len(subnetMasksParts) == 0 {
return "", fmt.Errorf("no subnetMasks provided got: %q", subnetMasks)
} else if len(subnetMasksParts) > 2 {
return "", fmt.Errorf("invalid subnetMasks provided got: %q", subnetMasks)
}
if slices.Contains(subnetMasksParts, "auto") {
return "", fmt.Errorf("auto subnet discovery only works for services: %q", subnetMasks)
}
// Parse the raw IP address
ip := net.ParseIP(rawIP)
if ip == nil {
return "", fmt.Errorf("invalid IP address: %s", rawIP)
}
if ip.To4() != nil {
return subnetMasksParts[0], nil
}
if ip.To16() != nil {
subnetMask := subnetMasksParts[0]
if len(subnetMasksParts) == 2 {
subnetMask = subnetMasksParts[1]
}
return subnetMask, nil
}
return "", fmt.Errorf("unable to select subnet mask for: IP %q and masks %q", rawIP, subnetMasks)
}

View File

@@ -135,7 +135,9 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
Halen: m.hardwareAddressLength,
}
target := ethernetBroadcast
copy(ll.Addr[:], target)
for i := 0; i < len(target); i++ { //nolint
ll.Addr[i] = target[i]
}
b, err := m.bytes()
if err != nil {

View File

@@ -5,9 +5,8 @@ import (
"net"
"time"
log "log/slog"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// DDNSManager will start a dhclient to retrieve and keep the lease for the IP
@@ -56,7 +55,7 @@ func (ddns *ddnsManager) Start() (string, error) {
client.Stop()
return "", errors.New("failed to get IP from dhcp for ddns in 1 minutes")
case ip = <-client.IPChannel():
log.Info("got address from dhcp", "ip", ip)
log.Info("got ip from dhcp: ", ip)
}
// lease.FixedAddress.String() could return <nil>
@@ -75,7 +74,7 @@ func (ddns *ddnsManager) Start() (string, error) {
client.Stop()
return
case ip := <-client.IPChannel():
log.Info("got address from dhcp", "ip", ip)
log.Info("got ip from dhcp: ", ip)
}
}
}(ddns.ctx)

View File

@@ -8,11 +8,10 @@ import (
"net"
"time"
log "log/slog"
"github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/jpillora/backoff"
log "github.com/sirupsen/logrus"
)
const dhcpClientPort = "68"
@@ -130,7 +129,7 @@ func (c *DHCPClient) Start() {
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
log.Debug("dhcp timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
log.Debugf("t1 %v t2 %v", t1Timeout, t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
for {
@@ -144,25 +143,25 @@ func (c *DHCPClient) Start() {
lease, err := c.renew()
if err == nil {
c.lease = lease
log.Info("renew", "lease", lease)
log.Infof("renew, lease: %+v", lease)
t2.Reset(t2Timeout)
} else {
log.Error("renew failed", "err", err)
log.Errorf("renew failed, error: %s", err.Error())
}
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(true)
if err == nil {
c.lease = lease
log.Info("rebind", "lease", lease)
log.Infof("rebind, lease: %+v", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("rebind failed", "err", err)
log.Errorf("rebind failed, error: %s", err.Error())
return
}
log.Warn("ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
log.Warnf("ip %s may have changed: %s", c.lease.ACK.YourIPAddr, err.Error())
c.initRebootFlag = false
c.lease = c.requestWithBackoff()
}
@@ -172,9 +171,9 @@ func (c *DHCPClient) Start() {
case <-c.stopChan:
// release is a unicast request of the IP release.
if err := c.release(); err != nil {
log.Error("release lease failed", "lease", lease, "err", err)
log.Errorf("release lease failed, error: %s, lease: %+v", err.Error(), c.lease)
} else {
log.Info("release", "lease", lease)
log.Infof("release, lease: %+v", c.lease)
}
t1.Stop()
t2.Stop()
@@ -206,18 +205,18 @@ func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
var err error
for {
log.Debug("trying to get a new IP", "attempt", backoff.Attempt())
log.Debugf("trying to get a new IP, attempt %f", backoff.Attempt())
lease, err = c.request(false)
if err != nil {
dur := backoff.Duration()
if backoff.Attempt() > maxBackoffAttempts-1 {
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
log.Error(errMsg.Error())
log.Error(errMsg)
c.errorChan <- errMsg
c.Stop()
return nil
}
log.Error("request failed", "err", err.Error(), "waiting", dur)
log.Errorf("request failed, error: %s (waiting %v)", err.Error(), dur)
time.Sleep(dur)
continue
}
@@ -226,7 +225,7 @@ func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
}
if c.ipChan != nil {
log.Debug("using channel")
log.Debugf("using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
}
@@ -252,13 +251,13 @@ func (c *DHCPClient) request(rebind bool) (*nclient4.Lease, error) {
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
if c.initRebootFlag {
log.Debug("init-reboot", "ip", c.requestedIP)
log.Debugf("init-reboot ip %s", c.requestedIP)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.requestedIP)))
}
// if this is a rebind, then the IP we should set is the one that already exists in lease
if rebind {
log.Debug("rebinding", "ip", c.lease.ACK.YourIPAddr)
log.Debugf("rebinding ip %s", c.lease.ACK.YourIPAddr)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.lease.ACK.YourIPAddr)))
}

View File

@@ -4,7 +4,7 @@ import (
"context"
"time"
log "log/slog"
log "github.com/sirupsen/logrus"
)
// IPUpdater is the interface to plug dns updaters
@@ -29,7 +29,7 @@ func (d *ipUpdater) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
log.Info("stop ipUpdater")
log.Infof("stop ipUpdater")
return
default:
mode := "ipv4"
@@ -39,18 +39,18 @@ func (d *ipUpdater) Run(ctx context.Context) {
ip, err := LookupHost(d.vip.DNSName(), mode)
if err != nil {
log.Warn("cannot lookup", "name", d.vip.DNSName(), "err", err)
log.Warnf("cannot lookup %s: %v", d.vip.DNSName(), err)
// fallback to renewing the existing IP
ip = []string{d.vip.IP()}
}
log.Info("setting IP", "address", ip)
log.Infof("setting %s as an IP", ip)
if err := d.vip.SetIP(ip[0]); err != nil {
log.Error("setting IP", "address", ip, "err", err)
log.Errorf("setting %s as an IP: %v", ip, err)
}
if _, err := d.vip.AddIP(false); err != nil {
log.Error("error adding virtual IP", "err", err)
if err := d.vip.AddIP(false); err != nil {
log.Errorf("error adding virtual IP: %v", err)
}
}

View File

@@ -5,9 +5,8 @@ import (
"strconv"
"strings"
log "log/slog"
iptables "github.com/kube-vip/kube-vip/pkg/iptables"
log "github.com/sirupsen/logrus"
ct "github.com/florianl/go-conntrack"
)
@@ -40,7 +39,7 @@ func CreateIptablesClient(nftables bool, namespace string, protocol iptables.Pro
if protocol == iptables.ProtocolIPv6 {
proto = "IPv6"
}
log.Info("[egress] Creating an iptables client", "nftables", nftables, "protocol", proto)
log.Infof("[egress] Creating an iptables client, nftables mode [%t], protocol [%s]", nftables, proto)
e := new(Egress)
var err error
@@ -57,7 +56,7 @@ func CreateIptablesClient(nftables bool, namespace string, protocol iptables.Pro
}
func (e *Egress) CheckMangleChain(name string) (bool, error) {
log.Info("[egress] chain exists", "name", name)
log.Infof("[egress] Checking for Chain [%s]", name)
return e.ipTablesClient.ChainExists("mangle", name)
}
@@ -69,12 +68,8 @@ func (e *Egress) DeleteManglePrerouting(name string) error {
return e.ipTablesClient.Delete("mangle", "PREROUTING", "-j", name)
}
func (e *Egress) DeleteMangleReturnForNetwork(name, network string) error {
return e.ipTablesClient.Delete("mangle", name, "-d", network, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
}
func (e *Egress) DeleteMangleMarking(podIP, name string) error {
log.Info("[egress] Stopping marking packets on network", "podIP", podIP)
log.Infof("[egress] Stopping marking packets on network [%s]", podIP)
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", podIP, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
@@ -84,19 +79,8 @@ func (e *Egress) DeleteMangleMarking(podIP, name string) error {
return e.ipTablesClient.Delete("mangle", name, "-s", podIP, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
}
func (e *Egress) DeleteMangleMarkingForNetwork(podIP, name, network string) error {
log.Info("[egress] Stopping marking packets", "podIP", podIP, "network", network)
// exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", podIP, "-d", network, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
// if !exists {
// return fmt.Errorf("unable to find source Mangle rule for [%s]", podIP)
// }
return e.ipTablesClient.Delete("mangle", name, "-s", podIP, "-d", network, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
}
func (e *Egress) DeleteSourceNat(podIP, vip string) error {
log.Info("[egress] Removing source nat", "podIP", podIP, "vip", vip)
log.Infof("[egress] Removing source nat from [%s] => [%s]", podIP, vip)
exists, _ := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-m", "comment", "--comment", e.comment)
@@ -107,7 +91,7 @@ func (e *Egress) DeleteSourceNat(podIP, vip string) error {
}
func (e *Egress) DeleteSourceNatForDestinationPort(podIP, vip, port, proto string) error {
log.Info("[egress] Removing source nat", "podIP", podIP, "vip", vip, "destination port", port)
log.Infof("[egress] Removing source nat from [%s] => [%s], with destination port [%s]", podIP, vip, port)
exists, _ := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-p", proto, "--dport", port, "-m", "comment", "--comment", e.comment)
@@ -119,13 +103,13 @@ func (e *Egress) DeleteSourceNatForDestinationPort(podIP, vip, port, proto strin
func (e *Egress) CreateMangleChain(name string) error {
log.Info("[egress] Creating Chain", "name", name)
log.Infof("[egress] Creating Chain [%s]", name)
// Creates a new chain in the mangle table
return e.ipTablesClient.NewChain("mangle", name)
}
func (e *Egress) AppendReturnRulesForDestinationSubnet(name, subnet string) error {
log.Info("[egress] Adding jump for subnet to RETURN to previous chain/rules", "subnet", subnet)
log.Infof("[egress] Adding jump for subnet [%s] to RETURN to previous chain/rules", subnet)
exists, _ := e.ipTablesClient.Exists("mangle", name, "-d", subnet, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
if !exists {
return e.ipTablesClient.Append("mangle", name, "-d", subnet, "-j", "RETURN", "-m", "comment", "--comment", e.comment)
@@ -134,7 +118,7 @@ func (e *Egress) AppendReturnRulesForDestinationSubnet(name, subnet string) erro
}
func (e *Egress) AppendReturnRulesForMarking(name, subnet string) error {
log.Info("[egress] Marking packets on network", "subnet", subnet)
log.Infof("[egress] Marking packets on network [%s]", subnet)
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", subnet, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
if !exists {
return e.ipTablesClient.Append("mangle", name, "-s", subnet, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
@@ -142,17 +126,8 @@ func (e *Egress) AppendReturnRulesForMarking(name, subnet string) error {
return nil
}
func (e *Egress) AppendReturnRulesForMarkingForNetwork(name, subnet, destination string) error {
log.Info("[egress] Marking packets on network", "subnet", subnet)
exists, _ := e.ipTablesClient.Exists("mangle", name, "-s", subnet, "-d", destination, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
if !exists {
return e.ipTablesClient.Append("mangle", name, "-s", subnet, "-d", destination, "-j", "MARK", "--set-mark", "64/64", "-m", "comment", "--comment", e.comment)
}
return nil
}
func (e *Egress) InsertMangeTableIntoPrerouting(name string) error {
log.Info("[egress] Adding jump from mangle prerouting", "destination", name)
log.Infof("[egress] Adding jump from mangle prerouting to [%s]", name)
if exists, err := e.ipTablesClient.Exists("mangle", "PREROUTING", "-j", name, "-m", "comment", "--comment", e.comment); err != nil {
return err
} else if exists {
@@ -165,7 +140,7 @@ func (e *Egress) InsertMangeTableIntoPrerouting(name string) error {
}
func (e *Egress) InsertSourceNat(vip, podIP string) error {
log.Info("[egress] Adding source nat", "original source", podIP, "new source", vip)
log.Infof("[egress] Adding source nat from [%s] => [%s]", podIP, vip)
if exists, err := e.ipTablesClient.Exists("nat", "POSTROUTING", "-s", podIP+"/32", "-m", "mark", "--mark", "64/64", "-j", "SNAT", "--to-source", vip, "-m", "comment", "--comment", e.comment); err != nil {
return err
} else if exists {
@@ -178,17 +153,17 @@ func (e *Egress) InsertSourceNat(vip, podIP string) error {
}
func (e *Egress) InsertSourceNatForDestinationPort(vip, podIP, port, proto string) error {
log.Info("[egress] Adding source nat", "from", podIP, "to", vip, "port", port)
log.Infof("[egress] Adding source nat from [%s] => [%s], with destination port [%s]", podIP, vip, port)
natRules, err := e.ipTablesClient.List("nat", "POSTROUTING")
if err != nil {
return err
}
foundNatRules := e.findExistingVIP(natRules, vip)
log.Warn("[egress] Cleaning existing postrouting nat rules for vip", "rulecount", len(foundNatRules), "vip", vip)
log.Warnf("[egress] Cleaning [%d] existing postrouting nat rules for vip [%s]", len(foundNatRules), vip)
for x := range foundNatRules {
err = e.ipTablesClient.Delete("nat", "POSTROUTING", foundNatRules[x][2:]...)
if err != nil {
log.Error("[egress] removing rule", "err", err)
log.Errorf("[egress] Error removing rule [%v]", err)
}
}
@@ -207,13 +182,13 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
nfct, err := ct.Open(&ct.Config{})
if err != nil {
log.Error("create conntrack client", "err", err)
log.Errorf("could not create nfct: %v", err)
return err
}
defer nfct.Close()
sessions, err := nfct.Dump(ct.Conntrack, ct.IPv4)
if err != nil {
log.Error("could not dump sessions", "err", err)
log.Errorf("could not dump sessions: %v", err)
return err
}
destPortProtocol := make(map[uint16]uint8)
@@ -240,7 +215,7 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
case strings.ToLower("sctp"):
destPortProtocol[uint16(port)] = ProtocolSCTP
default:
log.Error("[egress] annotation protocol isn't supported", "protocol ID", data[0])
log.Errorf("[egress] annotation protocol [%s] isn't supported", data[0])
}
}
}
@@ -266,7 +241,7 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
case strings.ToLower("sctp"):
srcPortProtocol[uint16(port)] = ProtocolSCTP
default:
log.Error("[egress] annotation protocol isn't supported", "protocol ID", data[0])
log.Errorf("[egress] annotation protocol [%s] isn't supported", data[0])
}
}
}
@@ -279,14 +254,14 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
if destinationPorts != "" {
proto := destPortProtocol[*session.Origin.Proto.DstPort]
if proto == *session.Origin.Proto.Number {
log.Info("[egress] cleaning existing connection", "src", session.Origin.Src.String(), "dst", session.Origin.Dst.String(), "dst port", *session.Origin.Proto.DstPort, "protocol", *session.Origin.Proto.Number)
log.Infof("[egress] cleaning existing connection Source [%s] -> [%s:%d] proto: [%d] ", session.Origin.Src.String(), session.Origin.Dst.String(), *session.Origin.Proto.DstPort, *session.Origin.Proto.Number)
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
}
} else {
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
}
if err != nil {
log.Error("could not delete sessions", "err", err)
log.Errorf("could not delete sessions: %v", err)
}
}
}
@@ -299,14 +274,14 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
if srcPorts != "" {
proto := srcPortProtocol[*session.Origin.Proto.DstPort]
if proto == *session.Origin.Proto.Number {
log.Info("[egress] cleaning existing connection", "src", session.Origin.Src.String(), "dst", session.Origin.Dst.String(), "dst port", *session.Origin.Proto.DstPort, "protocol", *session.Origin.Proto.Number)
log.Infof("[egress] cleaning existing connection Source [%s] -> [%s:%d] proto: [%d] ", session.Origin.Src.String(), session.Origin.Dst.String(), *session.Origin.Proto.DstPort, *session.Origin.Proto.Number)
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
}
} else {
err = nfct.Delete(ct.Conntrack, ct.IPv4, session)
}
if err != nil {
log.Error("could not delete sessions", "err", err)
log.Errorf("could not delete sessions: %v", err)
}
}
}
@@ -318,13 +293,13 @@ func DeleteExistingSessions(sessionIP string, destination bool, destinationPorts
// Debug functions
func (e *Egress) DumpChain(name string) error {
log.Info("Dumping chain", "name", name)
log.Infof("Dumping chain [%s]", name)
c, err := e.ipTablesClient.List("mangle", name)
if err != nil {
return err
}
for x := range c {
log.Info("", "rule", c[x])
log.Infof("Rule -> %s", c[x])
}
return nil
}
@@ -335,16 +310,16 @@ func (e *Egress) CleanIPtables() error {
return err
}
foundNatRules := e.findRules(natRules)
log.Warn("[egress] Cleaning dangling postrouting nat rules", "rulecount", len(foundNatRules))
log.Warnf("[egress] Cleaning [%d] dangling postrouting nat rules", len(foundNatRules))
for x := range foundNatRules {
err = e.ipTablesClient.Delete("nat", "POSTROUTING", foundNatRules[x][2:]...)
if err != nil {
log.Error("[egress] Error removing rule", "err", err)
log.Errorf("[egress] Error removing rule [%v]", err)
}
}
exists, err := e.CheckMangleChain(MangleChainName)
if err != nil {
log.Debug("[egress] No Mangle chain exists", "err", err)
log.Debugf("[egress] No Mangle chain exists [%v]", err)
}
if exists {
mangleRules, err := e.ipTablesClient.List("mangle", MangleChainName)
@@ -352,11 +327,11 @@ func (e *Egress) CleanIPtables() error {
return err
}
foundNatRules = e.findRules(mangleRules)
log.Warn("[egress] Cleaning dangling prerouting mangle rules", "rulecount", len(foundNatRules))
log.Warnf("[egress] Cleaning [%d] dangling prerouting mangle rules", len(foundNatRules))
for x := range foundNatRules {
err = e.ipTablesClient.Delete("mangle", MangleChainName, foundNatRules[x][2:]...)
if err != nil {
log.Error("[egress] Error removing rule", "err", err)
log.Errorf("[egress] Error removing rule [%v]", err)
}
}
@@ -368,7 +343,7 @@ func (e *Egress) CleanIPtables() error {
// log.Errorf("[egress] Error removing flushing table [%v]", err)
// }
} else {
log.Warn("No existing mangle chain exists", "chain name", MangleChainName)
log.Warnf("No existing mangle chain [%s] exists", MangleChainName)
}
return nil
}
@@ -409,13 +384,13 @@ func (e *Egress) findExistingVIP(rules []string, vip string) [][]string {
func ClearIPTables(useNftables bool, namespace string, protocol iptables.Protocol) {
i, err := CreateIptablesClient(useNftables, namespace, protocol)
if err != nil {
log.Warn("[egress] Unable to clean any dangling egress rules", "err", err)
log.Warnf("[egress] Unable to clean any dangling egress rules [%v]", err)
log.Warn("[egress] Can be ignored in non iptables release of kube-vip")
} else {
log.Info("[egress] Cleaning any dangling kube-vip egress rules")
cleanErr := i.CleanIPtables()
if cleanErr != nil {
log.Error("Error cleaning rules", "err", cleanErr)
log.Errorf("Error cleaning rules [%v]", cleanErr)
}
}
}

View File

@@ -7,7 +7,7 @@ import (
"github.com/mdlayher/ndp"
log "log/slog"
log "github.com/sirupsen/logrus"
)
// NdpResponder defines the parameters for the NDP connection.
@@ -50,7 +50,7 @@ func (n *NdpResponder) SendGratuitous(address string) error {
return fmt.Errorf("failed to parse address %s", ip)
}
log.Info("Broadcasting NDP update", "ip", address, "hwaddr", n.hardwareAddr, "interface", n.intf)
log.Infof("Broadcasting NDP update for %s (%s) via %s", address, n.hardwareAddr, n.intf)
return n.advertise(netip.IPv6LinkLocalAllNodes(), ip, true)
}
@@ -66,7 +66,6 @@ func (n *NdpResponder) advertise(dst, target netip.Addr, gratuitous bool) error
},
},
}
log.Debug("ndp", "advertisement", m)
log.Infof("ndp: %v", m)
return n.conn.WriteTo(m, nil, dst)
}

View File

@@ -8,9 +8,8 @@ import (
"strings"
"syscall"
log "log/slog"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
@@ -121,6 +120,17 @@ func IsIPv6CIDR(cidr string) bool {
return ip.To4() == nil
}
// GetFullMask returns /32 for an IPv4 address and /128 for an IPv6 address
func GetFullMask(address string) (string, error) {
if IsIPv4(address) {
return "/32", nil
}
if IsIPv6(address) {
return "/128", nil
}
return "", fmt.Errorf("failed to parse %s as either IPv4 or IPv6", address)
}
// GetDefaultGatewayInterface return default gateway interface link
func GetDefaultGatewayInterface() (*net.Interface, error) {
routes, err := netlink.RouteList(nil, syscall.AF_INET)
@@ -157,7 +167,7 @@ func MonitorDefaultInterface(ctx context.Context, defaultIF *net.Interface) erro
for {
select {
case r := <-routeCh:
log.Debug(fmt.Sprintf("type: %d, route: %+v", r.Type, r.Route))
log.Debugf("type: %d, route: %+v", r.Type, r.Route)
if r.Type == syscall.RTM_DELROUTE && (r.Dst == nil || r.Dst.String() == "0.0.0.0/0") && r.LinkIndex == defaultIF.Index {
return fmt.Errorf("default route deleted and the default interface may be invalid")
}
@@ -181,7 +191,7 @@ func GenerateMac() (mac string) {
* - https://macaddress.io/database-download
*/
mac = fmt.Sprintf("%s:%s:%s:%02x:%02x:%02x", "00", "00", "6C", buf[0], buf[1], buf[2])
log.Info("Generated mac", "address", mac)
log.Infof("Generated mac: %s", mac)
return mac
}
@@ -192,50 +202,3 @@ func Split(values string) []string {
}
return result
}
// GetInterfaceByIP returns the network interface that has the specified IP address assigned.
func GetInterfaceByIP(ipAddr string) (*netlink.Link, error) {
ip := net.ParseIP(ipAddr)
if ip == nil {
return nil, fmt.Errorf("invalid IP address: %s", ipAddr)
}
links, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("failed to list network interfaces: %v", err)
}
for i := range links {
addrs, err := netlink.AddrList(links[i], netlink.FAMILY_ALL)
if err != nil {
return nil, fmt.Errorf("failed to list addresses for interface %s: %v", links[i].Attrs().Name, err)
}
for _, addr := range addrs {
if addr.IP.Equal(ip) {
return &links[i], nil
}
}
}
return nil, fmt.Errorf("no interface found with IP address: %s", ipAddr)
}
// GetNonLinkLocalIP returns the first non link-local IPv4/IPv6 address on the given interface.
func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
a, err := netlink.AddrList(*iface, family)
if err != nil {
return "", fmt.Errorf("failed to list addresses for interface %s: %v", (*iface).Attrs().Name, err)
}
for _, addr := range a {
if addr.IPNet != nil {
ip := addr.IPNet.IP
if !ip.IsLinkLocalUnicast() {
return ip.String(), nil
}
}
}
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
}

13
test.sh
View File

@@ -1,13 +0,0 @@
#!/usr/bin/bash
rm -f logs.txt
for i in $(seq 1 100);
do
echo "RUN $i"
GOMAXPROCS=4 make e2e-tests129
if [[ "$?" -ne 0 ]]; then
echo "FAILED AT RUN $i"
break
fi
done

View File

@@ -1,3 +0,0 @@
[global.config]
as = {{ .AS }}
router-id = "1.1.1.1"

View File

@@ -9,7 +9,7 @@ import (
"strings"
"time"
log "log/slog"
log "github.com/sirupsen/logrus"
)
func main() {

View File

@@ -1,683 +0,0 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"text/template"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
api "github.com/osrg/gobgp/v3/api"
)
const (
goBGPAS uint32 = 65500
kubevipAS uint32 = 65501
goBGPPort uint32 = 50051
defaultFixedNexthopv6 = "fc00:1000:1000:1000::100"
defaultFixedNexthopv4 = "172.18.0.100"
)
var _ = Describe("kube-vip BGP mode", Ordered, func() {
if Mode == ModeBGP {
var (
logger log.Logger
imagePath string
k8sImagePath string
configPath string
kubeVIPBGPManifestTemplate *template.Template
goBGPConfigTemplate *template.Template
tempDirPath string
v129 bool
localIPv4 string
localIPv6 string
curDir string
networkInterface string
bgpKill chan any
)
BeforeAll(func() {
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
if configPath == "" {
configPath = "/etc/kubernetes/admin.conf"
}
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
_, v129 = os.LookupEnv("V129")
var err error
curDir, err = os.Getwd()
Expect(err).NotTo(HaveOccurred())
templateBGPPath := filepath.Join(curDir, "kube-vip-bgp.yaml.tmpl")
kubeVIPBGPManifestTemplate, err = template.New("kube-vip-bgp.yaml.tmpl").ParseFiles(templateBGPPath)
Expect(err).NotTo(HaveOccurred())
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).NotTo(HaveOccurred())
v4addr, _, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv4 = v4addr.String()
v6addr, _, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv6 = v6addr.String()
goBGPConfig := &e2e.BGPPeerValues{
AS: goBGPAS,
}
bgpKill = make(chan any)
goBGPConfigPath := filepath.Join(filepath.Join(curDir, "bgp"), "config.toml.tmpl")
goBGPConfigTemplate, err = template.New("config.toml.tmpl").ParseFiles(goBGPConfigPath)
Expect(err).ToNot(HaveOccurred())
goBGPConfigPath = filepath.Join(tempDirPath, "config.toml")
f, err := os.OpenFile(goBGPConfigPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
Expect(err).ToNot(HaveOccurred())
defer f.Close()
err = goBGPConfigTemplate.Execute(f, goBGPConfig)
Expect(err).ToNot(HaveOccurred())
go startGoBGP(goBGPConfigPath, bgpKill)
})
AfterAll(func() {
close(bgpKill)
})
Describe("kube-vip IPv4 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv4Family, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip IPv6 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv6Family, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - auto_sourceif nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
_, containerIP = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
containerIP, _ = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
}
})
func testBGP(offset uint, lbFamily string, afiFamily api.Family_Afi, svcFamily []corev1.IPFamily, svcName string,
trafficPolicy corev1.ServiceExternalTrafficPolicy, client kubernetes.Interface, numberOfServices int, gobgpClient api.GobgpApiClient, expectedNexthop string) {
lbAddress := e2e.GenerateVIP(lbFamily, offset)
routeCheckFamily := &api.Family{
Afi: afiFamily,
Safi: api.Family_SAFI_UNICAST,
}
testServiceBGP(svcName, lbAddress, trafficPolicy, client, svcFamily, numberOfServices, gobgpClient, routeCheckFamily, expectedNexthop)
}
func setupEnv(tempDirPath, cpVIP, clusterName *string, manifestValues *e2e.KubevipManifestValues,
localIPv4, localIPv6, imagePath, configPath, k8sImagePath, clusterAddrFamily, bgpClientAddrFamily string, peerAddrFamily []string, client *kubernetes.Interface,
gobgpPeers *[]*e2e.BGPPeerValues, v129 bool, kubeVIPBGPManifestTemplate *template.Template, gobgpClient *api.GobgpApiClient,
logger log.Logger, nodesNumber int, mpbgpnexthop, clusterNameSuffix string) (string, string) {
var err error
*tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).ToNot(HaveOccurred())
*cpVIP = e2e.GenerateVIP(clusterAddrFamily, SOffset.Get())
var clusterIPFamily kindconfigv1alpha4.ClusterIPFamily
var podSubnet, serviceSubnet string
switch clusterAddrFamily {
case e2e.IPv6Family:
clusterIPFamily = kindconfigv1alpha4.IPv6Family
case e2e.DualstackFamily:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
case e2e.DualstackFamilyIPv6:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
podSubnet = "fd00:10:244::/56,10.244.0.0/16"
serviceSubnet = "fd00:10:96::/112,10.96.0.0/16"
default:
clusterIPFamily = kindconfigv1alpha4.IPv4Family
}
networking := &kindconfigv1alpha4.Networking{
IPFamily: clusterIPFamily,
}
if podSubnet != "" && serviceSubnet != "" {
networking.PodSubnet = podSubnet
networking.ServiceSubnet = serviceSubnet
}
kvPeers := []*e2e.BGPPeerValues{}
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv4,
AS: goBGPAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv6,
AS: goBGPAS,
IPFamily: e2e.IPv6Family,
})
}
kvPeersStr := []string{}
for _, p := range kvPeers {
kvPeersStr = append(kvPeersStr, p.String())
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: *cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
BGPAS: kubevipAS,
BGPPeers: strings.Join(kvPeersStr, ","),
MPBGPNexthop: mpbgpnexthop,
MPBGPNexthopIPv4: defaultFixedNexthopv4,
MPBGPNexthopIPv6: defaultFixedNexthopv6,
}
By(manifestValues.BGPPeers)
*clusterName, *client = prepareCluster(*tempDirPath, clusterNameSuffix, k8sImagePath, v129, kubeVIPBGPManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
container := fmt.Sprintf("%s-control-plane", *clusterName)
containerIPv4, containerIPv6, err := GetContainerIPs(container)
Expect(err).ToNot(HaveOccurred())
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv4,
AS: kubevipAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv6,
AS: kubevipAS,
IPFamily: e2e.IPv6Family,
})
}
if bgpClientAddrFamily == e2e.IPv6Family {
*gobgpClient, err = newGoBGPClient(localIPv6, goBGPPort)
} else {
*gobgpClient, err = newGoBGPClient(localIPv4, goBGPPort)
}
Expect(err).ToNot(HaveOccurred())
for _, p := range *gobgpPeers {
if slices.Contains(peerAddrFamily, p.IPFamily) {
peerCtx := context.TODO()
Eventually(peerCtx, func() error {
_, err = (*gobgpClient).AddPeer(context.TODO(), &api.AddPeerRequest{
Peer: &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: p.IP,
PeerAsn: uint32(p.AS),
},
AfiSafis: []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
},
},
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
},
},
},
},
})
return err
}, "120s", "100ms").Should(Succeed())
}
}
return containerIPv4, containerIPv6
}
func testServiceBGP(svcName, lbAddress string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
client kubernetes.Interface, serviceAddrFamily []corev1.IPFamily, numberOfServices int,
gobgpClient api.GobgpApiClient, gobgpFamily *api.Family, expectedNexthop string) {
lbAddresses := vip.Split(lbAddress)
services := []string{}
for i := range numberOfServices {
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
}
for _, svc := range services {
createTestService(svc, dsNamespace, dsName, lbAddress,
client, corev1.IPFamilyPolicyPreferDualStack, serviceAddrFamily, trafficPolicy)
}
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
for i := range numberOfServices {
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
if i < numberOfServices-1 {
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
}
}
for _, addr := range lbAddresses {
checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 0)
}
}
func GetContainerIPs(containerName string) (string, string, error) {
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
panic(err)
}
containers, err := cli.ContainerList(context.Background(), container.ListOptions{})
if err != nil {
return "", "", fmt.Errorf("failed to list containers: %w", err)
}
for _, c := range containers {
for _, n := range c.Names {
if n[1:] == containerName {
fmt.Println(n)
for _, n := range c.NetworkSettings.Networks {
return n.IPAddress, n.GlobalIPv6Address, nil
}
}
}
}
return "", "", nil
}
func newGoBGPClient(address string, port uint32) (api.GobgpApiClient, error) {
grpcOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
target := net.JoinHostPort(address, strconv.Itoa(int(port)))
conn, err := grpc.NewClient(target, grpcOpts...)
if err != nil {
return nil, fmt.Errorf("failed to connect to GoBGP server %q: %w", target, err)
}
return api.NewGobgpApiClient(conn), nil
}
func checkGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedPaths int) []*api.Destination {
var paths []*api.Destination
Eventually(func() error {
var err error
paths, err = getGoBGPPaths(ctx, client, family, prefixes)
if err != nil {
return err
}
if len(paths) != expectedPaths {
return fmt.Errorf("expected %d paths, but found %d", expectedPaths, len(paths))
}
return nil
}, "120s").ShouldNot(HaveOccurred())
return paths
}
func getGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix) ([]*api.Destination, error) {
pathCtx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
stream, err := client.ListPath(pathCtx, &api.ListPathRequest{
TableType: api.TableType_GLOBAL,
Family: family,
Name: "",
Prefixes: prefixes,
SortType: api.ListPathRequest_PREFIX,
})
if err != nil {
return nil, err
}
rib := make([]*api.Destination, 0)
for {
r, err := stream.Recv()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
rib = append(rib, r.Destination)
}
return rib, nil
}
func startGoBGP(config string, kill chan any) {
By("starting GoBGP server")
cmd := exec.Command("../../bin/gobgpd", "-f", config)
go cmd.Run()
<-kill
By("stopping GoBGP server")
err := cmd.Process.Kill()
Expect(err).ToNot(HaveOccurred())
}

View File

@@ -1,583 +0,0 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"text/template"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
)
var _ = Describe("kube-vip routing table mode", func() {
if Mode == ModeRT {
var (
logger log.Logger
imagePath string
k8sImagePath string
configPath string
kubeVIPRoutingTableManifestTemplate *template.Template
tempDirPath string
v129 bool
)
BeforeEach(func() {
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
if configPath == "" {
configPath = "/etc/kubernetes/admin.conf"
}
_, v129 = os.LookupEnv("V129")
curDir, err := os.Getwd()
Expect(err).NotTo(HaveOccurred())
templateRoutingTablePath := filepath.Join(curDir, "kube-vip-routing-table.yaml.tmpl")
kubeVIPRoutingTableManifestTemplate, err = template.New("kube-vip-routing-table.yaml.tmpl").ParseFiles(templateRoutingTablePath)
Expect(err).NotTo(HaveOccurred())
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).NotTo(HaveOccurred())
})
Describe("kube-vip IPv4 control-plane routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv4Family,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups IPv4 address and route on control-plane node", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
checkIPAddress(cpVIP, container, true)
e2e.CheckRoutePresence(cpVIP, container, true)
}
})
})
Describe("kube-vip IPv6 control-plane routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv6Family,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups IPv6 address and route on control-plane node", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
checkIPAddress(cpVIP, container, true)
e2e.CheckRoutePresence(cpVIP, container, true)
}
})
})
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv4 primary", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv6, localIPv6Net, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv6,
ipnet: localIPv6Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups DualStack addresses and routes on control-plane nodes", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
addresses := vip.Split(cpVIP)
for _, addr := range addresses {
checkIPAddress(addr, container, true)
e2e.CheckRoutePresence(addr, container, true)
}
}
})
})
Describe("kube-vip DualStack control-plane routing table mode functionality - IPv6 primary", Ordered, func() {
var (
cpVIP string
clusterName string
nodesNumber = 3
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
}
manifestValues := &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "true",
SvcEnable: "false",
SvcElectionEnable: "false",
}
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv4, localIPv4Net, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv4,
ipnet: localIPv4Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
It("setups DualStack addresses and routes on control-plane nodes", func() {
By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
time.Sleep(30 * time.Second)
for i := 1; i <= nodesNumber; i++ {
var container string
if i > 1 {
container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
addresses := vip.Split(cpVIP)
for _, addr := range addresses {
checkIPAddress(addr, container, true)
e2e.CheckRoutePresence(addr, container, true)
}
}
})
})
Describe("kube-vip IPv4 services routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv4Family,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip IPv6 services routing table mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.IPv6Family,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services routing table mode functionality - IPv4 primary", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
EnableEndpointslices: "true",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 and IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services routing table mode functionality - IPv6 primary", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
svcElection bool
ipFamily []corev1.IPFamily
nodesNumber = 1
)
BeforeAll(func() {
cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
networking := &kindconfigv1alpha4.Networking{
IPFamily: kindconfigv1alpha4.DualStackFamily,
PodSubnet: "fd00:10:244::/56,10.244.0.0/16",
ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
EnableEndpointslices: "true",
}
var err error
svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
Expect(err).ToNot(HaveOccurred())
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("configures an IPv4 and IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateDualStackVIP(offset)
testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
}
})
func testServiceRT(svcName, lbAddress, leaseName, leaseNamespace, clusterName string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
client kubernetes.Interface, serviceElection bool, ipFamily []corev1.IPFamily, numberOfServices int) {
lbAddresses := vip.Split(lbAddress)
services := []string{}
for i := range numberOfServices {
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
}
for _, svc := range services {
createTestService(svc, dsNamespace, dsName, lbAddress,
client, corev1.IPFamilyPolicyPreferDualStack, ipFamily, trafficPolicy)
}
var container string
if serviceElection {
container = e2e.GetLeaseHolder(leaseName, leaseNamespace, client)
} else {
container = fmt.Sprintf("%s-control-plane", clusterName)
}
for _, addr := range lbAddresses {
e2e.CheckRoutePresence(addr, container, true)
}
for i := range numberOfServices {
expected := i < numberOfServices-1
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
for _, addr := range lbAddresses {
e2e.CheckRoutePresence(addr, container, expected)
}
}
}
type san struct {
ip *net.IP
ipnet *net.IPNet
}

View File

@@ -4,41 +4,15 @@
package e2e_test
import (
"log"
"os"
"sync"
"testing"
"github.com/kube-vip/kube-vip/testing/e2e"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
const (
ModeEnv = "TEST_MODE"
ModeARP = "arp"
ModeRT = "rt"
ModeBGP = "bgp"
)
var (
SOffset *e2e.SecureOffset
ConfigMtx *sync.Mutex
Mode string
"github.com/kube-vip/kube-vip/testing/e2e"
)
func TestE2E(t *testing.T) {
mode := os.Getenv(ModeEnv)
if mode == "" {
Mode = ModeARP
} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
log.Fatal("invalid", "mode", mode)
os.Exit(1)
} else {
Mode = mode
}
SOffset = e2e.NewOffset(5)
ConfigMtx = &sync.Mutex{}
RegisterFailHandler(Fail)
RunSpecs(t, "E2E Suite")
}

File diff suppressed because it is too large Load Diff

View File

@@ -268,6 +268,15 @@ func (c *Cluster) Delete() {
Expect(c.provider.Delete(c.Name, "")).To(Succeed())
}
func startKubeletForEtcd(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
)
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
)
}
func createCerts(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm",

View File

@@ -60,7 +60,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
test.vip = e2e.GenerateVIP(e2e.IPv4Family, 5)
test.vip = e2e.GenerateIPv4VIP()
test.logger.Printf("Selected VIP %s", test.vip)
test.currentDir, err = os.Getwd()

View File

@@ -32,7 +32,7 @@ spec:
- name: cp_enable
value: "true"
- name: vip_loglevel
value: "-4"
value: "5"
- name: vip_nodename
valueFrom:
fieldRef:

View File

@@ -6,34 +6,22 @@ package e2e
import (
"bufio"
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"net"
"os/exec"
"strings"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/vip"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
coordinationv1 "k8s.io/api/coordination/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/cluster"
)
const (
IPv4Family = "IPv4"
IPv6Family = "IPv6"
DualstackFamily = "DualStack"
DualstackFamilyIPv6 = "DualStackIPv6"
)
func EnsureKindNetwork() {
By("checking if the Docker \"kind\" network exists")
cmd := exec.Command("docker", "inspect", "kind")
@@ -70,47 +58,53 @@ func EnsureKindNetwork() {
Eventually(session).Should(gexec.Exit(0))
}
func GenerateVIP(family string, offset uint) string {
if family == DualstackFamily || family == DualstackFamilyIPv6 {
return fmt.Sprintf("%s,%s", GenerateVIP(IPv4Family, offset), GenerateVIP(IPv6Family, offset))
}
func GenerateIPv6VIP() string {
cidrs := getKindNetworkSubnetCIDRs()
for _, cidr := range cidrs {
if cidr != "" {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if ip.To4() == nil && family == IPv6Family {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
if ip.To4() == nil {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
return net.IP(chosenVIP).String()
} else if ip.To4() != nil && family == IPv4Family {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
return net.IP(chosenVIP).String()
}
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
return net.IP(chosenVIP).String()
}
}
Fail("Could not find any " + family + " CIDRs in the Docker \"kind\" network")
Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
return ""
}
func GenerateDualStackVIP(offset uint) string {
return GenerateVIP(IPv4Family, offset) + "," + GenerateVIP(IPv6Family, offset)
func GenerateIPv4VIP() string {
cidrs := getKindNetworkSubnetCIDRs()
for _, cidr := range cidrs {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if ip.To4() != nil {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-5)
return net.IP(chosenVIP).String()
}
}
Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
return ""
}
func GenerateDualStackVIP() string {
return GenerateIPv4VIP() + "," + GenerateIPv6VIP()
}
func getKindNetworkSubnetCIDRs() []string {
@@ -139,88 +133,30 @@ func getKindNetworkSubnetCIDRs() []string {
return cidrs
}
func CheckIPAddressPresence(ip string, container string, expected bool) bool {
func CheckIPAddressPresence(ip string, container string) bool {
cmdOut := new(bytes.Buffer)
family := "-4"
if vip.IsIPv6(ip) {
family = "-6"
}
cmd := exec.Command(
"docker", "exec", container, "ip", family, "addr", "show", "dev", "eth0",
)
cmdOut := new(bytes.Buffer)
cmdErr := new(bytes.Buffer)
cmd.Stdout = cmdOut
cmd.Stderr = cmdErr
if err := cmd.Run(); err != nil {
return false
}
return strings.Contains(cmdOut.String(), ip) == expected
Eventually(cmd.Run(), "20s").Should(Succeed())
return strings.Contains(cmdOut.String(), ip)
}
func CheckIPAddressPresenceByLease(name, namespace, ip string, client kubernetes.Interface, expected bool) bool {
container := GetLeaseHolder(name, namespace, client)
if container == "" {
return false
}
return CheckIPAddressPresence(ip, container, expected)
}
func CheckRoutePresence(ip string, container string, expected bool) bool {
func CheckRoutePresence(ip string, container string) bool {
cmdOut := new(bytes.Buffer)
family := "-4"
if vip.IsIPv6(ip) {
family = "-6"
}
result := false
Eventually(func() bool {
cmdOut := new(bytes.Buffer)
cmdErr := new(bytes.Buffer)
cmd := exec.Command(
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
)
cmd.Stdout = cmdOut
cmd.Stderr = cmdErr
cmd.Run()
result = strings.Contains(cmdOut.String(), ip) == expected
return result
}, "60s", "1s").Should(BeTrue())
return result
}
func GetLeaseHolder(name, namespace string, client kubernetes.Interface) string {
var lease *coordinationv1.Lease
Eventually(func() error {
var err error
lease, err = client.CoordinationV1().Leases(namespace).Get(context.TODO(), name, metav1.GetOptions{})
return err
}, "300s").ShouldNot(HaveOccurred())
Expect(lease).ToNot(BeNil())
return *lease.Spec.HolderIdentity
}
type SecureOffset struct {
value int
mu sync.Mutex
}
func NewOffset(value int) *SecureOffset {
return &SecureOffset{
value: value,
}
}
func (so *SecureOffset) Get() uint {
so.mu.Lock()
defer so.mu.Unlock()
so.value++
return uint(so.value - 1)
cmd := exec.Command(
"docker", "exec", container, "ip", family, "route", "show", "table", "198",
)
cmd.Stdout = cmdOut
Eventually(cmd.Run(), "20s").Should(Succeed())
return strings.Contains(cmdOut.String(), ip)
}

View File

@@ -1,82 +0,0 @@
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
spec:
containers:
- name: kube-vip
args:
- manager
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: lb_class_legacy_handling
value: "false"
- name: lb_class_name
- name: prometheus_server
value: :2112
- name: disable_service_updates
value: "false"
- name: vip_arp
value: "false"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: vip_leaderelection
value: "false"
- name: vip_interface
value: lo
- name: bgp_enable
value: "true"
- name: bgp_hold_time
value: "3"
- name: bgp_keepalive_interval
value: "1"
- name: bgp_sourceif
value: "eth0"
- name: bgp_routerid
value: "2.2.2.2"
- name: bgp_as
value: "{{ .BGPAS }}"
- name: bgp_peers
value: "{{ .BGPPeers }}"
- name: mpbgp_nexthop
value: "{{ .MPBGPNexthop }}"
- name: mpbgp_ipv4
value: "{{ .MPBGPNexthopIPv4 }}"
- name: mpbgp_ipv6
value: "{{ .MPBGPNexthopIPv6 }}"
- name: address
value: "{{ .ControlPlaneVIP }}"
- name: cp_enable
value: "{{ .ControlPlaneEnable }}"
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "{{ .ConfigPath }}"
name: kubeconfig

View File

@@ -12,8 +12,6 @@ spec:
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: vip_arp
value: "false"
- name: vip_interface
@@ -29,23 +27,19 @@ spec:
- name: vip_retryperiod
value: "1"
- name: cp_enable
value: "{{ .ControlPlaneEnable }}"
value: "true"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_subnet
value: '32,128'
- name: vip_cidr
value: '32'
- name: cp_namespace
value: kube-system
- name: svc_enable
value: 'false'
- name: vip_routingtable
value: 'true'
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:

View File

@@ -12,8 +12,6 @@ spec:
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: vip_arp
value: "true"
- name: vip_interface
@@ -34,12 +32,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:

View File

@@ -1,4 +1,4 @@
package deployment
package main
import (
"bufio"
@@ -13,7 +13,7 @@ import (
"path/filepath"
"strings"
"github.com/gookit/slog"
log "github.com/sirupsen/logrus"
)
func getKindNetworkSubnetCIDRs() ([]string, error) {
@@ -95,7 +95,7 @@ func generateIPv6VIP() (string, error) {
}
func (config *TestConfig) manifestGen() error {
func (config *testConfig) manifestGen() error {
curDir, err := os.Getwd()
if err != nil {
return err
@@ -140,7 +140,7 @@ func (config *TestConfig) manifestGen() error {
return err
}
}
slog.Infof("🗃️ Manifest path %s", config.ManifestPath)
log.Infof("🗃️ Manifest path %s", config.ManifestPath)
err = kubeVIPManifestTemplate.Execute(manifestFile, kubevipManifestValues{
ControlPlaneVIP: config.ControlPlaneAddress,
ImagePath: config.ImagePath,

View File

@@ -1,15 +1,13 @@
package deployment
package main
import (
"bytes"
"fmt"
"net"
"os/exec"
"strings"
"time"
"github.com/gookit/slog"
"github.com/vishvananda/netlink"
log "github.com/sirupsen/logrus"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/cluster"
"sigs.k8s.io/kind/pkg/cmd"
@@ -18,11 +16,6 @@ import (
var provider *cluster.Provider
const (
defaultIPv4Pool = "172.18.100.10-172.18.100.30"
defaultIPv6Pool = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
)
type kubevipManifestValues struct {
ControlPlaneVIP string
ImagePath string
@@ -33,7 +26,7 @@ type nodeAddresses struct {
addresses []string
}
func (config *TestConfig) CreateKind() error {
func (config *testConfig) createKind() error {
clusterConfig := kindconfigv1alpha4.Cluster{
Networking: kindconfigv1alpha4.Networking{
@@ -45,17 +38,13 @@ func (config *TestConfig) CreateKind() error {
},
},
}
if config.DualStack {
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
} else {
if config.IPv6 || !config.Egress && config.EgressIPv6 {
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
}
if config.IPv6 {
// Change Networking Family
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
}
if config.Cilium {
clusterConfig.Networking.DisableDefaultCNI = true
if config.Dualstack {
// Change Networking Family
clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
}
if config.ControlPlane {
@@ -85,13 +74,6 @@ func (config *TestConfig) CreateKind() error {
//clusterConfig.Nodes = append(clusterConfig.Nodes, kindconfigv1alpha4.Node{Role: kindconfigv1alpha4.WorkerRole})
}
// Change the default image if required
if config.KindVersionImage != "" {
for x := range clusterConfig.Nodes {
clusterConfig.Nodes[x].Image = config.KindVersionImage
}
}
provider = cluster.NewProvider(cluster.ProviderWithLogger(cmd.NewLogger()), cluster.ProviderWithDocker())
clusters, err := provider.List()
if err != nil {
@@ -100,7 +82,7 @@ func (config *TestConfig) CreateKind() error {
found := false
for x := range clusters {
if clusters[x] == "services" {
slog.Infof("Cluster already exists")
log.Infof("Cluster already exists")
found = true
}
}
@@ -121,8 +103,8 @@ func (config *TestConfig) CreateKind() error {
return err
}
if !config.SkipHostnameChange {
slog.Infof("⚙️ changing hostnames on nodes to force using proper node names for service selection")
if !config.skipHostnameChange {
log.Infof("⚙️ changing hostnames on nodes to force using proper node names for service selection")
for _, node := range nodes {
nodeName := node.String()
cmd := exec.Command("docker", "exec", nodeName, "hostname", nodeName+"-modified")
@@ -132,11 +114,6 @@ func (config *TestConfig) CreateKind() error {
}
}
if config.Cilium {
cmd := exec.Command("cilium", "install", "--helm-set", "ipv6.enabled=true", "--helm-set", "--enable-ipv6-ndp=true")
_, _ = cmd.CombinedOutput()
}
// HMMM, if we want to run workloads on the control planes (todo)
if config.ControlPlane {
for _, node := range nodes {
@@ -146,52 +123,11 @@ func (config *TestConfig) CreateKind() error {
}
}
ipv4Pool := defaultIPv4Pool
ipv6Pool := defaultIPv6Pool
var ipv4Net *net.IPNet
var ipv6Net *net.IPNet
links, err := netlink.LinkList()
if err != nil {
return fmt.Errorf("netlink: failed to list links: %w", err)
}
for _, link := range links {
if strings.Contains(link.Attrs().Name, config.DockerNIC) {
_, ipv4Net, err = getNetwork(link, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to get IPv4 network: %w", err)
}
if ipv4Net == nil {
return fmt.Errorf("failed to find IPv4 network on the docker interface")
}
ipv4Pool = generateRange(ipv4Net)
_, ipv6Net, err = getNetwork(link, netlink.FAMILY_V6)
if err != nil {
return fmt.Errorf("failed to get IPv6 network: %w", err)
}
if ipv6Net == nil {
return fmt.Errorf("failed to find IPv6 network on the docker interface")
}
ipv6Pool = generateRange(ipv6Net)
}
}
globalRange := ipv4Pool
globalRange := "172.18.100.10-172.18.100.30"
if config.IPv6 {
globalRange = ipv6Pool
globalRange = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
}
if config.DualStack {
globalRange = fmt.Sprintf("%s,%s", ipv4Pool, ipv6Pool)
}
slog.Infof("Cloud provider's range: %s", globalRange)
cmd := exec.Command("kubectl", "create", "configmap", "--namespace", "kube-system", "kubevip", "--from-literal", "range-global="+globalRange)
if _, err := cmd.CombinedOutput(); err != nil {
return err
@@ -204,51 +140,14 @@ func (config *TestConfig) CreateKind() error {
if _, err := cmd.CombinedOutput(); err != nil {
return err
}
slog.Infof("💤 sleeping for a few seconds to let controllers start")
log.Infof("💤 sleeping for a few seconds to let controllers start")
time.Sleep(time.Second * 5)
}
return nil
}
func getNetwork(link netlink.Link, family int) (*net.IP, *net.IPNet, error) {
addrs, err := netlink.AddrList(link, family)
if err != nil {
return nil, nil, fmt.Errorf("netlink: failed to get addresses for link %q: %w", link.Attrs().Name, err)
}
if len(addrs) > 0 {
for _, a := range addrs {
if a.Scope == int(netlink.SCOPE_UNIVERSE) {
ip, cidr, err := net.ParseCIDR(a.IPNet.String())
if err != nil {
return nil, nil, fmt.Errorf("failed to parse CIDR: %w", err)
}
return &ip, cidr, nil
}
}
}
return nil, nil, nil
}
func generateRange(network *net.IPNet) string {
if network.IP.To4() != nil {
parts := strings.Split(network.IP.String(), ".")
p1 := fmt.Sprintf("%s.%s.100.10", parts[0], parts[1])
p2 := fmt.Sprintf("%s.%s.100.100", parts[0], parts[1])
return fmt.Sprintf("%s-%s", p1, p2)
}
if network.IP.To16() != nil {
parts := strings.Split(network.IP.String(), ":")
p1 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0010", parts[0], parts[1], parts[2], parts[3])
p2 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0100", parts[0], parts[1], parts[2], parts[3])
return fmt.Sprintf("%s-%s", p1, p2)
}
return ""
}
func DeleteKind() error {
slog.Info("🧽 deleting Kind cluster")
func deleteKind() error {
log.Info("🧽 deleting Kind cluster")
return provider.Delete("services", "")
}

View File

@@ -1,15 +1,15 @@
package deployment
package main
import (
"context"
"fmt"
"time"
"github.com/gookit/slog"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
@@ -17,19 +17,18 @@ import (
)
// service defines the settings for a new service
type Service struct {
name string
egress bool // enable egress
egressInternal bool // enable Internal egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
type service struct {
name string
egress bool // enable egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
}
type Deployment struct {
replicas int32
type deployment struct {
replicas int
server bool
client bool
address string
@@ -37,7 +36,7 @@ type Deployment struct {
name string
}
func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clientset, imagepath string) error {
func (d *deployment) createKVDs(ctx context.Context, clientset *kubernetes.Clientset, imagepath string) error {
ds := appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-vip-ds",
@@ -72,7 +71,7 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
Value: "true",
},
{
Name: "vip_subnet",
Name: "vip_cidr",
Value: "32",
},
{
@@ -93,7 +92,7 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
},
{
Name: "vip_loglevel",
Value: "-4",
Value: "5",
},
{
Name: "egress_withnftables",
@@ -133,8 +132,8 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
return nil
}
func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
replicas := d.replicas
func (d *deployment) createDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
replicas := int32(d.replicas)
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: d.name,
@@ -156,7 +155,7 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
Containers: []v1.Container{
{
Name: "kube-vip-web",
Image: "docker.io/plndr/e2e:0.0.1",
Image: "plndr/e2e:0.0.1",
Ports: []v1.ContainerPort{
{
Name: "http",
@@ -205,11 +204,11 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
return err
}
slog.Infof("📝 created deployment [%s]", result.GetObjectMeta().GetName())
log.Infof("📝 created deployment [%s]", result.GetObjectMeta().GetName())
return nil
}
func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clientset) (currentLeader string, loadBalancerAddresses []string, err error) {
func (s *service) createService(ctx context.Context, clientset *kubernetes.Clientset) (currentLeader string, loadBalancerAddresses []string, err error) {
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: s.name,
@@ -239,20 +238,10 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
"kube-vip.io/egress": "true",
}
}
if s.egressInternal {
svc.Annotations["kube-vip.io/egress-internal"] = "true"
}
if s.egressIPv6 {
svc.Annotations["kube-vip.io/egress-ipv6"] = "true"
}
//svc.Annotations["kube-vip.io/egress-denied-networks"] = "172.18.0.0/24"
//svc.Annotations["kube-vip.io/egress-allowed-networks"] = "172.18.0.0/24"
//svc.Annotations["kube-vip.io/egress-allowed-networks"] = "192.168.0.0/24, 172.18.0.0/24"
if s.policyLocal {
svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
}
@@ -263,11 +252,11 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
}
ipv4VIP, err := generateIPv4VIP()
if err != nil {
slog.Fatal(err)
log.Fatal(err)
}
ipv6VIP, err := generateIPv6VIP()
if err != nil {
slog.Fatal(err)
log.Fatal(err)
}
svc.Annotations["kube-vip.io/loadbalancerIPs"] = fmt.Sprintf("%s,%s", ipv4VIP, ipv6VIP)
svc.Labels["implementation"] = "kube-vip"
@@ -276,19 +265,19 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
svc.Spec.IPFamilyPolicy = &ipfPolicy
}
slog.Infof("🌍 creating service [%s]", svc.Name)
log.Infof("🌍 creating service [%s]", svc.Name)
_, err = clientset.CoreV1().Services(v1.NamespaceDefault).Create(ctx, svc, metav1.CreateOptions{})
if err != nil {
slog.Fatal(err)
log.Fatal(err)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
slog.Fatal(err)
log.Fatal(err)
}
ch := rw.ResultChan()
go func() {
@@ -303,16 +292,17 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
slog.Fatalf("unable to parse Kubernetes services from API watcher")
log.Fatalf("unable to parse Kubernetes services from API watcher")
}
if svc.Name == s.name {
if len(svc.Status.LoadBalancer.Ingress) != 0 {
for _, ingress := range svc.Status.LoadBalancer.Ingress {
loadBalancerAddresses = append(loadBalancerAddresses, ingress.IP)
}
slog.Infof("🔎 found load balancer addresses [%s] on node [%s]", loadBalancerAddresses, svc.Annotations["kube-vip.io/vipHost"])
log.Infof("🔎 found load balancer addresses [%s] on node [%s]", loadBalancerAddresses, svc.Annotations["kube-vip.io/vipHost"])
ready = true
currentLeader = svc.Annotations["kube-vip.io/vipHost"]
}
@@ -328,7 +318,6 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
for _, lbAddress := range loadBalancerAddresses {
err = httpTest(lbAddress)
if err != nil {
slog.Infof("web retrieval err: %s", err.Error())
return "", nil, fmt.Errorf("web retrieval timeout ")
}

View File

@@ -0,0 +1,725 @@
//nolint:govet
package main
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"time"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
watchtools "k8s.io/client-go/tools/watch"
)
// Methodology
// 1. Create a deployment
// 2. Expose the deployment
func (config *testConfig) startServiceTest(ctx context.Context, clientset *kubernetes.Clientset) {
nodeTolerate := os.Getenv("NODE_TOLERATE")
d := "kube-vip-deploy"
s := "kube-vip-service"
l := "kube-vip-deploy-leader"
if !config.ignoreSimple {
// Simple Deployment test
log.Infof("🧪 ---> simple deployment <---")
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 2,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
name: s,
testHTTP: true,
timeout: 10,
}
_, _, err = svc.createService(ctx, clientset)
if err != nil {
log.Error(err)
} else {
config.successCounter++
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreDeployments {
// Multiple deployment tests
log.Infof("🧪 ---> multiple deployments <---")
deploy := deployment{
name: l,
nodeAffinity: nodeTolerate,
replicas: 2,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
if err != nil {
log.Fatal(err)
}
for i := 1; i < 5; i++ {
svc := service{
name: fmt.Sprintf("%s-%d", s, i),
testHTTP: true,
timeout: 30,
}
_, _, err = svc.createService(ctx, clientset)
if err != nil {
log.Fatal(err)
}
config.successCounter++
}
for i := 1; i < 5; i++ {
log.Infof("🧹 deleting service [%s]", fmt.Sprintf("%s-%d", s, i))
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, fmt.Sprintf("%s-%d", s, i), metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
log.Infof("🧹 deleting deployment [%s]", d)
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, l, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreLeaderFailover {
// Failover tests
log.Infof("🧪 ---> leader failover deployment (local policy) <---")
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 2,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
name: s,
egress: false,
policyLocal: true,
testHTTP: true,
timeout: 180,
}
leader, lbAddresses, err := svc.createService(ctx, clientset)
if err != nil {
log.Error(err)
}
lbAddress := lbAddresses[0]
err = leaderFailover(ctx, &s, &leader, clientset)
if err != nil {
log.Error(err)
} else {
config.successCounter++
}
// Get all addresses on all nodes
nodes, err := getAddressesOnNodes()
if err != nil {
log.Error(err)
}
// Make sure we don't exist in two places
err = checkNodesForDuplicateAddresses(nodes, lbAddress)
if err != nil {
log.Fatal(err)
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreLeaderActive {
// pod Failover tests
log.Infof("🧪 ---> active pod failover deployment (local policy) <---")
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 1,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
name: s,
policyLocal: true,
testHTTP: true,
timeout: 30,
}
leader, _, err := svc.createService(ctx, clientset)
if err != nil {
log.Error(err)
}
err = podFailover(ctx, &s, &leader, clientset)
if err != nil {
log.Error(err)
} else {
config.successCounter++
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreLocalDeploy {
// Multiple deployment tests
log.Infof("🧪 ---> multiple deployments (local policy) <---")
timeout := 30
deploy := deployment{
name: l,
nodeAffinity: nodeTolerate,
replicas: 2,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
for i := 1; i < 5; i++ {
svc := service{
policyLocal: true,
name: fmt.Sprintf("%s-%d", s, i),
testHTTP: true,
timeout: timeout,
}
_, lbAddresses, err := svc.createService(ctx, clientset)
if err != nil {
log.Fatal(err)
}
lbAddress := lbAddresses[0]
config.successCounter++
nodes, err := getAddressesOnNodes()
if err != nil {
log.Error(err)
}
err = checkNodesForDuplicateAddresses(nodes, lbAddress)
if err != nil {
log.Fatal(err)
}
}
for i := 1; i < 5; i++ {
log.Infof("🧹 deleting service [%s]", fmt.Sprintf("%s-%d", s, i))
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, fmt.Sprintf("%s-%d", s, i), metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
log.Infof("🧹 deleting deployment [%s]", d)
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, l, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreEgress {
// pod Failover tests
log.Infof("🧪 ---> egress IP re-write (local policy) <---")
var egress string
var found bool
timeout := 30
// Set up a local listener
go func() {
found = tcpServer(false, &egress, timeout)
}()
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 1,
client: true,
}
// Find this machines IP address
deploy.address = GetLocalIPv4()
if deploy.address == "" {
log.Fatalf("Unable to detect local IP address")
}
log.Infof("📠 found local address [%s]", deploy.address)
// Create a deployment that connects back to this machines IP address
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
policyLocal: true,
name: s,
egress: true,
testHTTP: false,
timeout: 30,
}
_, lbAddresses, err := svc.createService(ctx, clientset)
if err != nil {
log.Fatal(err)
}
egress = lbAddresses[0]
for i := 1; i < 10; i++ {
if found {
log.Infof("🕵️ egress has correct IP address")
config.successCounter++
break
}
time.Sleep(time.Second * 1)
}
if !found {
log.Error("😱 No traffic found from loadbalancer address ")
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreEgressIPv6 {
// pod Failover tests
log.Infof("🧪 ---> egress IP re-write IPv6 (local policy) <---")
var egress string
var found bool
timeout := 60
// Set up a local listener
go func() {
found = tcpServer(true, &egress, timeout)
}()
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 1,
client: true,
}
// Find this machines IP address
deploy.address = GetLocalIPv6()
if deploy.address == "" {
log.Fatalf("Unable to detect local IP address")
}
log.Infof("📠 found local address [%s]", deploy.address)
// Create a deployment that connects back to this machines IP address
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
policyLocal: true,
name: s,
egress: true,
egressIPv6: true,
testHTTP: false,
timeout: timeout,
testDualstack: true,
}
_, lbAddresses, err := svc.createService(ctx, clientset)
if err != nil {
log.Fatal(err)
}
for x := range lbAddresses {
ip := net.ParseIP(lbAddresses[x])
// if ip == nil {
// return errors.New("invalid address")
// }
if ip.To4() == nil {
// use brackets for IPv6 address
egress = lbAddresses[x]
break
}
}
for i := 1; i < timeout; i++ {
if found {
log.Infof("🕵️ egress has correct IP address")
config.successCounter++
break
}
time.Sleep(time.Second * 1)
}
if !found {
log.Error("😱 No traffic found from loadbalancer address ")
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
if !config.ignoreDualStack {
// Dualstack loadbalancer test
log.Infof("🧪 ---> testing dualstack loadbalancer service <---")
deploy := deployment{
name: d,
nodeAffinity: nodeTolerate,
replicas: 2,
server: true,
}
err := deploy.createDeployment(ctx, clientset)
if err != nil {
log.Fatal(err)
}
svc := service{
name: s,
testHTTP: true,
testDualstack: true,
timeout: 30,
}
_, _, err = svc.createService(ctx, clientset)
if err != nil {
log.Error(err)
} else {
config.successCounter++
}
log.Infof("🧹 deleting Service [%s], deployment [%s]", s, d)
err = clientset.CoreV1().Services(v1.NamespaceDefault).Delete(ctx, s, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
err = clientset.AppsV1().Deployments(v1.NamespaceDefault).Delete(ctx, d, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
}
log.Infof("🏆 Testing Complete [%d] passed", config.successCounter)
}
func httpTest(address string) error {
log.Infof("🕷️ testing HTTP request against [%s]", address)
Client := http.Client{
Timeout: 2 * time.Second,
}
ip := net.ParseIP(address)
if ip == nil {
return errors.New("invalid address")
}
if ip.To4() == nil {
// use brackets for IPv6 address
address = fmt.Sprintf("[%s]", address)
}
var err error
for i := 0; i < 5; i++ {
var r *http.Response
//nolint
r, err = Client.Get(fmt.Sprintf("http://%s", address)) //nolint
if err == nil {
log.Infof("🕸️ successfully retrieved web data in [%ds]", i)
r.Body.Close()
return nil
}
time.Sleep(time.Second * 2)
}
return err
}
func leaderFailover(ctx context.Context, name, leaderNode *string, clientset *kubernetes.Clientset) error {
go func() {
log.Infof("💀 killing leader five times")
for i := 0; i < 5; i++ {
p, err := clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{})
if err != nil {
log.Fatal(err)
}
for x := range p.Items {
if p.Items[x].Spec.NodeName == *leaderNode {
if p.Items[x].Spec.Containers[0].Name == "kube-vip" {
err = clientset.CoreV1().Pods("kube-system").Delete(ctx, p.Items[x].Name, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
log.Infof("🔪 leader pod [%s] has been deleted", p.Items[x].Name)
}
}
}
time.Sleep(time.Second * 5)
}
}()
log.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return err
}
ch := rw.ResultChan()
go func() {
time.Sleep(time.Second * 30)
rw.Stop()
}()
// Used for tracking an active endpoint / pod
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
log.Fatalf("unable to parse Kubernetes services from API watcher")
}
if svc.Name == *name {
if len(svc.Status.LoadBalancer.Ingress) != 0 {
log.Infof("🔎 found load balancer address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
}
}
case watch.Modified:
svc, ok := event.Object.(*v1.Service)
if !ok {
log.Fatalf("unable to parse Kubernetes services from API watcher")
}
if svc.Name == *name {
if len(svc.Status.LoadBalancer.Ingress) != 0 {
log.Infof("🔍 updated with address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
err = httpTest(svc.Status.LoadBalancer.Ingress[0].IP)
if err != nil {
return err
}
*leaderNode = svc.Annotations["kube-vip.io/vipHost"]
}
}
default:
}
}
return nil
}
func podFailover(ctx context.Context, name, leaderNode *string, clientset *kubernetes.Clientset) error {
go func() {
log.Infof("💀 killing active pod five times")
for i := 0; i < 5; i++ {
p, err := clientset.CoreV1().Pods(v1.NamespaceDefault).List(ctx, metav1.ListOptions{})
if err != nil {
log.Fatal(err)
}
found := false
for x := range p.Items {
if p.Items[x].Spec.NodeName == *leaderNode {
if p.Items[x].Spec.Containers[0].Name == "kube-vip-web" {
found = true
err = clientset.CoreV1().Pods(v1.NamespaceDefault).Delete(ctx, p.Items[x].Name, metav1.DeleteOptions{})
if err != nil {
log.Fatal(err)
}
log.Infof("🔪 active pod [%s] on [%s] has been deleted", p.Items[x].Name, p.Items[x].Spec.NodeName)
}
}
}
if !found {
log.Warnf("😱 No Pod found on [%s]", *leaderNode)
}
time.Sleep(time.Second * 5)
}
}()
log.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return err
}
ch := rw.ResultChan()
go func() {
time.Sleep(time.Second * 30)
rw.Stop()
}()
// Used for tracking an active endpoint / pod
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
log.Fatalf("unable to parse Kubernetes services from API watcher")
}
if svc.Name == *name {
if len(svc.Status.LoadBalancer.Ingress) != 0 {
log.Infof("🔎 found load balancer address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
}
}
case watch.Modified:
svc, ok := event.Object.(*v1.Service)
if !ok {
log.Fatalf("unable to parse Kubernetes services from API watcher")
}
if svc.Name == *name {
if len(svc.Status.LoadBalancer.Ingress) != 0 {
log.Infof("🔍 updated with address [%s] on node [%s]", svc.Status.LoadBalancer.Ingress[0].IP, svc.Annotations["kube-vip.io/vipHost"])
err = httpTest(svc.Status.LoadBalancer.Ingress[0].IP)
if err != nil {
log.Fatal(err)
}
*leaderNode = svc.Annotations["kube-vip.io/vipHost"]
}
}
default:
}
}
return nil
}
func tcpServer(IPv6 bool, egressAddress *string, timeout int) bool {
var listen net.Listener
var err error
if !IPv6 {
listen, err = net.Listen("tcp", ":12345") //nolint
} else {
listen, err = net.Listen("tcp6", ":12346") //nolint
}
if err != nil {
log.Error(err)
}
// close listener
go func() {
time.Sleep(time.Second * time.Duration(timeout))
listen.Close()
}()
for {
conn, err := listen.Accept()
if err != nil {
return false
// log.Fatal(err)
}
remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
// If it is IPv6 expand the address
// if IPv6 {
// addr, _ := netip.ParseAddr(remoteAddress)
// remoteAddress = addr.StringExpanded()
// }
if remoteAddress == *egressAddress {
log.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
return true
}
log.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
go handleRequest(conn)
}
}
func handleRequest(conn net.Conn) {
// incoming request
buffer := make([]byte, 1024)
_, err := conn.Read(buffer)
if err != nil {
log.Error(err)
}
// write data to response
time := time.Now().Format(time.ANSIC)
responseStr := fmt.Sprintf("Your message is: %v. Received time: %v", string(buffer[:]), time)
_, err = conn.Write([]byte(responseStr))
if err != nil {
log.Error(err)
}
// close conn
conn.Close()
}
func GetLocalIPv4() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
return ""
}
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
return ipnet.IP.String()
}
}
}
return ""
}
func GetLocalIPv6() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
return ""
}
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() == nil && !ipnet.IP.IsLinkLocalUnicast() {
if ipnet.IP.To16() != nil {
return ipnet.IP.String()
}
}
}
return ""
}

Some files were not shown because too many files have changed in this diff Show More