feat(egress): discover cluster CIDRs from Kubernetes APIs

Read ServiceCIDRs and Node PodCIDRs through the Processor client, retain a Service-only controller-manager fallback, and grant generated ClusterRoles get/list/watch access to ServiceCIDRs.

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

pkg/kubevip/config_generator.go

@@ -125,6 +125,13 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
 			},
 		},
 	}
+	if !role {
+		newManifest.Rules = append(newManifest.Rules, applyRbacV1.PolicyRuleApplyConfiguration{
+			APIGroups: []string{"networking.k8s.io"},
+			Resources: []string{"servicecidrs"},
+			Verbs:     []string{"list", "get", "watch"},
+		})
+	}
 	return newManifest
 }
 

pkg/kubevip/config_generator_test.go

@@ -2,10 +2,38 @@ package kubevip
 
 import (
 	"os"
+	"slices"
 	"strings"
 	"testing"
+
+	applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
 )
 
+func TestGenerateRoleServiceCIDRAccess(t *testing.T) {
+	clusterRole := GenerateRole(&Config{}, false)
+	if !hasServiceCIDRRule(clusterRole) {
+		t.Fatal("generated ClusterRole is missing ServiceCIDR access")
+	}
+
+	role := GenerateRole(&Config{ServiceNamespace: "kube-vip"}, true)
+	if hasServiceCIDRRule(role) {
+		t.Fatal("generated namespaced Role contains ineffective ServiceCIDR access")
+	}
+}
+
+func hasServiceCIDRRule(role *applyRbacV1.RoleApplyConfiguration) bool {
+	for _, rule := range role.Rules {
+		if slices.Contains(rule.APIGroups, "networking.k8s.io") &&
+			slices.Contains(rule.Resources, "servicecidrs") &&
+			slices.Contains(rule.Verbs, "get") &&
+			slices.Contains(rule.Verbs, "list") &&
+			slices.Contains(rule.Verbs, "watch") {
+			return true
+		}
+	}
+	return false
+}
+
 func TestParseEnvironment(t *testing.T) {
 
 	tests := []struct {

pkg/services/egress.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"slices"
 	"strings"
 
 	log "log/slog"
@@ -17,6 +18,7 @@ import (
 	"github.com/kube-vip/kube-vip/pkg/utils"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/retry"
 )
@@ -392,31 +394,88 @@ func (p *Processor) prepareEgressNftablesTable(serviceUID string, ipv6 bool) err
 
 func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
 	log.Debug("Trying to automatically discover Service and Pod CIDRs")
+	serviceCIDR, podCIDR = p.discoverCIDRsFromAPI(ctx)
+	if serviceCIDR == "" {
+		legacyServiceCIDR, legacyErr := p.discoverServiceCIDRFromControllerManager(ctx)
+		if legacyErr != nil {
+			return serviceCIDR, podCIDR, legacyErr
+		}
+		serviceCIDR = legacyServiceCIDR
+	}
+	if podCIDR == "" {
+		return serviceCIDR, podCIDR, fmt.Errorf("unable to determine CNI Pod CIDRs from Node objects; configure egress_podcidr explicitly")
+	}
+
+	return serviceCIDR, podCIDR, nil
+}
+
+func (p *Processor) discoverCIDRsFromAPI(ctx context.Context) (serviceCIDR, podCIDR string) {
+	serviceCIDRs, err := p.clientSet.NetworkingV1().ServiceCIDRs().List(ctx, metav1.ListOptions{})
+	if err == nil {
+		serviceCIDR = serviceCIDRsFromItems(serviceCIDRs.Items)
+	} else {
+		log.Debug("Unable to discover Service CIDRs from ServiceCIDR API", "err", err)
+	}
+
+	nodes, err := p.clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err == nil {
+		podCIDR = podCIDRsFromNodes(nodes.Items)
+	} else {
+		log.Debug("Unable to discover CNI Pod CIDRs from Node API", "err", err)
+	}
+
+	return serviceCIDR, podCIDR
+}
+
+func (p *Processor) discoverServiceCIDRFromControllerManager(ctx context.Context) (string, error) {
 	options := metav1.ListOptions{
 		LabelSelector: "component=kube-controller-manager",
 	}
 	podList, err := p.clientSet.CoreV1().Pods("kube-system").List(ctx, options)
 	if err != nil {
-		return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
+		return "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
 	}
 	if len(podList.Items) < 1 {
-		return "", "", fmt.Errorf("[Egress] Unable to auto-discover the pod/service CIDRs: kube-controller-manager not found")
+		return "", fmt.Errorf("[Egress] Unable to auto-discover the Service CIDRs: kube-controller-manager not found")
 	}
 
 	pod := podList.Items[0]
-	for flags := range pod.Spec.Containers[0].Command {
-		if strings.Contains(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=") {
-			podCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--cluster-cidr=", "")
-		}
-		if strings.Contains(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=") {
-			serviceCIDR = strings.ReplaceAll(pod.Spec.Containers[0].Command[flags], "--service-cluster-ip-range=", "")
+	for _, flag := range pod.Spec.Containers[0].Command {
+		if strings.Contains(flag, "--service-cluster-ip-range=") {
+			return strings.ReplaceAll(flag, "--service-cluster-ip-range=", ""), nil
 		}
 	}
-	if podCIDR == "" || serviceCIDR == "" {
-		err = fmt.Errorf("unable to fully determine cluster CIDR configurations")
-	}
+	return "", fmt.Errorf("unable to determine Service CIDR configuration")
+}
 
-	return
+func serviceCIDRsFromItems(items []networkingv1.ServiceCIDR) string {
+	var cidrs []string
+	for _, item := range items {
+		cidrs = appendUnique(cidrs, item.Spec.CIDRs...)
+	}
+	return strings.Join(cidrs, ",")
+}
+
+func podCIDRsFromNodes(nodes []corev1.Node) string {
+	var cidrs []string
+	for _, node := range nodes {
+		if len(node.Spec.PodCIDRs) > 0 {
+			cidrs = appendUnique(cidrs, node.Spec.PodCIDRs...)
+		} else if node.Spec.PodCIDR != "" {
+			cidrs = appendUnique(cidrs, node.Spec.PodCIDR)
+		}
+	}
+	return strings.Join(cidrs, ",")
+}
+
+func appendUnique(values []string, additions ...string) []string {
+	for _, addition := range additions {
+		if addition == "" || slices.Contains(values, addition) {
+			continue
+		}
+		values = append(values, addition)
+	}
+	return values
 }
 
 func (p *Processor) updateEgressNftablesTableAnnotation(ctx context.Context, service *corev1.Service) error {

pkg/services/egress_cidr_test.go

@@ -0,0 +1,79 @@
+package services
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestServiceCIDRsFromItems(t *testing.T) {
+	serviceCIDR := serviceCIDRsFromItems([]networkingv1.ServiceCIDR{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "kubernetes"},
+			Spec: networkingv1.ServiceCIDRSpec{
+				CIDRs: []string{"10.96.0.0/16", "fd00:10:96::/112"},
+			},
+		},
+	})
+
+	if serviceCIDR != "10.96.0.0/16,fd00:10:96::/112" {
+		t.Fatalf("serviceCIDR = %q", serviceCIDR)
+	}
+}
+
+func TestPodCIDRsFromNodes(t *testing.T) {
+	tests := []struct {
+		name  string
+		nodes []corev1.Node
+		want  string
+	}{
+		{
+			name: "dual stack across nodes",
+			nodes: []corev1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "node-a"},
+					Spec: corev1.NodeSpec{
+						PodCIDRs: []string{"10.244.0.0/24", "fd00:10:244::/64"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "node-b"},
+					Spec: corev1.NodeSpec{
+						PodCIDRs: []string{"10.244.1.0/24", "fd00:10:244:1::/64"},
+					},
+				},
+			},
+			want: "10.244.0.0/24,fd00:10:244::/64,10.244.1.0/24,fd00:10:244:1::/64",
+		},
+		{
+			name: "legacy singular CIDR",
+			nodes: []corev1.Node{
+				{Spec: corev1.NodeSpec{PodCIDR: "10.244.0.0/24"}},
+			},
+			want: "10.244.0.0/24",
+		},
+		{
+			name: "duplicate CIDRs",
+			nodes: []corev1.Node{
+				{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
+				{Spec: corev1.NodeSpec{PodCIDRs: []string{"10.244.0.0/16"}}},
+			},
+			want: "10.244.0.0/16",
+		},
+		{
+			name:  "missing CNI CIDRs",
+			nodes: []corev1.Node{{}},
+			want:  "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := podCIDRsFromNodes(test.nodes); got != test.want {
+				t.Fatalf("podCIDRsFromNodes() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}