fix(egress): isolate nftables tables by instance

Signed-off-by: Gabriele Pennacchia <gabriele@pennacchia.it>
This commit is contained in:
Gabriele Pennacchia
2026-07-14 09:47:34 +02:00
parent 18fd79aa5d
commit 60c786b537
19 changed files with 613 additions and 59 deletions

View File

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
- Reintroduce BGP config via node annotations. Fixes #1488.
- Fail fast in runtime `manager` and `service` paths when legacy `vip_address` is used without `vip_subnet` in control-plane ARP, BGP, or Routing Table mode.

View File

@@ -167,6 +167,7 @@ func init() {
// Configuration file flag
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.InstanceName, "instanceName", "", "Unique name for this kube-vip instance (currently used to isolate nftables egress tables)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EgressWithNftables, "egressWithNftables", true, "Use nftables-based egress implementation")

View File

@@ -19,15 +19,13 @@ func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[strin
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
IPv6 := false
if utils.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" || useNftables {
return nftables.DeleteSNAT(IPv6, serviceUUID)
return nftables.DeleteSNATFromAllTables(serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)

View File

@@ -37,6 +37,9 @@ const (
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// The nftables egress table base name that owns this Service's SNAT chain
EgressNftablesTable = "kube-vip.io/egress-nftables-table"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"

View File

@@ -32,6 +32,13 @@ func ParseEnvironment(c *Config) error {
c.Logging = int32(logLevel)
}
if env = os.Getenv(instanceName); env == "" {
env = os.Getenv(strings.ToUpper(instanceName))
}
if env != "" {
c.InstanceName = env
}
// Find interface
env = os.Getenv(vipInterface)
if env != "" {
@@ -908,7 +915,6 @@ func mergeConfigValues(baseConfig, fileConfig *Config) {
if baseConfig.ServicesLeaseName == "" && fileConfig.ServicesLeaseName != "" {
baseConfig.ServicesLeaseName = fileConfig.ServicesLeaseName
}
// LoadBalancer configuration
if baseConfig.LoadBalancerPort == 0 && fileConfig.LoadBalancerPort != 0 {
baseConfig.LoadBalancerPort = fileConfig.LoadBalancerPort
@@ -970,6 +976,11 @@ func mergeConfigValues(baseConfig, fileConfig *Config) {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort
}
// Instance configuration
if baseConfig.InstanceName == "" && fileConfig.InstanceName != "" {
baseConfig.InstanceName = fileConfig.InstanceName
}
// Egress configuration
if baseConfig.EgressPodCidr == "" && fileConfig.EgressPodCidr != "" {
baseConfig.EgressPodCidr = fileConfig.EgressPodCidr
@@ -977,7 +988,6 @@ func mergeConfigValues(baseConfig, fileConfig *Config) {
if baseConfig.EgressServiceCidr == "" && fileConfig.EgressServiceCidr != "" {
baseConfig.EgressServiceCidr = fileConfig.EgressServiceCidr
}
// Mirror configuration
if baseConfig.MirrorDestInterface == "" && fileConfig.MirrorDestInterface != "" {
baseConfig.MirrorDestInterface = fileConfig.MirrorDestInterface

View File

@@ -33,6 +33,9 @@ const (
// vipLogLevel - defines the level of logging to produce (5 being the most verbose)
vipLogLevel = "vip_loglevel"
// instanceName identifies this kube-vip deployment when naming host-global resources
instanceName = "instance_name"
// vipInterface - defines the interface that the vip should bind too
vipInterface = "vip_interface"

View File

@@ -34,6 +34,7 @@ address: "192.168.1.100"
port: 6443
interface: "eth0"
namespace: "kube-system"
instanceName: "release_a"
vipSubnet: "192.168.1.0/24"
leaseName: "test-lease"
leaseDuration: 15
@@ -50,6 +51,7 @@ prometheusHTTPServer: ":2112"
Port: 6443,
Interface: "eth0",
Namespace: "kube-system",
InstanceName: "release_a",
VIPSubnet: "192.168.1.0/24",
PrometheusHTTPServer: ":2112",
KubernetesLeaderElection: KubernetesLeaderElection{
@@ -239,6 +241,9 @@ bgpConfig:
if config.Interface != tt.expectedConfig.Interface {
t.Errorf("Interface = %v, expected %v", config.Interface, tt.expectedConfig.Interface)
}
if config.InstanceName != tt.expectedConfig.InstanceName {
t.Errorf("InstanceName = %v, expected %v", config.InstanceName, tt.expectedConfig.InstanceName)
}
// Test BGP config if present
if tt.expectedConfig.EnableBGP {
@@ -424,12 +429,14 @@ func TestMergeConfigValues(t *testing.T) {
Port: 6443,
Interface: "eth0",
Address: "192.168.1.100",
InstanceName: "release_a",
},
expectedBase: &Config{
Logging: 5, // From base (non-zero)
Port: 6443, // From file (base was zero)
Interface: "eth0", // From file (base was empty)
Address: "192.168.1.100", // From file (base was empty)
InstanceName: "release_a", // From file (base was empty)
},
},
{
@@ -516,6 +523,9 @@ func TestMergeConfigValues(t *testing.T) {
if tt.baseConfig.Interface != tt.expectedBase.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expectedBase.Interface)
}
if tt.baseConfig.InstanceName != tt.expectedBase.InstanceName {
t.Errorf("InstanceName = %v, expected %v", tt.baseConfig.InstanceName, tt.expectedBase.InstanceName)
}
if tt.baseConfig.EnableARP != tt.expectedBase.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expectedBase.EnableARP)
}

View File

@@ -202,6 +202,12 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
},
},
}
if c.InstanceName != "" {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: instanceName,
Value: c.InstanceName,
})
}
// If we're specifically saying which interface to use then add it to the manifest
if c.Interface != "" {

View File

@@ -2,6 +2,7 @@ package kubevip
import (
"os"
"strings"
"testing"
)
@@ -25,6 +26,70 @@ func TestParseEnvironment(t *testing.T) {
}
}
func TestParseEnvironmentInstanceName(t *testing.T) {
tests := []struct {
name string
lowercase string
uppercase string
want string
}{
{name: "lowercase", lowercase: "release_a", want: "release_a"},
{name: "uppercase fallback", uppercase: "release_b", want: "release_b"},
{name: "lowercase takes precedence", lowercase: "release_a", uppercase: "release_b", want: "release_a"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv(instanceName, tt.lowercase)
t.Setenv(strings.ToUpper(instanceName), tt.uppercase)
config := &Config{}
if err := ParseEnvironment(config); err != nil {
t.Fatalf("ParseEnvironment() error = %v", err)
}
if config.InstanceName != tt.want {
t.Fatalf("InstanceName = %q, want %q", config.InstanceName, tt.want)
}
})
}
}
func TestGeneratePodSpecInstanceName(t *testing.T) {
tests := []struct {
name string
instanceName string
wantPresent bool
}{
{name: "configured", instanceName: "release_a", wantPresent: true},
{name: "empty", wantPresent: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pod, err := generatePodSpec(&Config{InstanceName: tt.instanceName}, "ghcr.io/kube-vip/kube-vip", "v0.0.0", true)
if err != nil {
t.Fatalf("generatePodSpec() error = %v", err)
}
var value string
found := false
for _, env := range pod.Spec.Containers[0].Env {
if env.Name == instanceName {
found = true
value = env.Value
break
}
}
if found != tt.wantPresent {
t.Fatalf("instance_name present = %t, want %t", found, tt.wantPresent)
}
if found && value != tt.instanceName {
t.Fatalf("instance_name = %q, want %q", value, tt.instanceName)
}
})
}
}
func TestParseEnvironmentConfigFile(t *testing.T) {
// Save original environment
originalConfigFile := os.Getenv("config_file")

View File

@@ -91,6 +91,9 @@ type Config struct {
// Namespace will define which namespace the control plane pods will run in
Namespace string `yaml:"namespace"`
// InstanceName identifies this kube-vip deployment when naming host-global resources.
InstanceName string `yaml:"instanceName"`
// Namespace will define which namespace the control plane pods will run in
ServiceNamespace string `yaml:"serviceNamespace"`

View File

@@ -6,16 +6,54 @@ import (
"strings"
)
const (
// nftables object names are limited to 255 bytes. Reserve space for the
// prefix and address-family suffix added to the instance name.
nftablesNameMaxLength = 255
egressNftablesTablePrefix = "kube_vip_"
egressNftablesTableSuffix = "_v4"
instanceNameMaxLength = nftablesNameMaxLength - len(egressNftablesTablePrefix) - len(egressNftablesTableSuffix)
)
// Validate runs configuration checks that are independent of host state.
// This should be called after all config sources (flags, file, env vars) are merged.
func (c *Config) Validate() error {
if err := validateHealthCheckAddress(c.ControlPlaneHealthCheck.Address); err != nil {
return err
}
if err := validateInstanceName(c.InstanceName); err != nil {
return err
}
return nil
}
func validateInstanceName(name string) error {
if name == "" {
return nil
}
if len(name) > instanceNameMaxLength {
return fmt.Errorf("instance_name is %d bytes, must not exceed %d bytes so the %q prefix and %q or %q suffix fit within the nftables %d-byte name limit",
len(name), instanceNameMaxLength, egressNftablesTablePrefix, "_v4", "_v6", nftablesNameMaxLength)
}
for position, char := range name {
if isValidNftablesNameCharacter(char) {
continue
}
return fmt.Errorf("instance_name %q contains invalid character %q at byte %d; only ASCII letters, digits, '.', '-' and '_' are allowed",
name, char, position)
}
return nil
}
func isValidNftablesNameCharacter(char rune) bool {
return char >= 'a' && char <= 'z' ||
char >= 'A' && char <= 'Z' ||
char >= '0' && char <= '9' ||
char == '_' || char == '-' || char == '.'
}
func validateHealthCheckAddress(address string) error {
if address == "" {
return nil

View File

@@ -1,6 +1,7 @@
package kubevip
import (
"strings"
"testing"
)
@@ -30,3 +31,40 @@ func TestValidate_HealthCheckAddress(t *testing.T) {
})
}
}
func TestValidate_InstanceName(t *testing.T) {
tests := []struct {
name string
instanceName string
wantErr bool
}{
{name: "empty uses legacy default", instanceName: "", wantErr: false},
{name: "letters digits and separators", instanceName: "release_01.prod-a", wantErr: false},
{name: "exact maximum length", instanceName: strings.Repeat("a", instanceNameMaxLength), wantErr: false},
{name: "exceeds maximum length", instanceName: strings.Repeat("a", instanceNameMaxLength+1), wantErr: true},
{name: "space", instanceName: "release a", wantErr: true},
{name: "slash", instanceName: "namespace/release", wantErr: true},
{name: "dollar sign", instanceName: "release$a", wantErr: true},
{name: "at sign", instanceName: "release@a", wantErr: true},
{name: "newline", instanceName: "release\na", wantErr: true},
{name: "null byte", instanceName: "release\x00a", wantErr: true},
{name: "unicode", instanceName: "rilascio-à", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := &Config{InstanceName: tt.instanceName}
err := config.Validate()
if (err != nil) != tt.wantErr {
t.Fatalf("Validate() error = %v, wantErr %t", err, tt.wantErr)
}
})
}
}
func TestInstanceNameLimitReservesNftablesPrefixAndFamilySuffix(t *testing.T) {
name := strings.Repeat("a", instanceNameMaxLength)
if got := len(egressNftablesTablePrefix + name + egressNftablesTableSuffix); got != nftablesNameMaxLength {
t.Fatalf("family-specific table name length = %d, want %d", got, nftablesNameMaxLength)
}
}

View File

@@ -388,7 +388,8 @@ func (sm *Manager) startMode(ctx context.Context) error {
if sm.config.EnableServices {
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
err := nftables.ClearTables()
tableName := nftables.EgressTableBaseNameForInstance(sm.config.InstanceName)
err := nftables.ClearTablesWithName(tableName)
if err != nil {
log.Warn("[egress]", "mode", "nftables-internal", "clearing error", err)
} else {

View File

@@ -46,6 +46,7 @@ func (sm *Manager) dumpConfigSection() {
fmt.Printf("VIP Subnet: %s\n", sm.config.VIPSubnet)
fmt.Printf("Port: %d\n", sm.config.Port)
fmt.Printf("Namespace: %s\n", sm.config.Namespace)
fmt.Printf("Instance Name: %s\n", sm.config.InstanceName)
fmt.Printf("Service Namespace: %s\n", sm.config.ServiceNamespace)
fmt.Printf("Interface: %s\n", sm.config.Interface)
fmt.Printf("Services Interface: %s\n", sm.config.ServicesInterface)
@@ -218,6 +219,7 @@ func (sm *Manager) dumpRuntimeSection() {
fmt.Printf("Egress Clean Enabled: %t\n", sm.config.EgressClean)
if sm.config.EgressClean {
fmt.Printf("Egress with nftables: %t\n", sm.config.EgressWithNftables)
fmt.Printf("Egress nftables table name: %s\n", nftables.EgressTableBaseNameForInstance(sm.config.InstanceName))
fmt.Printf("Egress Pod CIDR: %s\n", sm.config.EgressPodCidr)
fmt.Printf("Egress Service CIDR: %s\n", sm.config.EgressServiceCidr)
}

View File

@@ -21,6 +21,7 @@ import (
const (
NatTable = "kube_vip_%s"
SNatChain = "kube_vip_snat_%s"
DefaultEgressTableName = "kube_vip"
)
const (
@@ -54,33 +55,33 @@ const (
const connmarkOffset = 0x10000 // 65536 - added to listenPort to get our connmark value
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, allowCIDR []string, IPv6 bool) error {
return ApplySNATWithTable(podIP, vipIP, service, destinationPorts, ignoreCIDR, allowCIDR, IPv6, "")
}
// ApplySNATWithTable applies an egress SNAT rule in the configured table.
func ApplySNATWithTable(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, allowCIDR []string, IPv6 bool, tableName string) error {
conn, err := nftables.New()
if err != nil {
return err
}
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
resolvedTableName := egressTableName(tableName, IPv6)
// Look up the table
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
if t, err := FilterTable(conn, resolvedTableName, IPv6); err != nil {
if t == nil {
// If it doesn't exist then create it
slog.Debug("[egress]", "Creating Table", tableName)
conn.AddTable(GetTable(IPv6))
slog.Debug("[egress]", "Creating Table", resolvedTableName)
conn.AddTable(GetEgressTable(IPv6, tableName))
}
}
slog.Debug("[egress]", "Creating Chain for service", service, utils.IPv6Family, IPv6)
conn.AddChain(GetSNatChain(IPv6, service))
conn.AddChain(GetSNatChainForTable(IPv6, service, tableName))
err = conn.Flush()
if err != nil {
return err
}
portExpressions, err := portSet(conn, IPv6, destinationPorts)
portExpressions, err := portSet(conn, IPv6, destinationPorts, tableName)
if err != nil {
return err
}
@@ -90,7 +91,7 @@ func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []stri
// Create our nftables rule
if len(allowCIDR) == 0 {
// No allowed CIDRs
rule, err = CreateRule(podIP, vipIP, service, ignoreCIDR, "", conn, IPv6, portExpressions[x])
rule, err = CreateRuleForTable(podIP, vipIP, service, ignoreCIDR, "", conn, IPv6, portExpressions[x], tableName)
if err != nil {
return err
}
@@ -99,7 +100,7 @@ func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []stri
} else {
// Create a rule for each allowed CIDR
for y := range allowCIDR {
rule, err = CreateRule(podIP, vipIP, service, ignoreCIDR, allowCIDR[y], conn, IPv6, portExpressions[x])
rule, err = CreateRuleForTable(podIP, vipIP, service, ignoreCIDR, allowCIDR[y], conn, IPv6, portExpressions[x], tableName)
if err != nil {
return err
}
@@ -113,7 +114,7 @@ func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []stri
// Create our nftables rule
if len(allowCIDR) == 0 {
// No allowed CIDRs
rule, err = CreateRule(podIP, vipIP, service, ignoreCIDR, "", conn, IPv6, nil)
rule, err = CreateRuleForTable(podIP, vipIP, service, ignoreCIDR, "", conn, IPv6, nil, tableName)
if err != nil {
return err
}
@@ -122,7 +123,7 @@ func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []stri
} else {
// Create a rule for each allowed CIDR
for y := range allowCIDR {
rule, err = CreateRule(podIP, vipIP, service, ignoreCIDR, allowCIDR[y], conn, IPv6, nil)
rule, err = CreateRuleForTable(podIP, vipIP, service, ignoreCIDR, allowCIDR[y], conn, IPv6, nil, tableName)
if err != nil {
return err
}
@@ -140,10 +141,10 @@ func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []stri
return conn.CloseLasting() // Close out any remaining netlink communication
}
func portSet(conn *nftables.Conn, IPv6 bool, destinationPorts string) (setExpression [][]expr.Any, err error) {
func portSet(conn *nftables.Conn, IPv6 bool, destinationPorts, tableName string) (setExpression [][]expr.Any, err error) {
// If we filter on ports protocols then parse them
if destinationPorts != "" {
table := GetTable(IPv6)
table := GetEgressTable(IPv6, tableName)
fixedPorts := strings.Split(destinationPorts, ",")
// Create an element using our pod IP
@@ -303,6 +304,11 @@ func portSet(conn *nftables.Conn, IPv6 bool, destinationPorts string) (setExpres
}
func DeleteSNAT(IPv6 bool, service string) error {
return DeleteSNATFromTable(IPv6, service, "")
}
// DeleteSNATFromTable deletes an egress SNAT chain from the configured table.
func DeleteSNATFromTable(IPv6 bool, service, tableName string) error {
conn, err := nftables.New()
if err != nil {
return err
@@ -311,7 +317,7 @@ func DeleteSNAT(IPv6 bool, service string) error {
var chainName = fmt.Sprintf(SNatChain, service)
slog.Info("[egress]", "Looking for", chainName)
chain, err := conn.ListChain(GetTable(IPv6), chainName)
chain, err := conn.ListChain(GetEgressTable(IPv6, tableName), chainName)
if err != nil {
return err
}
@@ -325,13 +331,96 @@ func DeleteSNAT(IPv6 bool, service string) error {
return fmt.Errorf("unable to find chain [%s]", chainName)
}
func GetTable(IPv6 bool) *nftables.Table {
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
// DeleteSNATFromTableIfExists deletes a Service's egress SNAT chain when it is
// present. It is used while migrating a Service between instance-owned tables.
func DeleteSNATFromTableIfExists(IPv6 bool, service, tableName string) error {
conn, err := nftables.New()
if err != nil {
return err
}
table := GetEgressTable(IPv6, tableName)
chainName := fmt.Sprintf(SNatChain, service)
chains, err := conn.ListChainsOfTableFamily(table.Family)
if err != nil {
return fmt.Errorf("failed to list nftables chains: %w", err)
}
for _, chain := range chains {
if chain.Table != nil && chain.Table.Name == table.Name && chain.Name == chainName {
slog.Info("[egress] deleting service chain", "table", table.Name, "chain", chainName)
conn.DelChain(chain)
return conn.Flush()
}
}
return nil
}
// DeleteSNATFromOtherTables removes a Service's SNAT chain from every table in
// one address family except the table currently owned by this kube-vip
// instance. Chain names include the Service UID, so cleanup remains
// service-scoped even when the legacy table is shared by multiple instances.
func DeleteSNATFromOtherTables(IPv6 bool, service, keepTableName string) error {
return deleteSNATFromTables(IPv6, service, EgressTableBaseName(keepTableName))
}
// DeleteSNATFromAllTables removes a Service's SNAT chain from all IPv4 and
// IPv6 tables. This does not depend on endpoint data and is safe for teardown
// of IPv4-only, IPv6-only, and dual-stack Services.
func DeleteSNATFromAllTables(service string) error {
return errors.Join(
deleteSNATFromTables(false, service, ""),
deleteSNATFromTables(true, service, ""),
)
}
func deleteSNATFromTables(IPv6 bool, service, keepTableName string) error {
conn, err := nftables.New()
if err != nil {
return err
}
family := nftables.TableFamilyIPv4
if IPv6 {
family = nftables.TableFamilyIPv6
}
keepTable := ""
if keepTableName != "" {
keepTable = GetEgressTable(IPv6, keepTableName).Name
}
chainName := fmt.Sprintf(SNatChain, service)
chains, err := conn.ListChainsOfTableFamily(family)
if err != nil {
return fmt.Errorf("failed to list nftables chains: %w", err)
}
deleted := false
for _, chain := range chains {
if !shouldDeleteSNATChain(chain, chainName, keepTable) {
continue
}
slog.Info("[egress] deleting stale service chain", "table", chain.Table.Name, "chain", chainName)
conn.DelChain(chain)
deleted = true
}
if deleted {
return conn.Flush()
}
return nil
}
func shouldDeleteSNATChain(chain *nftables.Chain, chainName, keepTable string) bool {
return chain.Table != nil && chain.Name == chainName && chain.Table.Name != keepTable
}
func GetTable(IPv6 bool) *nftables.Table {
return GetEgressTable(IPv6, "")
}
// GetEgressTable returns the family-specific egress table.
func GetEgressTable(IPv6 bool, tableName string) *nftables.Table {
tableName = egressTableName(tableName, IPv6)
// Default to IPv4
table := &nftables.Table{
Family: nftables.TableFamilyIPv4,
@@ -345,12 +434,42 @@ func GetTable(IPv6 bool) *nftables.Table {
return table
}
func egressTableName(baseName string, IPv6 bool) string {
family := "v4"
if IPv6 {
family = "v6"
}
return EgressTableBaseName(baseName) + "_" + family
}
// EgressTableBaseName returns the effective table base name. An empty
// configured value retains the legacy kube_vip table names.
func EgressTableBaseName(baseName string) string {
if baseName == "" {
return DefaultEgressTableName
}
return baseName
}
// EgressTableBaseNameForInstance returns the egress table base owned by a
// kube-vip instance. An empty instance name retains the legacy table names.
func EgressTableBaseNameForInstance(instanceName string) string {
if instanceName == "" {
return DefaultEgressTableName
}
return DefaultEgressTableName + "_" + instanceName
}
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
return GetSNatChainForTable(IPv6, service, "")
}
func GetSNatChainForTable(IPv6 bool, service, tableName string) *nftables.Chain {
var chainName = fmt.Sprintf(SNatChain, service)
policy := nftables.ChainPolicyAccept
return &nftables.Chain{
Name: chainName,
Table: GetTable(IPv6),
Table: GetEgressTable(IPv6, tableName),
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityNATSource,
@@ -367,33 +486,42 @@ func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Ta
// ClearTable will remove the original tables and create new empty ones
func ClearTables() error {
return ClearTablesWithName("")
}
// ClearTablesWithName clears only the egress tables owned by this kube-vip instance.
func ClearTablesWithName(tableName string) error {
conn, err := nftables.New()
if err != nil {
return err
}
tableName := fmt.Sprintf(NatTable, "v6")
if t, err := FilterTable(conn, tableName, false); err != nil {
ipv6TableName := egressTableName(tableName, true)
if t, err := FilterTable(conn, ipv6TableName, true); err != nil {
slog.Debug("[egress]", "Cleaning IPv6 finding tables error", err)
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(true))
tableName = fmt.Sprintf(NatTable, "v4")
if t, err := FilterTable(conn, tableName, true); err != nil {
conn.AddTable(GetEgressTable(true, tableName))
ipv4TableName := egressTableName(tableName, false)
if t, err := FilterTable(conn, ipv4TableName, false); err != nil {
slog.Debug("[egress]", "Cleaning IPv4 finding tables error", err)
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(false))
return nil
conn.AddTable(GetEgressTable(false, tableName))
return conn.Flush()
}
// Create our nftables rule
func CreateRule(podIP, vipIP, service string, ignoreCIDR []string, allowCIDR string, conn *nftables.Conn, IPv6 bool, portExpression []expr.Any) (*nftables.Rule, error) {
return CreateRuleForTable(podIP, vipIP, service, ignoreCIDR, allowCIDR, conn, IPv6, portExpression, "")
}
func CreateRuleForTable(podIP, vipIP, service string, ignoreCIDR []string, allowCIDR string, conn *nftables.Conn, IPv6 bool, portExpression []expr.Any, tableName string) (*nftables.Rule, error) {
// Validate pod IP
if net.ParseIP(podIP) == nil {
@@ -406,7 +534,7 @@ func CreateRule(podIP, vipIP, service string, ignoreCIDR []string, allowCIDR str
}
// Get the kube-vip table
table := GetTable(IPv6)
table := GetEgressTable(IPv6, tableName)
// Create our rule
rule := &nftables.Rule{
@@ -414,7 +542,7 @@ func CreateRule(podIP, vipIP, service string, ignoreCIDR []string, allowCIDR str
Exprs: []expr.Any{},
}
// Set the correct chain
rule.Chain = GetSNatChain(IPv6, service)
rule.Chain = GetSNatChainForTable(IPv6, service, tableName)
// Create a set for our original/source address
set := &nftables.Set{

View File

@@ -0,0 +1,109 @@
package nftables
import (
"testing"
googlenftables "github.com/google/nftables"
)
func TestEgressTableName(t *testing.T) {
tests := []struct {
name string
baseName string
ipv6 bool
want string
}{
{name: "default IPv4", want: "kube_vip_v4"},
{name: "default IPv6", ipv6: true, want: "kube_vip_v6"},
{name: "release IPv4", baseName: "kube_vip_release_a", want: "kube_vip_release_a_v4"},
{name: "namespace IPv6", baseName: "kube-vip-networking", ipv6: true, want: "kube-vip-networking_v6"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := egressTableName(tt.baseName, tt.ipv6); got != tt.want {
t.Fatalf("egressTableName(%q, %t) = %q, want %q", tt.baseName, tt.ipv6, got, tt.want)
}
})
}
}
func TestGetEgressTable(t *testing.T) {
table := GetEgressTable(false, "release_a")
if table.Name != "release_a_v4" {
t.Fatalf("table name = %q, want %q", table.Name, "release_a_v4")
}
}
func TestEgressTableBaseName(t *testing.T) {
if got := EgressTableBaseName(""); got != DefaultEgressTableName {
t.Fatalf("EgressTableBaseName(\"\") = %q, want %q", got, DefaultEgressTableName)
}
if got := EgressTableBaseName("release_a"); got != "release_a" {
t.Fatalf("EgressTableBaseName(\"release_a\") = %q, want %q", got, "release_a")
}
}
func TestEgressTableBaseNameForInstance(t *testing.T) {
if got := EgressTableBaseNameForInstance(""); got != DefaultEgressTableName {
t.Fatalf("EgressTableBaseNameForInstance(\"\") = %q, want %q", got, DefaultEgressTableName)
}
if got := EgressTableBaseNameForInstance("release_a"); got != "kube_vip_release_a" {
t.Fatalf("EgressTableBaseNameForInstance(\"release_a\") = %q, want %q", got, "kube_vip_release_a")
}
}
func TestEgressTableNameForInstance(t *testing.T) {
baseName := EgressTableBaseNameForInstance("release_a")
if got := egressTableName(baseName, false); got != "kube_vip_release_a_v4" {
t.Fatalf("IPv4 table name = %q, want %q", got, "kube_vip_release_a_v4")
}
if got := egressTableName(baseName, true); got != "kube_vip_release_a_v6" {
t.Fatalf("IPv6 table name = %q, want %q", got, "kube_vip_release_a_v6")
}
}
func TestShouldDeleteSNATChain(t *testing.T) {
tests := []struct {
name string
chain *googlenftables.Chain
keepTable string
want bool
}{
{
name: "stale table for matching Service UID",
chain: &googlenftables.Chain{
Name: "kube_vip_snat_service-a",
Table: &googlenftables.Table{Name: "release_a_v4"},
},
keepTable: "release_b_v4",
want: true,
},
{
name: "current table for matching Service UID",
chain: &googlenftables.Chain{
Name: "kube_vip_snat_service-a",
Table: &googlenftables.Table{Name: "release_b_v4"},
},
keepTable: "release_b_v4",
want: false,
},
{
name: "different Service UID in stale table",
chain: &googlenftables.Chain{
Name: "kube_vip_snat_service-b",
Table: &googlenftables.Table{Name: "release_a_v4"},
},
keepTable: "release_b_v4",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := shouldDeleteSNATChain(tt.chain, "kube_vip_snat_service-a", tt.keepTable); got != tt.want {
t.Fatalf("shouldDeleteSNATChain() = %t, want %t", got, tt.want)
}
})
}
}

View File

@@ -3,6 +3,7 @@ package services
import (
"bufio"
"context"
"errors"
"fmt"
"net"
"os"
@@ -15,7 +16,9 @@ import (
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
)
// DEBUG
@@ -143,7 +146,10 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace, serviceUUID string, annotations map[string]string, applied *bool) error {
if applied != nil {
*applied = false
}
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
@@ -234,11 +240,18 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
allowCIDRs = append(allowCIDRs, networks...)
}
currentTableName := nftables.EgressTableBaseNameForInstance(p.config.InstanceName)
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, allowCIDRs, utils.IsIPv6(vipIP))
err := nftables.ApplySNATWithTable(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, allowCIDRs, utils.IsIPv6(vipIP), currentTableName)
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
if err := nftables.DeleteSNATFromOtherTables(utils.IsIPv6(vipIP), serviceUUID, currentTableName); err != nil {
return fmt.Errorf("cleaning stale nftables egress chains: %w", err)
}
if applied != nil {
*applied = true
}
return nil
}
@@ -346,9 +359,17 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
return nil
}
func (p *Processor) prepareEgressNftablesTable(serviceUID string, ipv6 bool) error {
tableName := nftables.EgressTableBaseNameForInstance(p.config.InstanceName)
if err := nftables.DeleteSNATFromTableIfExists(ipv6, serviceUID, tableName); err != nil {
return fmt.Errorf("preparing nftables table %q for egress reconciliation: %w", tableName, err)
}
return nil
}
func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
options := metav1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(ctx, options)
@@ -374,3 +395,44 @@ func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR
return
}
func (p *Processor) updateEgressNftablesTableAnnotation(ctx context.Context, service *corev1.Service) error {
tableName := nftables.EgressTableBaseNameForInstance(p.config.InstanceName)
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
currentService, err := p.clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
if err != nil {
return err
}
if currentService.Annotations[kubevip.EgressNftablesTable] == tableName {
return nil
}
serviceCopy := currentService.DeepCopy()
if serviceCopy.Annotations == nil {
serviceCopy.Annotations = make(map[string]string)
}
serviceCopy.Annotations[kubevip.EgressNftablesTable] = tableName
_, err = p.clientSet.CoreV1().Services(service.Namespace).Update(ctx, serviceCopy, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("updating nftables table annotation on Service %s/%s: %w", service.Namespace, service.Name, err)
}
return nil
}
func (p *Processor) cleanupStaleEgressNftablesChains(service *corev1.Service) error {
if service.Annotations[kubevip.Egress] != "true" ||
(service.Annotations[kubevip.EgressInternal] == "" && !p.config.EgressWithNftables) {
return nftables.DeleteSNATFromAllTables(string(service.UID))
}
tableName := nftables.EgressTableBaseNameForInstance(p.config.InstanceName)
if service.Annotations[kubevip.EgressNftablesTable] != tableName {
return nil
}
return errors.Join(
nftables.DeleteSNATFromOtherTables(false, string(service.UID), tableName),
nftables.DeleteSNATFromOtherTables(true, string(service.UID), tableName),
)
}

View File

@@ -116,6 +116,15 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
return nil
}
// The Service annotation is cluster-wide while nftables state is local to
// each node. Reconcile stale per-Service chains on every node after a table
// migration, even when this kube-vip pod is not the Service leader.
if svc.Annotations[kubevip.EgressNftablesTable] != "" {
if err := p.cleanupStaleEgressNftablesChains(svc); err != nil {
log.Warn("failed to clean stale nftables egress chains", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address

View File

@@ -27,6 +27,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
@@ -305,6 +306,9 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
}
var podIP string
errList := []error{}
configuredRules := 0
useInternalNftables := svc.Annotations[kubevip.EgressInternal] != "" || p.config.EgressWithNftables
preparedFamilies := map[bool]bool{}
// Should egress be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
@@ -314,11 +318,21 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
if useInternalNftables && !preparedFamilies[true] {
if err := p.prepareEgressNftablesTable(string(svc.UID), true); err != nil {
errList = append(errList, err)
continue
}
preparedFamilies[true] = true
}
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
applied := false
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations, &applied)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
} else if applied {
configuredRules++
}
}
}
@@ -329,14 +343,31 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
ipv6 := utils.IsIPv6(serviceIP)
if useInternalNftables && !preparedFamilies[ipv6] {
if err := p.prepareEgressNftablesTable(string(svc.UID), ipv6); err != nil {
errList = append(errList, err)
continue
}
preparedFamilies[ipv6] = true
}
applied := false
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations, &applied)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
} else if applied {
configuredRules++
}
}
}
if len(errList) == 0 {
if configuredRules > 0 && useInternalNftables {
if err := p.updateEgressNftablesTableAnnotation(ctx, svc); err != nil {
return err
}
}
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
@@ -431,6 +462,13 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
}
}
internalNftablesEgress := serviceInstance.ServiceSnapshot.Annotations[kubevip.EgressInternal] != "" || p.config.EgressWithNftables
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" && internalNftablesEgress {
if err := nftables.DeleteSNATFromAllTables(string(serviceInstance.ServiceSnapshot.UID)); err != nil {
log.Error("[service] nftables egress teardown", "service", serviceInstance.ServiceSnapshot.Name, "err", err)
}
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
@@ -469,7 +507,7 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" && !internalNftablesEgress {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
@@ -545,6 +583,9 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
// Apply new egress rules with updated endpoint
serviceIPs, _ := instance.FetchServiceAddresses(svc)
errList := []error{}
configuredRules := 0
useInternalNftables := svc.Annotations[kubevip.EgressInternal] != "" || p.config.EgressWithNftables
preparedFamilies := map[bool]bool{}
// Check if egress should be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
@@ -553,10 +594,20 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIP := newIPv6
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if useInternalNftables && !preparedFamilies[true] {
if err := p.prepareEgressNftablesTable(string(svc.UID), true); err != nil {
errList = append(errList, err)
continue
}
preparedFamilies[true] = true
}
applied := false
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations, &applied)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
} else if applied {
configuredRules++
}
}
}
@@ -567,10 +618,21 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIPs = newIPv6
}
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
ipv6 := utils.IsIPv6(serviceIP)
if useInternalNftables && !preparedFamilies[ipv6] {
if err := p.prepareEgressNftablesTable(string(svc.UID), ipv6); err != nil {
errList = append(errList, err)
continue
}
preparedFamilies[ipv6] = true
}
applied := false
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations, &applied)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
} else if applied {
configuredRules++
}
}
}
@@ -578,6 +640,11 @@ func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Servi
if len(errList) > 0 {
return fmt.Errorf("errors configuring egress: %v", errList)
}
if configuredRules > 0 && useInternalNftables {
if err := p.updateEgressNftablesTableAnnotation(ctx, svc); err != nil {
return err
}
}
// Update the service snapshot to reflect the new state
// NOTE: Do NOT call UpdateServiceAnnotation here - the annotation was already updated