diff --git a/cmd/kube-vip.go b/cmd/kube-vip.go index 76071947..3c64d9c0 100644 --- a/cmd/kube-vip.go +++ b/cmd/kube-vip.go @@ -67,6 +67,7 @@ func init() { kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes") kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs") kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs") + kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)") // LoadBalancer flags kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS") diff --git a/pkg/arp/arp.go b/pkg/arp/arp.go index 259079e4..71153405 100644 --- a/pkg/arp/arp.go +++ b/pkg/arp/arp.go @@ -59,6 +59,19 @@ func (m *Manager) Insert(instance *Instance) { } func (m *Manager) Remove(instance *Instance) { + m.RemoveWithIPDelete(instance, true) +} + +// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost +func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) { + // Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP + // If preserve is true, don't delete IP (deleteIP = false) + // If preserve is false, delete IP (deleteIP = true), This is the legacy behavior + deleteIP := !m.config.PreserveVIPOnLeadershipLoss + m.RemoveWithIPDelete(instance, deleteIP) +} + +func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) { i, err := m.get(instance.Name()) if err != nil { log.Error("[ARP manager] unable to remove the instance", "err", err) @@ -71,8 +84,10 @@ func (m *Manager) Remove(instance *Instance) { i.counter-- } else { log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name()) - if _, err := instance.network.DeleteIP(); err != nil { - log.Error("failed to delete IP", "address", instance.network.IP(), "err", err) + if deleteIP { + if _, err := instance.network.DeleteIP(); err != nil { + log.Error("failed to delete IP", "address", instance.network.IP(), "err", err) + } } m.instances.Delete(instance.Name()) } diff --git a/pkg/cluster/clusterLeaderElection.go b/pkg/cluster/clusterLeaderElection.go index 1f4a2e14..41fe678f 100644 --- a/pkg/cluster/clusterLeaderElection.go +++ b/pkg/cluster/clusterLeaderElection.go @@ -165,6 +165,26 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer * leaseID: c.NodeName, sm: sm, onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code + // When we become leader, ensure we can take over VIPs even if they're preserved on other nodes + if c.PreserveVIPOnLeadershipLoss { + log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover") + // Force add the VIPs (this will work even if they exist due to the precheck logic) + for i := range cluster.Network { + added, err := cluster.Network[i].AddIP(true) + if err != nil { + log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err) + } else if added { + log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + } else { + log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + } + } + } + + // Start ARP advertisements now that we have leadership + log.Info("Start ARP/NDP advertisement") + go cluster.arpMgr.StartAdvertisement(ctxArp) + // As we're leading lets start the vip service err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel) if err != nil { @@ -188,13 +208,20 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer * } } - for i := range cluster.Network { - deleted, err := cluster.Network[i].DeleteIP() - if err != nil { - log.Warn("delete VIP", "err", err) - } - if deleted { - log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + // Handle VIP cleanup based on configuration + if c.PreserveVIPOnLeadershipLoss { + log.Info("VIP addresses remain on interface, only stopped ARP/NDP broadcasting") + } else { + // Legacy behavior: delete VIP addresses on leadership loss + log.Info("Deleting VIP addresses on leadership loss (legacy behavior)") + for i := range cluster.Network { + deleted, err := cluster.Network[i].DeleteIP() + if err != nil { + log.Warn("delete VIP", "err", err) + } + if deleted { + log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + } } } @@ -204,6 +231,23 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer * onNewLeader: func(identity string) { // we're notified when new leader elected log.Info("New leader", "leader", identity) + + // If we're not the new leader and we have VIPs preserved from previous leadership, + // we need to clean them up to avoid conflicts. + if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss { + log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity) + for i := range cluster.Network { + deleted, err := cluster.Network[i].DeleteIP() + if err != nil { + log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err) + } + if deleted { + log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface(), "new_leader", identity) + } else { + log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + } + } + } }, } diff --git a/pkg/cluster/service.go b/pkg/cluster/service.go index 546cf727..8cf89cc7 100644 --- a/pkg/cluster/service.go +++ b/pkg/cluster/service.go @@ -346,13 +346,20 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 { continue } - log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP()) - deleted, err := cluster.Network[i].DeleteIP() - if err != nil { - log.Warn(err.Error()) - } - if deleted { - log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + + // Handle VIP cleanup based on configuration + if c.PreserveVIPOnLeadershipLoss { + log.Info("[VIP] Preserving VIP address on interface, only stopped ARP/NDP broadcasting", "ip", cluster.Network[i].IP()) + } else { + // Legacy behavior: delete VIP addresses on leadership loss + log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP()) + deleted, err := cluster.Network[i].DeleteIP() + if err != nil { + log.Warn(err.Error()) + } + if deleted { + log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface()) + } } } @@ -390,5 +397,5 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c <-ctx.Done() // if cancel() execute log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate) - cluster.arpMgr.Remove(arpInstance) + cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance) } diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 9526ab97..6cefe0ae 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -151,21 +151,22 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf // Generate new Virtual IP configuration newVips = append(newVips, &kubevip.Config{ - VIP: address, - Interface: svcInterface, - SingleNode: true, - EnableARP: config.EnableARP, - EnableBGP: config.EnableBGP, - VIPSubnet: subnet, - EnableRoutingTable: config.EnableRoutingTable, - RoutingTableID: config.RoutingTableID, - RoutingTableType: config.RoutingTableType, - RoutingProtocol: config.RoutingProtocol, - ArpBroadcastRate: config.ArpBroadcastRate, - EnableServiceSecurity: config.EnableServiceSecurity, - DNSMode: config.DNSMode, - DisableServiceUpdates: config.DisableServiceUpdates, - EnableServicesElection: config.EnableServicesElection, + VIP: address, + Interface: svcInterface, + SingleNode: true, + EnableARP: config.EnableARP, + EnableBGP: config.EnableBGP, + VIPSubnet: subnet, + EnableRoutingTable: config.EnableRoutingTable, + RoutingTableID: config.RoutingTableID, + RoutingTableType: config.RoutingTableType, + RoutingProtocol: config.RoutingProtocol, + ArpBroadcastRate: config.ArpBroadcastRate, + EnableServiceSecurity: config.EnableServiceSecurity, + DNSMode: config.DNSMode, + DisableServiceUpdates: config.DisableServiceUpdates, + EnableServicesElection: config.EnableServicesElection, + PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss, KubernetesLeaderElection: kubevip.KubernetesLeaderElection{ EnableLeaderElection: config.EnableLeaderElection, }, diff --git a/pkg/kubevip/config_environment.go b/pkg/kubevip/config_environment.go index 38983213..ba0efaa1 100644 --- a/pkg/kubevip/config_environment.go +++ b/pkg/kubevip/config_environment.go @@ -291,6 +291,21 @@ func ParseEnvironment(c *Config) error { c.ArpBroadcastRate = 3000 } + // Determine if VIP should be preserved on leadership loss + // true: VIP addresses remain on interface, only ARP/NDP broadcasting stops + // false (default): VIP addresses are deleted on leadership loss (legacy behavior) + env = os.Getenv(vipPreserveOnLeadershipLoss) + if env != "" { + b, err := strconv.ParseBool(env) + if err != nil { + return err + } + c.PreserveVIPOnLeadershipLoss = b + } else { + // Default to false for backward compatibility + c.PreserveVIPOnLeadershipLoss = false + } + // Wireguard Mode env = os.Getenv(vipWireguard) if env != "" { diff --git a/pkg/kubevip/config_envvar.go b/pkg/kubevip/config_envvar.go index a4de47bb..3426b1ba 100644 --- a/pkg/kubevip/config_envvar.go +++ b/pkg/kubevip/config_envvar.go @@ -9,6 +9,9 @@ const ( // vip_arpRate - defines the rate of gARP broadcasts vipArpRate = "vip_arpRate" + // vipPreserveOnLeadershipLoss - if true, VIP addresses will remain on interface when leadership is lost + vipPreserveOnLeadershipLoss = "vip_preserve_on_leadership_loss" + // vipLeaderElection - defines if the kubernetes algorithm should be used vipLeaderElection = "vip_leaderelection" diff --git a/pkg/kubevip/config_types.go b/pkg/kubevip/config_types.go index 44d5d4af..f39c6e1e 100644 --- a/pkg/kubevip/config_types.go +++ b/pkg/kubevip/config_types.go @@ -50,6 +50,10 @@ type Config struct { // ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network ArpBroadcastRate int64 `yaml:"arpBroadcastRate"` + // PreserveVIPOnLeadershipLoss, if true, VIP addresses will remain on interface when leadership is lost (only ARP/NDP broadcasting stops) + // If false, VIP addresses are deleted on leadership loss (legacy behavior) + PreserveVIPOnLeadershipLoss bool `yaml:"preserveVipOnLeadershipLoss"` + // Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations Annotations string