fanhuadesenlinnn/kube-vip
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/kube-vip/kube-vip.git synced 2026-09-20 08:03:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed service DNS resolve

Browse Source 
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
This commit is contained in:
Patryk Strusiewicz-Surmacki
2025-11-04 11:06:00 +01:00
parent 9ad84e3ae6
commit c8e0a72be6
27 changed files with 1302 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/kube-vip.go
View File

@@ -7,6 +7,7 @@ import (
"net/http"
"os"
"slices"
"strconv"
"strings"
"time"
@@ -133,6 +134,7 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
@@ -483,23 +485,23 @@ func GenerateCidrRange(address string, dnsMode string) (string, error) {
ip := net.ParseIP(a)
if ip == nil {
// we probably are a DNS name
ips, err := utils.LookupHost(a, dnsMode)
ips, err := utils.LookupHost(a, dnsMode, true)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
for _, addr := range ips {
ip = net.ParseIP(addr)
if ip.To4() != nil {
cidrs = append(cidrs, "32")
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, "128")
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
}
} else {
if ip.To4() != nil {
cidrs = append(cidrs, "32")
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, "128")
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
}
}

10
pkg/bgp/peers.go
View File

@@ -4,10 +4,12 @@ import (
"context"
"fmt"
"net"
"strconv"
//nolint
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/kube-vip/kube-vip/pkg/utils"
@@ -74,11 +76,11 @@ func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
mask := "128"
mask := strconv.Itoa(vip.DefaultMaskIPv6)
address := ipv4Address
family := api.Family_AFI_IP
if utils.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
mask = strconv.Itoa(vip.DefaultMaskIPv4)
address = ipv6Address
family = api.Family_AFI_IP6
}
@@ -128,7 +130,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 32,
PrefixLen: vip.DefaultMaskIPv4,
})
//nolint
@@ -148,7 +150,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 128,
PrefixLen: vip.DefaultMaskIPv6,
})
v6Family := &api.Family{

6
pkg/cluster/cluster.go
View File

@@ -54,9 +54,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}

4
pkg/cluster/clusterDDNS.go
View File

@@ -13,8 +13,8 @@ import (
// dnsUpdater already have the functionality to keep trying resolve the IP
// and update the VIP configuration if it changes
func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network) error {
ddnsMgr := vip.NewDDNSManager(ctx, network)
ip, err := ddnsMgr.Start()
ddnsMgr := vip.NewDDNSManager(network)
ip, err := ddnsMgr.Start(ctx)
if err != nil {
return err
}

69
pkg/cluster/service.go
View File

@@ -8,8 +8,10 @@ import (
"net"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
log "log/slog"
@@ -276,9 +278,28 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
var arpWG sync.WaitGroup
log.Debug("StartLoadBalancerService")
log.Debug("StartLoadBalancerService", "networks", len(cluster.Network))
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
ddnsReady := make(chan struct{})
go func() {
ctxDDNS, ddnsCancel := context.WithCancel(ctx)
defer ddnsCancel()
// start the DDNS if requested
log.Debug("(svcs) start DDNS", "name", network.DNSName())
if err := cluster.StartDDNS(ctxDDNS, cluster.Network[i]); err != nil {
log.Error("failed to start DDNS", "err", err)
}
close(ddnsReady)
<-cluster.stop
}()
<-ddnsReady
}
log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
@@ -322,6 +343,20 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
}
go func() {
for i := range cluster.Network {
network := cluster.Network[i]
ctxDNS, dnsCancel := context.WithCancel(ctx)
defer dnsCancel()
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
ipUpdater := vip.NewIPUpdater(network)
ipUpdater.Run(ctxDNS)
}
}
<-cluster.stop
// Stop the Arp context if it is running
cancelArp()
@@ -387,16 +422,14 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if utils.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
} else {
ndp, err = vip.NewNDPResponder(network.Interface())
ndp, err = waitNDPResponder(ctx, network.Interface())
if err != nil {
log.Error("failed to create new NDP Responder", "error", err)
} else {
@@ -407,6 +440,7 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
}
}
log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
arpInstance := arp.NewInstance(network, ndp)
@@ -416,3 +450,30 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
}
func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
ndp, err := vip.NewNDPResponder(ifaceName)
if err != nil && strings.Contains(err.Error(), "no such device") {
log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ticker := time.NewTicker(time.Second)
for {
select {
case <-ndpCreateCtx.Done():
return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
case <-ticker.C:
ndp, err = vip.NewNDPResponder(ifaceName)
if err != nil {
log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
} else {
return ndp, nil
}
}
}
} else if err != nil {
return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
}
return ndp, nil
}

279
pkg/instance/instance.go
View File

@@ -1,8 +1,10 @@
package instance
import (
"context"
"fmt"
"net"
"slices"
"strconv"
"strings"
"time"
@@ -30,18 +32,24 @@ type Instance struct {
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCP bool
IsDHCPv4 bool
IsDHCPv6 bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPInterfaceIPv4 string
DHCPInterfaceIPv6 string
DHCPHostname string
DHCPClient *vip.DHCPClient
DHCPv4Client vip.DHCPClient
DHCPv6Client vip.DHCPClient
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
// Kubernetes service mapping
ServiceSnapshot *v1.Service
dnsAddresses []string
}
type Port struct {
@@ -49,12 +57,14 @@ type Port struct {
Type string
}
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses, _ := FetchServiceAddresses(svc)
func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses, instanceHostnames := FetchServiceAddresses(svc)
log.Info("NewInstance used", "instanceAddresses", instanceAddresses, "instanceHostnames", instanceHostnames)
var newVips []*kubevip.Config
var link netlink.Link
var err error
var dnsAddresses []string
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
@@ -130,7 +140,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
subnet = cidrs[0]
} else {
subnet = "32"
subnet = strconv.Itoa(vip.DefaultMaskIPv4)
}
}
} else {
@@ -143,7 +153,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
subnet = cidrs[1]
} else {
subnet = "128"
subnet = strconv.Itoa(vip.DefaultMaskIPv6)
}
}
}
@@ -163,6 +173,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
@@ -172,22 +183,78 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
})
}
for _, hostname := range instanceHostnames {
log.Info("hostname", "addr", hostname)
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
// If it is still blank then use the
if svcInterface == "" {
switch config.ServicesInterface {
case "":
svcInterface = config.Interface
default:
svcInterface = config.ServicesInterface
}
}
if link == nil {
if link, err = netlink.LinkByName(svcInterface); err != nil {
return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
}
if link == nil {
return nil, fmt.Errorf("failed to get interface %s", svcInterface)
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: hostname,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: config.EnableLeaderElection,
},
})
}
// Create new service
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
dnsAddresses: dnsAddresses,
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
// Port: uint16(port.Port), //nolint
// Type: string(port.Protocol),
// })
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
requestedIP := svc.Annotations[kubevip.RequestedIP]
if requestedIP != "" {
requestedIPs := strings.Split(requestedIP, ",")
if len(requestedIPs) > 2 {
return nil, fmt.Errorf("annotation %q cannot request more than one IPv4 and one Ipv6 address", kubevip.RequestedIP)
}
for _, ip := range requestedIPs {
netip := net.ParseIP(ip)
if netip.To4() != nil {
instance.DHCPInterfaceIPv4 = ip
} else {
instance.DHCPInterfaceIPv6 = ip
}
}
}
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
}
@@ -211,29 +278,82 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
// Create Add configuration to the new service
instance.VIPConfigs = newVips
// If this was purposely created with the address 0.0.0.0,
// If this was purposely created with the address '0.0.0.0', or '::'
// we will create a macvlan on the main interface and a DHCP client
// TODO: Consider how best to handle DHCP with multiple addresses
if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
err := instance.startDHCP()
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
}
if len(instanceAddresses) > 2 && (slices.Contains(instanceAddresses, "0.0.0.0") || slices.Contains(instanceAddresses, "::")) {
return nil, fmt.Errorf("DHCP cannot be used if more than 2 addresses (one IPv4 and one IPv6) were specified")
}
for i := range instance.VIPConfigs {
if instance.VIPConfigs[i].VIP == "0.0.0.0" {
err := instance.startDHCP(ctx, i)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv4Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv4Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv4 = ip
}
}
if instance.VIPConfigs[i].VIP == "::" {
err := instance.startDHCP(ctx, i)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv6Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv6Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv6 = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
ddnsAnnotation, exists := svc.Annotations[kubevip.ServiceDDNS]
if exists {
instance.VIPConfigs[i].DDNS, err = strconv.ParseBool(ddnsAnnotation)
if err != nil {
log.Error("Failed to add service", "err", err)
return nil, err
}
}
if len(svc.Spec.IPFamilies) > 0 {
if len(svc.Spec.IPFamilies) > 1 {
instance.VIPConfigs[i].DHCPMode = utils.DualFamily
instance.VIPConfigs[i].DNSMode = utils.DualFamily
switch *svc.Spec.IPFamilyPolicy {
case v1.IPFamilyPolicyRequireDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = true
case v1.IPFamilyPolicyPreferDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = false
default:
instance.VIPConfigs[i].IsDualStack = false
instance.VIPConfigs[i].RequireDualStack = false
}
} else {
if strings.EqualFold(string(svc.Spec.IPFamilies[0]), utils.IPv4Family) {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv4Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv4Family)
} else {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv6Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv6Family)
}
}
}
c, err := cluster.InitCluster(instance.VIPConfigs[i], false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
log.Error("failed to add service", "err", err)
return nil, err
}
@@ -242,7 +362,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
log.Info("(svcs) adding VIP", "ip", instance.VIPConfigs[i].VIP, "interface", instance.VIPConfigs[i].Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -305,11 +425,11 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
return link.Attrs().Name
}
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
func (i *Instance) startDHCP(ctx context.Context, index int) error {
if len(i.VIPConfigs) > 2 {
return fmt.Errorf("DHCP can be used with 2 VIP config maximally, got: %v", len(i.VIPConfigs))
}
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
parent, err := netlink.LinkByName(i.VIPConfigs[index].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
@@ -360,34 +480,61 @@ func (i *Instance) startDHCP() error {
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
var initRebootFlag bool
ip := net.ParseIP(i.VIPConfigs[index].VIP)
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
var client vip.DHCPClient
if ip.To4() != nil {
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
if i.DHCPInterfaceIPv4 != "" {
initRebootFlag = true
}
client = vip.NewDHCPv4Client(iface, initRebootFlag, i.DHCPInterfaceIPv4)
// Add the client so that we can call it to stop function
i.DHCPv4Client = client
// Set that DHCPv4 is enabled
i.IsDHCPv4 = true
} else {
if i.DHCPInterfaceIPv6 != "" {
initRebootFlag = true
}
client, err = vip.NewDHCPv6Client(iface, parent, initRebootFlag, i.DHCPInterfaceIPv6)
if err != nil {
return fmt.Errorf("unable to create client: %w", err)
}
// Add the client so that we can call it to stop function
i.DHCPv6Client = client
// Set that DHCPv6 is enabled
i.IsDHCPv6 = true
}
// Add hostname to dhcp client if annotated
if i.DHCPHostname != "" {
@@ -395,15 +542,15 @@ func (i *Instance) startDHCP() error {
client.WithHostName(i.DHCPHostname)
}
go client.Start()
go func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client error: %w")
}
}()
// Set that DHCP is enabled
i.IsDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
return nil
}

4
pkg/kubevip/annotations.go
View File

@@ -62,5 +62,9 @@ const (
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
// Name of the service lease object
ServiceLease = "kube-vip.io/leaseName"
// Enable DDNS for the service
ServiceDDNS = "kube-vip.io/ddns"
)

18
pkg/kubevip/config_environment.go
View File

@@ -11,6 +11,7 @@ import (
"strings"
"github.com/kube-vip/kube-vip/pkg/detector"
"github.com/kube-vip/kube-vip/pkg/utils"
"sigs.k8s.io/yaml"
)
@@ -379,6 +380,18 @@ func ParseEnvironment(c *Config) error {
c.DNSMode = env
}
// DHCP mode
env = os.Getenv(dhcpMode)
if env != "" {
c.DHCPMode = env
} else {
if c.DNSMode != "first" {
c.DHCPMode = c.DNSMode
} else {
c.DHCPMode = strings.ToLower(utils.IPv4Family)
}
}
// Disable updates for services (status.LoadBalancer.Ingress will not be updated)
env = os.Getenv(disableServiceUpdates)
if env != "" {
@@ -878,6 +891,11 @@ func mergeConfigValues(baseConfig, fileConfig *Config) {
baseConfig.DNSMode = fileConfig.DNSMode
}
// DHCP configuration
if baseConfig.DHCPMode == "" && fileConfig.DHCPMode != "" {
baseConfig.DHCPMode = fileConfig.DHCPMode
}
// Health check configuration
if baseConfig.HealthCheckPort == 0 && fileConfig.HealthCheckPort != 0 {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort

3
pkg/kubevip/config_envvar.go
View File

@@ -212,6 +212,9 @@ const (
// dnsMode defines mode that DNS lookup will be performed with (first, ipv4, ipv6, dual)
dnsMode = "dns_mode"
// dhcpMode defines mode that DHCP lookup will be performed with (ipv4, ipv6, dual)
dhcpMode = "dhcp_mode"
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"

11
pkg/kubevip/config_generator.go
View File

@@ -255,6 +255,17 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
newEnvironment = append(newEnvironment, dnsModeSelector...)
}
if c.DHCPMode != "" {
// build environment variables
dhcpModeSelector := []corev1.EnvVar{
{
Name: dhcpMode,
Value: c.DHCPMode,
},
}
newEnvironment = append(newEnvironment, dhcpModeSelector...)
}
// If we're doing the hybrid mode
if c.EnableControlPlane {
cp := []corev1.EnvVar{

10
pkg/kubevip/config_types.go
View File

@@ -158,6 +158,16 @@ type Config struct {
// DNSMode, this will set the mode DSN lookup will be performed (first, ipv4, ipv6, dual)
DNSMode string `yaml:"dnsDualStackMode"`
// IsDualStack reports if service is DualStack.
IsDualStack bool
// RequireDualStack defines if DualStack is required for the service. Based on service's Spec.ipFamilyPolicy field.
RequireDualStack bool
// DNSMode, this will set the mode DHCP lookup will be performed for DDNS (ipv4, ipv6, dual). By default will be the same as DNSMode.
// If DNSMode is 'first', IPv4 will be used.
DHCPMode string `yaml:"dhcpDualStackMode"`
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`

5
pkg/loadbalancer/ipvs.go
View File

@@ -17,6 +17,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
@@ -90,9 +91,9 @@ func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHeal
}
}
netMask := netmask.MaskFrom(31, 32) // For ipv4
netMask := netmask.MaskFrom(31, vip.DefaultMaskIPv4) // For ipv4
if family == ipvs.INET6 {
netMask = netmask.MaskFrom(128, 128) // For ipv6
netMask = netmask.MaskFrom(128, vip.DefaultMaskIPv6) // For ipv6
}
// Generate out API Server LoadBalancer instance

7
pkg/networkinterface/networkinterface.go
View File

@@ -1,6 +1,7 @@
package networkinterface
import (
log "log/slog"
"sync"
"github.com/vishvananda/netlink"
@@ -23,6 +24,12 @@ func NewManager() *Manager {
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
updated, err := netlink.LinkByName(l.Intf.Attrs().Name)
if err != nil {
log.Error("failed to get interface %q: %w", l.Intf.Attrs().Name, err)
return nil
}
l.Intf = updated
return l
}
result := &Link{

4
pkg/services/egress.go
View File

@@ -269,9 +269,9 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
}
}
mask := "/32"
mask := fmt.Sprintf("/%d", vip.DefaultMaskIPv4)
if !utils.IsIPv4(podIP) {
mask = "/128"
mask = fmt.Sprintf("/%d", vip.DefaultMaskIPv6)
}
if allowedNetworks != "" {

28
pkg/services/leader.go
View File

@@ -54,12 +54,6 @@ func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1
},
}
go func() {
// wait for the service context to end and delete the lease then
<-ctx.Done()
p.leaseMgr.Delete(service)
}()
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
@@ -70,6 +64,13 @@ func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1
svcCtx.IsActive = true
go func() {
// wait for the service context to end and delete the lease then
<-svcCtx.Ctx.Done()
svcCtx.IsActive = false
p.leaseMgr.Delete(service)
}()
svcLease, isNew := p.leaseMgr.Add(service)
// this service is sharing lease
if !isNew {
@@ -77,10 +78,8 @@ func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1
select {
case <-svcLease.Started:
case <-svcLease.Ctx.Done():
svcCtx.IsActive = false
return nil
}
<-svcLease.Started
if lease.UsesCommon(service) {
if err := p.SyncServices(ctx, service); err != nil {
@@ -88,18 +87,19 @@ func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1
svcLease.Cancel()
}
// just block until context is cancelled
<-ctx.Done()
<-svcCtx.Ctx.Done()
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
}
// wait for leaderelection to be finished
<-svcLease.Ctx.Done()
// Mark this service is inactive
svcCtx.IsActive = false
// Mark this service is inactive
svcCtx.IsActive = false
// wait for leaderelection to be finished
<-svcLease.Ctx.Done()
}
return nil
}
// start the leader election code loop

10
pkg/services/processor.go
View File

@@ -116,7 +116,7 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
if len(svcAddresses) <= 0 && len(svcHostnames) <= 0 {
return true, nil
}
@@ -143,7 +143,13 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
// Service hostnames changed
!reflect.DeepEqual(originalServiceHostnames, svcHostnames) ||
// ExternalTrafficPolicy changed
svc.Spec.ExternalTrafficPolicy != i.ServiceSnapshot.Spec.ExternalTrafficPolicy
svc.Spec.ExternalTrafficPolicy != i.ServiceSnapshot.Spec.ExternalTrafficPolicy ||
// IP stack configuration changed
!reflect.DeepEqual(svc.Spec.IPFamilies, i.ServiceSnapshot.Spec.IPFamilies) ||
*svc.Spec.IPFamilyPolicy != *i.ServiceSnapshot.Spec.IPFamilyPolicy ||
// DDNS was disabled/enabled
svc.Annotations[kubevip.ServiceDDNS] != i.ServiceSnapshot.Annotations[kubevip.ServiceDDNS]
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {

118
pkg/services/services.go
View File

@@ -3,6 +3,7 @@ package services
import (
"context"
"fmt"
"net"
"slices"
"strings"
"time"
@@ -86,16 +87,30 @@ func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAct
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCP {
if address != "0.0.0.0" && address != "::" {
if instance.IsDHCPv4 {
if address != "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIP) {
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv4) {
return ActionDelete
}
} else {
if address == "0.0.0.0" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete
}
}
if !instance.IsDHCP {
if address == "0.0.0.0" || address == "::" {
if instance.IsDHCPv6 {
if address != "::" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv6) {
return ActionDelete
}
} else {
if address == "::" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
@@ -140,7 +155,7 @@ func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
startTime := time.Now()
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
newService, err := instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
@@ -152,19 +167,60 @@ func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
p.upnpMap(ctx, newService)
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
if newService.IsDHCPv4 {
go func() {
for ip := range newService.DHCPClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
index := -1
for i := range newService.VIPConfigs {
ip := net.ParseIP(newService.VIPConfigs[i].VIP)
if ip.To4() != nil {
index = i
break
}
}
log.Debug("IP update channel closed, stopping")
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv4")
} else {
for ip := range newService.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[index].VIP = ip
newService.DHCPInterfaceIPv4 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IPv4 update channel closed, stopping")
}
}()
}
if newService.IsDHCPv6 {
go func() {
index := -1
for i := range newService.VIPConfigs {
ip := net.ParseIP(newService.VIPConfigs[i].VIP)
if ip.To4() == nil {
index = i
break
}
}
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv6")
} else {
for ip := range newService.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[index].VIP = ip
newService.DHCPInterfaceIPv6 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IPv6 update channel closed, stopping")
}
}()
}
@@ -288,7 +344,7 @@ func (p *Processor) deleteService(uid types.UID) error {
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
log.Warn("unable to find/stop service", "uid", uid)
log.Error("unable to find/stop service", "uid", uid)
return nil
}
@@ -317,8 +373,15 @@ func (p *Processor) deleteService(uid types.UID) error {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
if serviceInstance.IsDHCPv4 || serviceInstance.IsDHCPv6 {
if serviceInstance.IsDHCPv4 {
serviceInstance.DHCPv4Client.Stop()
}
if serviceInstance.IsDHCPv6 {
serviceInstance.DHCPv6Client.Stop()
}
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
@@ -508,6 +571,7 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
}
func (p *Processor) updateStatus(i *instance.Instance) error {
log.Debug("updating status")
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
@@ -534,9 +598,19 @@ func (p *Processor) updateStatus(i *instance.Instance) error {
// Add the current host
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIPv4 != "" || i.DHCPInterfaceIPv6 != "" {
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
dhcpInterfaceIP := ""
if i.DHCPInterfaceIPv4 != "" {
dhcpInterfaceIP = i.DHCPInterfaceIPv4
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += ","
}
}
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += i.DHCPInterfaceIPv6
}
currentServiceCopy.Annotations[kubevip.RequestedIP] = dhcpInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
@@ -564,7 +638,7 @@ func (p *Processor) updateStatus(i *instance.Instance) error {
for _, c := range i.VIPConfigs {
if !utils.IsIP(c.VIP) {
ips, err := utils.LookupHost(c.VIP, p.config.DNSMode)
ips, err := utils.LookupHost(c.VIP, c.DNSMode, *i.ServiceSnapshot.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack)
if err != nil {
return err
}

15
pkg/utils/dns.go
View File

@@ -2,6 +2,7 @@ package utils
import (
"fmt"
"log/slog"
"net"
"strings"
@@ -15,7 +16,7 @@ const (
)
// LookupHost resolves dnsName and return an IP or an error
func LookupHost(dnsName, dnsMode string) ([]string, error) {
func LookupHost(dnsName, dnsMode string, requireDualStack bool) ([]string, error) {
result, err := net.LookupHost(dnsName)
if err != nil {
return nil, err
@@ -26,7 +27,7 @@ func LookupHost(dnsName, dnsMode string) ([]string, error) {
addrs := []string{}
switch dnsMode {
case strings.ToLower(IPv4Family), strings.ToLower(IPv6Family), DualFamily:
a, err := getIPbyFamily(result, dnsMode)
a, err := getIPbyFamily(result, dnsMode, requireDualStack)
if err != nil {
return nil, err
}
@@ -38,7 +39,7 @@ func LookupHost(dnsName, dnsMode string) ([]string, error) {
return addrs, nil
}
func getIPbyFamily(addresses []string, family string) ([]string, error) {
func getIPbyFamily(addresses []string, family string, requireDualStack bool) ([]string, error) {
var checkers []func(string) bool
families := []string{}
if family == DualFamily || family == strings.ToLower(IPv4Family) {
@@ -54,11 +55,19 @@ func getIPbyFamily(addresses []string, family string) ([]string, error) {
for i, c := range checkers {
addr, err := getIPbyChecker(addresses, c)
if err != nil {
if len(checkers) > 1 && !requireDualStack {
slog.Warn("no address found", "family", families[i])
continue
}
return nil, fmt.Errorf("error getting %s address: %w", families[i], err)
}
addrs = append(addrs, addr)
}
if len(addrs) == 0 {
return nil, fmt.Errorf("no addresses found")
}
return addrs, nil
}

137
pkg/vip/address.go
View File

@@ -17,7 +17,7 @@ import (
"golang.org/x/sys/unix"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/iptables"
iptables "github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
@@ -28,8 +28,9 @@ const (
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
iptablesCommentMarkRule = "kube-vip load balancer IP set mark for masquerade"
defaultMaskIPv6 = 128
defaultMaskIPv4 = 32
DefaultMaskIPv4 = 32
DefaultMaskIPv6 = 128
)
// Network is an interface that enable managing operations for a given IP
@@ -56,6 +57,8 @@ type Network interface {
SetHasEndpoints(value bool)
HasEndpoints() bool
ARPName() string
GetPossibleSubnets() string
DHCPFamily() string
}
// network - This allows network configuration
@@ -82,12 +85,18 @@ type network struct {
ipvsEnabled bool
hasEndpoints bool
possibleSubnets string
// used by DHCP to get address of proper family
dhcpFamily string
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool,
dhcpMode string, requireDualStack, isDualStack bool, tableID int, tableType int, routingProtocol int,
dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled, enableSecurity bool,
intfMgr *networkinterface.Manager) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -106,6 +115,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
ipvsEnabled: ipvsEnabled,
possibleSubnets: subnet,
}
subnet, err = SelectSubnet(address, subnet)
@@ -138,35 +148,50 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
} else {
// try to resolve the address
log.Debug("looking up host", "address", address, "dnsMode", dnsMode)
ips, err := utils.LookupHost(address, dnsMode)
if err != nil {
ips, err := utils.LookupHost(address, dnsMode, requireDualStack)
if (dnsMode == utils.DualFamily && isDDNS && isDualStack) || err != nil {
// return early for ddns if no IP is allocated for the domain
// when leader starts, should do get IP from DHCP for the domain
if isDDNS {
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
address: &netlink.Addr{ // create placeholder for the address
IPNet: &net.IPNet{}, // that will be added later in the process
Peer: &net.IPNet{},
},
log.Info("isDDNS true", "dhcpMode", dhcpMode)
if strings.EqualFold(dhcpMode, utils.IPv4Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
dhcpFamily: utils.IPv4Family,
}
networks = append(networks, result)
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
if strings.EqualFold(dhcpMode, utils.IPv6Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
dhcpFamily: utils.IPv6Family,
}
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
networks = append(networks, result)
}
networks = append(networks, result)
return networks, nil
}
return nil, err
@@ -184,15 +209,12 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
}
// we're able to resolve store this as the initial IP
subnets := Split(subnet)
s := subnet
if len(subnets) > 1 {
s = selectSubnet(ip, subnets)
s, err := SelectSubnet(ip, subnet)
if err != nil {
return nil, fmt.Errorf("failed to select subnet: %w", err)
}
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
@@ -204,6 +226,11 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
result.dhcpFamily = strings.ToLower(utils.IPv6Family)
if net.ParseIP(ip).To4() != nil {
result.dhcpFamily = strings.ToLower(utils.IPv4Family)
}
networks = append(networks, result)
}
@@ -356,7 +383,7 @@ func (configurator *network) AddIP(precheck bool, skipDAD bool) (bool, error) {
}
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not add ip")
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", configurator.link.Intf.Attrs().Name))
}
if err := configurator.configureIPTables(); err != nil {
@@ -675,6 +702,25 @@ func (configurator *network) SetIP(ip string) error {
if strings.Contains("/", ip) {
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
}
if configurator.address == nil {
log.Debug("possible", "subnets", configurator.possibleSubnets)
subnet, err := SelectSubnet(ip, configurator.possibleSubnets)
if err != nil {
return fmt.Errorf("unable to select subnet for IP %q from %q: %w", ip, subnet, err)
}
// Check if the subnet needs overriding
cidr, err := utils.FormatIPWithSubnetMask(ip, subnet)
if err != nil {
return errors.Wrapf(err, "2 could not format address %q with subnetMask %q", ip, subnet)
}
configurator.address, err = netlink.ParseAddr(cidr)
if err != nil {
return errors.Wrapf(err, "could not parse address %q", cidr)
}
}
ones, _ := configurator.address.Mask.Size()
cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
if err != nil {
@@ -804,11 +850,17 @@ func (configurator *network) SetMask(mask string) error {
selectedMask := mask
var err error
if mask == "" {
return fmt.Errorf("no mask provided")
}
if configurator.IP() != "" {
selectedMask, err = SelectSubnet(configurator.IP(), mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
} else if len(strings.Split(mask, ",")) > 1 {
return fmt.Errorf("cannot select mask from %q when IP address is unknown", mask)
}
m, err := strconv.Atoi(selectedMask)
@@ -816,19 +868,18 @@ func (configurator *network) SetMask(mask string) error {
return err
}
size := defaultMaskIPv4
size := DefaultMaskIPv4
family := utils.IPv4Family
if configurator.IP() != "" {
if utils.IsIPv6(configurator.IP()) {
size = defaultMaskIPv6
size = DefaultMaskIPv6
family = utils.IPv6Family
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
}
toSet := net.CIDRMask(m, size)
@@ -857,6 +908,14 @@ func (configurator *network) ARPName() string {
return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
}
func (configurator *network) GetPossibleSubnets() string {
return configurator.possibleSubnets
}
func (configurator *network) DHCPFamily() string {
return configurator.dhcpFamily
}
// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {

27
pkg/vip/ddns.go
View File

@@ -2,7 +2,7 @@ package vip
import (
"context"
"net"
"fmt"
"time"
log "log/slog"
@@ -14,42 +14,41 @@ import (
// for the dDNSHostName
// will return the IP allocated
type DDNSManager interface {
Start() (string, error)
Start(ctx context.Context) (string, error)
}
type ddnsManager struct {
ctx context.Context
network Network
}
// NewDDNSManager returns a newly created Dynamic DNS manager
func NewDDNSManager(ctx context.Context, network Network) DDNSManager {
func NewDDNSManager(network Network) DDNSManager {
return &ddnsManager{
ctx: ctx,
network: network,
}
}
// Start will start the dhcpclient routine to keep the lease
// and return the IP it got from DHCP
func (ddns *ddnsManager) Start() (string, error) {
interfaceName := ddns.network.Interface()
iface, err := net.InterfaceByName(interfaceName)
func (ddns *ddnsManager) Start(ctx context.Context) (string, error) {
client, err := NewDHCPClient(ddns.network)
if err != nil {
return "", err
return "", fmt.Errorf("unable to create DHCP client: %w", err)
}
client := NewDHCPClient(iface, false, "")
client.WithHostName(ddns.network.DDNSHostName())
go client.Start()
go func() {
if err := client.Start(ctx); err != nil {
log.Error("[ddns] DHCP client error: %w")
}
}()
log.Info("waiting for ip from dhcp")
ip, timeout := "", time.After(1*time.Minute)
select {
case <-ddns.ctx.Done():
case <-ctx.Done():
client.Stop()
return "", errors.New("context cancelled")
case <-timeout:
@@ -78,7 +77,7 @@ func (ddns *ddnsManager) Start() (string, error) {
log.Info("got address from dhcp", "ip", ip)
}
}
}(ddns.ctx)
}(ctx)
return ip, nil
}

290
pkg/vip/dhcp.go
View File

@@ -1,289 +1,39 @@
package vip
// DHCP client implementation that refers to https://www.rfc-editor.org/rfc/rfc2131.html
import (
"context"
"fmt"
"net"
"time"
"strings"
log "log/slog"
"github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/jpillora/backoff"
"github.com/kube-vip/kube-vip/pkg/utils"
)
const dhcpClientPort = "68"
const defaultDHCPRenew = time.Hour
const maxBackoffAttempts = 3
// DHCPClient is responsible for maintaining ipv4 lease for one specified interface
type DHCPClient struct {
iface *net.Interface
ddnsHostName string
lease *nclient4.Lease
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
type DHCPClient interface {
ErrorChannel() chan error
IPChannel() chan string
Start(ctx context.Context) error
Stop()
WithHostName(hostname string) DHCPClient
}
// NewDHCPClient returns a new DHCP Client.
func NewDHCPClient(iface *net.Interface, initRebootFlag bool, requestedIP string) *DHCPClient {
return &DHCPClient{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
ipChan: make(chan string),
}
}
func (c *DHCPClient) WithHostName(hostname string) *DHCPClient {
c.ddnsHostName = hostname
return c
}
// Stop state-transition process and close dhcp client
func (c *DHCPClient) Stop() {
close(c.ipChan)
close(c.stopChan)
<-c.releasedChan
}
// Gets the IPChannel for consumption
func (c *DHCPClient) IPChannel() chan string {
return c.ipChan
}
// Gets the ErrorChannel for consumption
func (c *DHCPClient) ErrorChannel() chan error {
return c.errorChan
}
// Start state-transition process of dhcp client
//
// -------- -------
//
// | | +-------------------------->| |<-------------------+
// | INIT- | | +-------------------->| INIT | |
// | REBOOT |DHCPNAK/ +---------->| |<---+ |
// | |Restart| | ------- | |
//
// -------- | DHCPNAK/ | | |
// | Discard offer | -/Send DHCPDISCOVER |
//
// -/Send DHCPREQUEST | | |
//
// | | | DHCPACK v | |
// ----------- | (not accept.)/ ----------- | |
//
// | | | Send DHCPDECLINE | | |
// | REBOOTING | | | | SELECTING |<----+ |
// | | | / | | |DHCPOFFER/ |
//
// ----------- | / ----------- | |Collect |
// | | / | | | replies |
//
// DHCPACK/ | / +----------------+ +-------+ |
// Record lease, set| | v Select offer/ |
// timers T1, T2 ------------ send DHCPREQUEST | |
//
// | +----->| | DHCPNAK, Lease expired/ |
// | | | REQUESTING | Halt network |
// DHCPOFFER/ | | | |
// Discard ------------ | |
// | | | | ----------- |
// | +--------+ DHCPACK/ | | |
// | Record lease, set -----| REBINDING | |
// | timers T1, T2 / | | |
// | | DHCPACK/ ----------- |
// | v Record lease, set ^ |
// +----------------> ------- /timers T1,T2 | |
// +----->| |<---+ | |
// | | BOUND |<---+ | |
// DHCPOFFER, DHCPACK, | | | T2 expires/ DHCPNAK/
// DHCPNAK/Discard ------- | Broadcast Halt network
// | | | | DHCPREQUEST |
// +-------+ | DHCPACK/ | |
// T1 expires/ Record lease, set | |
// Send DHCPREQUEST timers T1, T2 | |
// to leasing server | | |
// | ---------- | |
// | | |------------+ |
// +->| RENEWING | |
// | |----------------------------+
// ----------
// Figure: State-transition diagram for DHCP clients
func (c *DHCPClient) Start() {
lease := c.requestWithBackoff()
c.initRebootFlag = false
c.lease = lease
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
log.Debug("dhcp timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
for {
select {
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
lease, err := c.renew()
if err == nil {
c.lease = lease
log.Info("renew", "lease", lease)
t2.Reset(t2Timeout)
} else {
log.Error("renew failed", "err", err)
}
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(true)
if err == nil {
c.lease = lease
log.Info("rebind", "lease", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("rebind failed", "err", err)
return
}
log.Warn("ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
c.initRebootFlag = false
c.lease = c.requestWithBackoff()
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// release is a unicast request of the IP release.
if err := c.release(); err != nil {
log.Error("release lease failed", "lease", lease, "err", err)
} else {
log.Info("release", "lease", lease)
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return
}
}
}
// --------------------------------------------------------
// | |INIT-REBOOT | RENEWING |REBINDING |
// --------------------------------------------------------
// |broad/unicast |broadcast | unicast |broadcast |
// |server-ip |MUST NOT | MUST NOT |MUST NOT |
// |requested-ip |MUST | MUST NOT |MUST NOT |
// |ciaddr |zero | IP address |IP address|
// --------------------------------------------------------
func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
backoff := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 10 * time.Second,
Max: 1 * time.Minute,
func NewDHCPClient(network Network) (DHCPClient, error) {
interfaceName := network.Interface()
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
return nil, fmt.Errorf("failed to get interface %q: %w", interfaceName, err)
}
var lease *nclient4.Lease
var err error
var client DHCPClient
for {
log.Debug("trying to get a new IP", "attempt", backoff.Attempt())
lease, err = c.request(false)
if strings.EqualFold(network.DHCPFamily(), utils.IPv6Family) {
client, err = NewDHCPv6Client(iface, nil, false, "")
if err != nil {
dur := backoff.Duration()
if backoff.Attempt() > maxBackoffAttempts-1 {
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
log.Error(errMsg.Error())
c.errorChan <- errMsg
c.Stop()
return nil
}
log.Error("request failed", "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
return nil, fmt.Errorf("failed to create DHCP client: %w", err)
}
backoff.Reset()
break
} else {
client = NewDHCPv4Client(iface, false, "")
}
if c.ipChan != nil {
log.Debug("using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
}
return lease
}
func (c *DHCPClient) request(rebind bool) (*nclient4.Lease, error) {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return nil, fmt.Errorf("create a client for iface %s failed, error: %w", c.iface.Name, err)
}
defer dhclient.Close()
modifiers := make([]dhcpv4.Modifier, 0)
if c.ddnsHostName != "" {
modifiers = append(modifiers,
dhcpv4.WithOption(dhcpv4.OptHostName(c.ddnsHostName)),
dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(c.ddnsHostName))),
)
}
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
if c.initRebootFlag {
log.Debug("init-reboot", "ip", c.requestedIP)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.requestedIP)))
}
// if this is a rebind, then the IP we should set is the one that already exists in lease
if rebind {
log.Debug("rebinding", "ip", c.lease.ACK.YourIPAddr)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.lease.ACK.YourIPAddr)))
}
return dhclient.Request(context.TODO(), modifiers...)
}
func (c *DHCPClient) release() error {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return fmt.Errorf("create release client failed, error: %w, iface: %s, server ip: %v", err, c.iface.Name, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
// TODO modify lease
return dhclient.Release(c.lease)
}
func (c *DHCPClient) renew() (*nclient4.Lease, error) {
// renew needs a unicast client. This is due to some servers (like dnsmasq) require the exact request coming from the vip interface
dhclient, err := nclient4.New(c.iface.Name,
nclient4.WithUnicast(&net.UDPAddr{IP: c.lease.ACK.YourIPAddr, Port: nclient4.ClientPort}))
if err != nil {
return nil, fmt.Errorf("create renew client failed, error: %w, server ip: %v", err, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
return dhclient.Renew(context.TODO(), c.lease)
return client, nil
}

292
pkg/vip/dhcpv4.go Normal file
View File

@@ -0,0 +1,292 @@
package vip
// DHCP client implementation that refers to https://www.rfc-editor.org/rfc/rfc2131.html
import (
"context"
"fmt"
"net"
"time"
log "log/slog"
"github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/jpillora/backoff"
)
const dhcpClientPort = "68"
const defaultDHCPRenew = time.Hour
const maxBackoffAttempts = 3
// DHCPv4Client is responsible for maintaining ipv4 lease for one specified interface
type DHCPv4Client struct {
iface *net.Interface
ddnsHostName string
lease *nclient4.Lease
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
}
// NewDHCPv4Client returns a new DHCP Client.
func NewDHCPv4Client(iface *net.Interface, initRebootFlag bool, requestedIP string) *DHCPv4Client {
return &DHCPv4Client{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
ipChan: make(chan string),
}
}
func (c *DHCPv4Client) WithHostName(hostname string) DHCPClient {
c.ddnsHostName = hostname
return c
}
// Stop state-transition process and close dhcp client
func (c *DHCPv4Client) Stop() {
close(c.ipChan)
close(c.stopChan)
<-c.releasedChan
}
// Gets the IPChannel for consumption
func (c *DHCPv4Client) IPChannel() chan string {
return c.ipChan
}
// Gets the ErrorChannel for consumption
func (c *DHCPv4Client) ErrorChannel() chan error {
return c.errorChan
}
// Start state-transition process of dhcp client
//
// -------- -------
//
// | | +-------------------------->| |<-------------------+
// | INIT- | | +-------------------->| INIT | |
// | REBOOT |DHCPNAK/ +---------->| |<---+ |
// | |Restart| | ------- | |
//
// -------- | DHCPNAK/ | | |
// | Discard offer | -/Send DHCPDISCOVER |
//
// -/Send DHCPREQUEST | | |
//
// | | | DHCPACK v | |
// ----------- | (not accept.)/ ----------- | |
//
// | | | Send DHCPDECLINE | | |
// | REBOOTING | | | | SELECTING |<----+ |
// | | | / | | |DHCPOFFER/ |
//
// ----------- | / ----------- | |Collect |
// | | / | | | replies |
//
// DHCPACK/ | / +----------------+ +-------+ |
// Record lease, set| | v Select offer/ |
// timers T1, T2 ------------ send DHCPREQUEST | |
//
// | +----->| | DHCPNAK, Lease expired/ |
// | | | REQUESTING | Halt network |
// DHCPOFFER/ | | | |
// Discard ------------ | |
// | | | | ----------- |
// | +--------+ DHCPACK/ | | |
// | Record lease, set -----| REBINDING | |
// | timers T1, T2 / | | |
// | | DHCPACK/ ----------- |
// | v Record lease, set ^ |
// +----------------> ------- /timers T1,T2 | |
// +----->| |<---+ | |
// | | BOUND |<---+ | |
// DHCPOFFER, DHCPACK, | | | T2 expires/ DHCPNAK/
// DHCPNAK/Discard ------- | Broadcast Halt network
// | | | | DHCPREQUEST |
// +-------+ | DHCPACK/ | |
// T1 expires/ Record lease, set | |
// Send DHCPREQUEST timers T1, T2 | |
// to leasing server | | |
// | ---------- | |
// | | |------------+ |
// +->| RENEWING | |
// | |----------------------------+
// ----------
// Figure: State-transition diagram for DHCP clients
func (c *DHCPv4Client) Start(ctx context.Context) error {
dhcpContext, cancel := context.WithCancel(ctx)
defer cancel()
lease := c.requestWithBackoff(dhcpContext)
c.initRebootFlag = false
c.lease = lease
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
log.Debug("[DHCPv4] timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
for {
select {
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
lease, err := c.renew(dhcpContext)
if err == nil {
c.lease = lease
log.Info("[DHCPv4] renew", "lease", lease)
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv4] renew failed", "err", err)
}
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(dhcpContext, true)
if err == nil {
c.lease = lease
log.Info("[DHCPv4] rebind", "lease", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("[DHCPv4] rebind failed", "err", err)
}
log.Warn("[DHCPv4] ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
c.initRebootFlag = false
c.lease = c.requestWithBackoff(dhcpContext)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// release is a unicast request of the IP release.
var err error
if err = c.release(); err != nil {
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
} else {
log.Info("[DHCPv4] release", "lease", lease)
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
// --------------------------------------------------------
// | |INIT-REBOOT | RENEWING |REBINDING |
// --------------------------------------------------------
// |broad/unicast |broadcast | unicast |broadcast |
// |server-ip |MUST NOT | MUST NOT |MUST NOT |
// |requested-ip |MUST | MUST NOT |MUST NOT |
// |ciaddr |zero | IP address |IP address|
// --------------------------------------------------------
func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) *nclient4.Lease {
backoff := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 10 * time.Second,
Max: 1 * time.Minute,
}
var lease *nclient4.Lease
var err error
for {
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt())
lease, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if backoff.Attempt() > maxBackoffAttempts-1 {
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
log.Error(errMsg.Error())
c.errorChan <- errMsg
c.Stop()
return nil
}
log.Error("[DHCPv4] request failed", "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv4] using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
}
return lease
}
func (c *DHCPv4Client) request(ctx context.Context, rebind bool) (*nclient4.Lease, error) {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return nil, fmt.Errorf("create a client for iface %s failed, error: %w", c.iface.Name, err)
}
defer dhclient.Close()
modifiers := make([]dhcpv4.Modifier, 0)
if c.ddnsHostName != "" {
modifiers = append(modifiers,
dhcpv4.WithOption(dhcpv4.OptHostName(c.ddnsHostName)),
dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(c.ddnsHostName))),
)
}
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
if c.initRebootFlag {
log.Debug("[DHCPv4] init-reboot", "ip", c.requestedIP)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.requestedIP)))
}
// if this is a rebind, then the IP we should set is the one that already exists in lease
if rebind {
log.Debug("[DHCPv4] rebinding", "ip", c.lease.ACK.YourIPAddr)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.lease.ACK.YourIPAddr)))
}
return dhclient.Request(ctx, modifiers...)
}
func (c *DHCPv4Client) release() error {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return fmt.Errorf("create release client failed, error: %w, iface: %s, server ip: %v", err, c.iface.Name, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
// TODO modify lease
return dhclient.Release(c.lease)
}
func (c *DHCPv4Client) renew(ctx context.Context) (*nclient4.Lease, error) {
// renew needs a unicast client. This is due to some servers (like dnsmasq) require the exact request coming from the vip interface
dhclient, err := nclient4.New(c.iface.Name,
nclient4.WithUnicast(&net.UDPAddr{IP: c.lease.ACK.YourIPAddr, Port: nclient4.ClientPort}))
if err != nil {
return nil, fmt.Errorf("create renew client failed, error: %w, server ip: %v", err, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
return dhclient.Renew(ctx, c.lease)
}

387
pkg/vip/dhcpv6.go Normal file
View File

@@ -0,0 +1,387 @@
package vip
import (
"context"
"fmt"
log "log/slog"
"net"
"sync/atomic"
"time"
"github.com/insomniacslk/dhcp/dhcpv6"
"github.com/insomniacslk/dhcp/dhcpv6/nclient6"
"github.com/insomniacslk/dhcp/iana"
"github.com/jpillora/backoff"
"github.com/vishvananda/netlink"
)
var dhcpv6ClientManager *DHCPv6ClientManager
func init() {
dhcpv6ClientManager = NewDHCPv6ClientManager()
}
type DHCPv6ClientManager struct {
clients map[string]*DHCPv6InternalClient
}
func NewDHCPv6ClientManager() *DHCPv6ClientManager {
return &DHCPv6ClientManager{
clients: map[string]*DHCPv6InternalClient{},
}
}
func (m *DHCPv6ClientManager) Get(iface string) *DHCPv6InternalClient {
c, exists := m.clients[iface]
if !exists {
return nil
}
return c
}
func (m *DHCPv6ClientManager) Add(iface string) (*DHCPv6InternalClient, error) {
c := m.Get(iface)
if c != nil {
c.references.Add(1)
return c, nil
}
client, err := NewDHCPv6InternalClient(iface)
if err != nil {
return nil, err
}
m.clients[iface] = client
return client, nil
}
func (m *DHCPv6ClientManager) Delete(iface string) {
c := m.Get(iface)
if c != nil {
c.references.Add(-1)
ref := c.references.Load()
if ref < 1 {
c.client.Close()
delete(m.clients, iface)
}
}
}
type DHCPv6InternalClient struct {
client *nclient6.Client
references *atomic.Int32
}
func NewDHCPv6InternalClient(iface string) (*DHCPv6InternalClient, error) {
client, err := nclient6.New(iface)
if err != nil {
return nil, fmt.Errorf("failed to create DHCPv6 client for interface %q: %w", iface, err)
}
ref := &atomic.Int32{}
ref.Store(1)
return &DHCPv6InternalClient{
client: client,
references: ref,
}, nil
}
type DHCPv6Client struct {
iface *net.Interface
ddnsHostName string
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
ic *DHCPv6InternalClient
addr *dhcpv6.OptIAAddress
}
// NewDHCPv6Client returns a new DHCP6 Client.
func NewDHCPv6Client(iface *net.Interface, parent netlink.Link, initRebootFlag bool, requestedIP string) (*DHCPv6Client, error) {
name := iface.Name
if parent != nil {
name = parent.Attrs().Name
}
client, err := dhcpv6ClientManager.Add(name)
if err != nil {
return nil, fmt.Errorf("failed to create DHCPv6 client: %w", err)
}
return &DHCPv6Client{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
ipChan: make(chan string),
ic: client,
}, nil
}
func (c *DHCPv6Client) WithHostName(hostname string) DHCPClient {
c.ddnsHostName = hostname
return c
}
// Stop state-transition process and close dhcp client
func (c *DHCPv6Client) Stop() {
close(c.ipChan)
close(c.stopChan)
<-c.releasedChan
dhcpv6ClientManager.Delete(c.iface.Name)
}
// Gets the IPChannel for consumption
func (c *DHCPv6Client) IPChannel() chan string {
return c.ipChan
}
// Gets the ErrorChannel for consumption
func (c *DHCPv6Client) ErrorChannel() chan error {
return c.errorChan
}
func (c *DHCPv6Client) Start(ctx context.Context) error {
dhcpContext, cancel := context.WithCancel(ctx)
defer cancel()
// REQUEST WITH BACKOFF ACTION
addr, err := c.requestWithBackoff(dhcpContext)
if err != nil {
return fmt.Errorf("DHCPv6 client failed: %w", err)
}
c.addr = addr
c.initRebootFlag = false
// Set up two ticker to renew/rebind regularly
t1Timeout := c.addr.PreferredLifetime / 2
t2Timeout := (c.addr.ValidLifetime / 8) * 7
log.Debug("[DHCPv6] timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
for {
select {
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
addr, err := c.renew(dhcpContext)
if err == nil {
c.addr = addr
log.Info("[DHCPv6] renew", "addr", addr.IPv6Addr.String())
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv6] renew failed", "err", err)
}
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
addr, err := c.request(dhcpContext, true)
if err == nil {
c.addr = addr
log.Info("[DHCPv6] rebind", "lease", addr)
} else {
log.Warn("[DHCPv6] ip may have changed", "ip", addr.IPv6Addr.String(), "err", err)
c.initRebootFlag = false
c.addr, err = c.requestWithBackoff(dhcpContext)
log.Error("[DHCPv6] rebind failed", "err", err)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
dhcpStopContext, cancel := context.WithCancel(context.Background())
defer cancel()
// IP address release.
var err error
if err = c.release(dhcpStopContext); err != nil {
log.Error("[DHCPv6] release failed", "err", err)
} else {
log.Info("[DHCPv6] released", "address", c.addr.String())
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAddress, error) {
backoff := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 10 * time.Second,
Max: 1 * time.Minute,
}
var err error
var addr *dhcpv6.OptIAAddress
for {
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt())
addr, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if backoff.Attempt() > maxBackoffAttempts-1 {
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
log.Error(errMsg.Error())
c.errorChan <- errMsg
c.Stop()
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
}
log.Error("[DHCPv6] request failed", "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv6] using channel")
c.ipChan <- addr.IPv6Addr.String()
}
return addr, nil
}
func (c *DHCPv6Client) request(ctx context.Context, rebind bool) (*dhcpv6.OptIAAddress, error) {
modifiers := []dhcpv6.Modifier{}
modifiers = append(modifiers, dhcpv6.WithClientID(&dhcpv6.DUIDEN{EnterpriseNumber: 1, EnterpriseIdentifier: []byte(c.ddnsHostName)}))
modifiers = append(modifiers, dhcpv6.WithFQDN(4, c.ddnsHostName))
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
if c.initRebootFlag {
log.Debug("[DHCPv6] init-reboot", "ip", c.requestedIP)
addr := dhcpv6.OptIAAddress{
IPv6Addr: c.requestedIP,
}
modifiers = append(modifiers, dhcpv6.WithIANA(addr))
} else if rebind {
if c.addr == nil {
return nil, fmt.Errorf("unable to rebind - current IP unknown")
}
log.Debug("[DHCPv6] rebinding", "ip", c.addr.IPv6Addr)
modifiers = append(modifiers, dhcpv6.WithIANA(*c.addr))
}
var reply *dhcpv6.Message
if rebind || c.initRebootFlag {
request, err := dhcpv6.NewMessage(modifiers...)
if err != nil {
return nil, fmt.Errorf("failed to create rebind message: %w", err)
}
request.MessageType = dhcpv6.MessageTypeRebind
reply, err = c.ic.client.SendAndRead(ctx, c.ic.client.RemoteAddr(), request, nil)
if err != nil {
return nil, fmt.Errorf("rebind error: %w", err)
}
} else {
adv, err := c.ic.client.Solicit(ctx, modifiers...)
if err != nil {
return nil, fmt.Errorf("solicit error: %w", err)
}
request, err := dhcpv6.NewRequestFromAdvertise(adv, modifiers...)
if err != nil {
return nil, fmt.Errorf("unable to create request message: %w", err)
}
request.MessageType = dhcpv6.MessageTypeAdvertise
reply, err = c.ic.client.Request(ctx, request, modifiers...)
if err != nil {
return nil, fmt.Errorf("request error: %w", err)
}
}
if reply == nil {
return nil, fmt.Errorf("invalid request")
}
return getAddress(reply.Options.IANA())
}
func (c *DHCPv6Client) renew(ctx context.Context) (*dhcpv6.OptIAAddress, error) {
modifiers := []dhcpv6.Modifier{}
modifiers = append(modifiers, dhcpv6.WithClientID(&dhcpv6.DUIDEN{EnterpriseNumber: 1, EnterpriseIdentifier: []byte(c.ddnsHostName)}))
modifiers = append(modifiers, dhcpv6.WithFQDN(4, c.ddnsHostName))
modifiers = append(modifiers, dhcpv6.WithOption(&dhcpv6.OptionGeneric{OptionCode: dhcpv6.OptionUnicast}))
adv, err := c.ic.client.Solicit(ctx, modifiers...)
if err != nil {
return nil, fmt.Errorf("solicit error: %w", err)
}
request, err := dhcpv6.NewRequestFromAdvertise(adv)
if err != nil {
return nil, fmt.Errorf("failed to create request message: %w", err)
}
request.MessageType = dhcpv6.MessageTypeRenew
reply, err := c.ic.client.SendAndRead(ctx, c.ic.client.RemoteAddr(), request, nil)
if err != nil {
return nil, fmt.Errorf("failed to send renew: %w", err)
}
return getAddress(reply.Options.IANA())
}
func (c *DHCPv6Client) release(ctx context.Context) error {
modifiers := []dhcpv6.Modifier{}
modifiers = append(modifiers, dhcpv6.WithClientID(&dhcpv6.DUIDEN{EnterpriseNumber: 1, EnterpriseIdentifier: []byte(c.ddnsHostName)}))
modifiers = append(modifiers, dhcpv6.WithFQDN(4, c.ddnsHostName))
adv, err := c.ic.client.Solicit(ctx, modifiers...)
if err != nil {
return fmt.Errorf("solicit error: %w", err)
}
request, err := dhcpv6.NewRequestFromAdvertise(adv)
if err != nil {
return fmt.Errorf("failed to create release message: %w", err)
}
request.MessageType = dhcpv6.MessageTypeRelease
reply, err := c.ic.client.SendAndRead(ctx, c.ic.client.RemoteAddr(), request, nil)
if err != nil {
return fmt.Errorf("failed to send release: %w", err)
}
if reply.Options.Status().StatusCode != iana.StatusSuccess {
return fmt.Errorf("release failed with code %d: %s", reply.Options.Status().StatusCode, reply.Options.Status().StatusMessage)
}
return nil
}
func getAddress(iana []*dhcpv6.OptIANA) (*dhcpv6.OptIAAddress, error) {
if len(iana) < 1 {
return nil, fmt.Errorf("failed to get IANA")
}
if len(iana) < 1 {
return nil, fmt.Errorf("failed to get addresses data")
}
return iana[0].Options.Addresses()[0], nil
}

6
pkg/vip/dns.go
View File

@@ -39,20 +39,20 @@ func (d *ipUpdater) Run(ctx context.Context) {
mode = "ipv6"
}
ip, err := utils.LookupHost(d.vip.DNSName(), mode)
ip, err := utils.LookupHost(d.vip.DNSName(), mode, true)
if err != nil {
log.Warn("cannot lookup", "name", d.vip.DNSName(), "err", err)
// fallback to renewing the existing IP
ip = []string{d.vip.IP()}
}
log.Debug("setting IP", "address", ip)
log.Debug("(ipUpdater) setting IP", "address", ip)
if err := d.vip.SetIP(ip[0]); err != nil {
log.Error("setting IP", "address", ip, "err", err)
}
// Normal VIP addition for DNS, use skipDAD=false for normal DAD process
if _, err := d.vip.AddIP(false, false); err != nil {
if _, err := d.vip.AddIP(true, false); err != nil {
log.Error("error adding virtual IP", "err", err)
}

4
pkg/vip/ndp.go
View File

@@ -21,13 +21,13 @@ type NdpResponder struct {
func NewNDPResponder(ifaceName string) (*NdpResponder, error) {
iface, err := net.InterfaceByName(ifaceName)
if err != nil {
return nil, fmt.Errorf("failed to get interface %q: %v", ifaceName, err)
return nil, fmt.Errorf("failed to get interface %q: %w", ifaceName, err)
}
// Use link-local address as the source IPv6 address for NDP communications.
conn, _, err := ndp.Listen(iface, ndp.LinkLocal)
if err != nil {
return nil, fmt.Errorf("creating NDP responder for %q: %s", iface.Name, err)
return nil, fmt.Errorf("creating NDP responder for %s: %w", iface.Name, err)
}
ret := &NdpResponder{

21
pkg/vip/util.go
View File

@@ -5,13 +5,11 @@ import (
"crypto/rand"
"fmt"
"net"
"strconv"
"strings"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/pkg/errors"
"github.com/vishvananda/netlink"
)
@@ -144,22 +142,3 @@ func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
}
func selectSubnet(address string, subnets []string) string {
subnet := ""
if utils.IsIPv4(address) {
if subnets[0] != "" {
subnet = subnets[0]
} else {
subnet = strconv.Itoa(defaultMaskIPv4)
}
} else {
if len(subnets) > 1 && subnets[1] != "" {
subnet = subnets[1]
} else {
subnet = strconv.Itoa(defaultMaskIPv6)
}
}
return subnet
}

5
testing/e2e/logs.go
View File

@@ -25,7 +25,7 @@ func GetLogs(ctx context.Context, client kubernetes.Interface, tempDirPath strin
return nil
}
intCtx, cancel := context.WithTimeout(ctx, time.Second*10)
intCtx, cancel := context.WithTimeout(ctx, time.Second*20)
defer cancel()
path := filepath.Join(tempDirPath, "pods.json")
@@ -93,7 +93,8 @@ func GetLogs(ctx context.Context, client kubernetes.Interface, tempDirPath strin
LabelSelector: "app=kube-vip",
}
kvpods, err := client.CoreV1().Pods("").List(intCtx, listOptions)
var kvpods *corev1.PodList
kvpods, err = client.CoreV1().Pods("").List(intCtx, listOptions)
if err != nil {
return fmt.Errorf("failed to list pods: %w", err)
}