mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/kube-vip/kube-vip.git
synced 2026-09-20 08:03:47 +08:00
WireGuard for services (#1414)
* add tunnel manager Signed-off-by: Daniel Nägele <daniel@naegele.dev> * add wireguard endpoints and services Signed-off-by: Daniel Nägele <daniel@naegele.dev> * add udp support Signed-off-by: Daniel Nägele <daniel@naegele.dev> * fix endpoint watching Signed-off-by: Daniel Nägele <daniel@naegele.dev> * refactor code Signed-off-by: Daniel Nägele <daniel@naegele.dev> * fix after rebase Signed-off-by: Daniel Nägele <daniel@naegele.dev> --------- Signed-off-by: Daniel Nägele <daniel@naegele.dev> Co-authored-by: Marcel Fest <marcel.fest@live.de>
This commit is contained in:
2
go.mod
2
go.mod
@@ -34,6 +34,7 @@ require (
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
|
||||
google.golang.org/grpc v1.79.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
k8s.io/api v0.35.2
|
||||
k8s.io/apimachinery v0.35.2
|
||||
k8s.io/client-go v0.35.2
|
||||
@@ -150,7 +151,6 @@ require (
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
|
||||
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
|
||||
modernc.org/cc/v4 v4.24.4 // indirect
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
@@ -27,13 +28,13 @@ type Processor struct {
|
||||
}
|
||||
|
||||
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
|
||||
instances *[]*instance.Instance, leaseMgr *lease.Manager) *Processor {
|
||||
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *Processor {
|
||||
return &Processor{
|
||||
config: config,
|
||||
provider: provider,
|
||||
bgpServer: bgpServer,
|
||||
instances: instances,
|
||||
worker: newEndpointWorker(config, provider, bgpServer, instances, leaseMgr),
|
||||
worker: newEndpointWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +79,10 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
|
||||
}
|
||||
|
||||
// There are local endpoints available on the node
|
||||
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
|
||||
// Process immediately if:
|
||||
// - No services/leader election is enabled, OR
|
||||
// - WireGuard is enabled (it always needs immediate DNAT rule updates)
|
||||
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
|
||||
if err := p.worker.processInstance(svcCtx, service); err != nil {
|
||||
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
@@ -24,9 +25,12 @@ type endpointWorker interface {
|
||||
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
|
||||
}
|
||||
|
||||
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance, leaseMgr *lease.Manager) endpointWorker {
|
||||
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) endpointWorker {
|
||||
generic := newGeneric(config, provider, instances, leaseMgr)
|
||||
|
||||
if config.EnableWireguard {
|
||||
return newWireguardWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr)
|
||||
}
|
||||
if config.EnableRoutingTable {
|
||||
return newRoutingTable(generic)
|
||||
}
|
||||
|
||||
292
pkg/endpoints/endpoints_wireguard.go
Normal file
292
pkg/endpoints/endpoints_wireguard.go
Normal file
@@ -0,0 +1,292 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/bgp"
|
||||
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
|
||||
"github.com/kube-vip/kube-vip/pkg/instance"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/nftables"
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// wireguardWorker handles endpoint changes for WireGuard-based services
|
||||
type wireguardWorker struct {
|
||||
config *kubevip.Config
|
||||
provider providers.Provider
|
||||
bgpServer *bgp.Server
|
||||
instances *[]*instance.Instance
|
||||
leaseMgr *lease.Manager
|
||||
tunnelMgr *wireguard.TunnelManager
|
||||
}
|
||||
|
||||
func newWireguardWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
|
||||
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *wireguardWorker {
|
||||
return &wireguardWorker{
|
||||
config: config,
|
||||
provider: provider,
|
||||
bgpServer: bgpServer,
|
||||
instances: instances,
|
||||
leaseMgr: leaseMgr,
|
||||
tunnelMgr: tunnelMgr,
|
||||
}
|
||||
}
|
||||
|
||||
// processInstance updates nftables DNAT rules when endpoints change
|
||||
// This is called by the endpoint watcher when endpoints are added/modified
|
||||
func (w *wireguardWorker) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
|
||||
log.Debug("[wireguard] processing instance for endpoint change", "service", service.Name, "namespace", service.Namespace)
|
||||
|
||||
// Get the target endpoint for this service
|
||||
// For ExternalTrafficPolicy=Local, only use local endpoints
|
||||
// For ExternalTrafficPolicy=Cluster, use all endpoints
|
||||
var endpoints []string
|
||||
var err error
|
||||
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
endpoints, err = w.provider.GetLocalEndpoints(w.config.NodeName, w.config)
|
||||
} else {
|
||||
endpoints, err = w.provider.GetAllEndpoints()
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get endpoints: %w", err)
|
||||
}
|
||||
|
||||
if len(endpoints) == 0 {
|
||||
log.Debug("[wireguard] no endpoints available", "service", service.Name)
|
||||
w.clear(svcCtx, nil, service)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the service processor to call updateServiceWireguardEndpoints
|
||||
// Note: This requires access to the service processor which we don't have here
|
||||
// So we'll recreate the DNAT rules directly
|
||||
|
||||
// First, clear existing rules
|
||||
w.clear(svcCtx, nil, service)
|
||||
|
||||
// Get the first endpoint (simple round-robin could be added later)
|
||||
targetIP := endpoints[0]
|
||||
|
||||
// Get service VIPs
|
||||
serviceIPs, err := utils.FetchServiceIPs(service)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get service IPs: %w", err)
|
||||
}
|
||||
|
||||
// Create service identifier
|
||||
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
|
||||
|
||||
log.Info("[wireguard] updating DNAT rules for endpoint change",
|
||||
"service", service.Name,
|
||||
"namespace", service.Namespace,
|
||||
"targetIP", targetIP,
|
||||
"vips", serviceIPs)
|
||||
|
||||
// Update DNAT rules for each port
|
||||
for _, port := range service.Spec.Ports {
|
||||
// Determine protocol
|
||||
var protocol string
|
||||
switch port.Protocol {
|
||||
case v1.ProtocolTCP:
|
||||
protocol = "TCP"
|
||||
case v1.ProtocolUDP:
|
||||
protocol = "UDP"
|
||||
default:
|
||||
log.Warn("[wireguard] skipping unsupported protocol", "service", service.Name, "port", port.Port, "protocol", port.Protocol)
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine target port (resolve named ports if necessary)
|
||||
targetPort := w.provider.ResolvePort(port)
|
||||
log.Info("[wireguard] resolved port", "service", service.Name, "servicePort", port.Port, "targetPort", targetPort, "targetPortName", port.TargetPort.StrVal)
|
||||
|
||||
for _, vip := range serviceIPs {
|
||||
isIPv6 := isIPv6Address(vip)
|
||||
|
||||
// Strip CIDR notation if present
|
||||
vipAddr := utils.StripCIDR(vip)
|
||||
|
||||
// Get WireGuard interface name from TunnelManager for this VIP
|
||||
if w.tunnelMgr == nil {
|
||||
log.Error("[wireguard] TunnelManager not configured; cannot update DNAT rules",
|
||||
"service", service.Name,
|
||||
"namespace", service.Namespace)
|
||||
return fmt.Errorf("TunnelManager not configured")
|
||||
}
|
||||
tunnelConfig := w.tunnelMgr.GetConfigForVIP(vipAddr)
|
||||
if tunnelConfig == nil {
|
||||
log.Error("[wireguard] WireGuard interface name not configured; cannot update DNAT rules",
|
||||
"service", service.Name,
|
||||
"namespace", service.Namespace,
|
||||
"vip", vipAddr)
|
||||
return fmt.Errorf("wireguard interface name not configured for VIP %s", vipAddr)
|
||||
}
|
||||
wgInterface := tunnelConfig.InterfaceName
|
||||
|
||||
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
|
||||
|
||||
log.Info("[wireguard] applying DNAT rule",
|
||||
"service", service.Name,
|
||||
"vip", vipAddr,
|
||||
"interface", wgInterface,
|
||||
"sourcePort", port.Port,
|
||||
"target", targetIP,
|
||||
"targetPort", targetPort,
|
||||
"chainID", portServiceID)
|
||||
|
||||
// Apply the DNAT rule
|
||||
err := nftables.ApplyDNAT(
|
||||
wgInterface,
|
||||
vipAddr,
|
||||
targetIP,
|
||||
uint16(port.Port), //nolint:gosec // Port range validated by Kubernetes
|
||||
uint16(targetPort), //nolint:gosec // Port range validated by Kubernetes
|
||||
portServiceID,
|
||||
isIPv6,
|
||||
protocol,
|
||||
)
|
||||
if err != nil {
|
||||
log.Error("[wireguard] failed to update DNAT rule",
|
||||
"service", service.Name,
|
||||
"vip", vipAddr,
|
||||
"port", port.Port,
|
||||
"err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Debug("[wireguard] DNAT rule updated successfully",
|
||||
"service", service.Name,
|
||||
"vip", vipAddr,
|
||||
"port", port.Port,
|
||||
"target", fmt.Sprintf("%s:%d", targetIP, targetPort))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// clear removes DNAT rules when no endpoints are available
|
||||
func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
|
||||
log.Info("[wireguard] clearing DNAT rules (no endpoints)", "service", service.Name, "namespace", service.Namespace)
|
||||
|
||||
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
|
||||
|
||||
// Get service IPs to determine IPv4 vs IPv6
|
||||
serviceIPs, _ := utils.FetchServiceIPs(service)
|
||||
|
||||
// Delete DNAT chains for each port
|
||||
for _, port := range service.Spec.Ports {
|
||||
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
|
||||
continue
|
||||
}
|
||||
|
||||
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
|
||||
|
||||
// Determine if we have IPv4 or IPv6
|
||||
hasIPv4, hasIPv6 := false, false
|
||||
for _, vip := range serviceIPs {
|
||||
if isIPv6Address(vip) {
|
||||
hasIPv6 = true
|
||||
} else {
|
||||
hasIPv4 = true
|
||||
}
|
||||
}
|
||||
|
||||
if hasIPv4 {
|
||||
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
|
||||
log.Warn("[wireguard] failed to delete IPv4 DNAT chains",
|
||||
"service", service.Name,
|
||||
"port", port.Port,
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
|
||||
if hasIPv6 {
|
||||
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
|
||||
log.Warn("[wireguard] failed to delete IPv6 DNAT chains",
|
||||
"service", service.Name,
|
||||
"port", port.Port,
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getEndpoints retrieves the list of endpoints for a service
|
||||
// For ExternalTrafficPolicy=Local, only local endpoints are returned
|
||||
// For ExternalTrafficPolicy=Cluster, all endpoints are returned
|
||||
func (w *wireguardWorker) getEndpoints(service *v1.Service, id string) ([]string, error) {
|
||||
var endpoints []string
|
||||
var err error
|
||||
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
|
||||
endpoints, err = w.provider.GetLocalEndpoints(id, w.config)
|
||||
} else {
|
||||
endpoints, err = w.provider.GetAllEndpoints()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("[wireguard] failed to get endpoints: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("[wireguard] retrieved endpoints", "service", service.Name, "count", len(endpoints), "endpoints", endpoints)
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
// removeEgress is a no-op for WireGuard since egress is handled separately
|
||||
func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
|
||||
// WireGuard doesn't use egress in the same way as other modes
|
||||
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
|
||||
}
|
||||
|
||||
// delete removes all DNAT rules for a service
|
||||
func (w *wireguardWorker) delete(ctx context.Context, service *v1.Service, id string) error {
|
||||
log.Info("[wireguard] deleting DNAT rules for service", "service", service.Name, "namespace", service.Namespace)
|
||||
|
||||
w.clear(nil, nil, service)
|
||||
return nil
|
||||
}
|
||||
|
||||
// setInstanceEndpointsStatus updates the endpoint status on the service instance
|
||||
func (w *wireguardWorker) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
|
||||
hasEndpoints := len(endpoints) > 0
|
||||
|
||||
log.Debug("[wireguard] setting instance endpoint status",
|
||||
"service", service.Name,
|
||||
"hasEndpoints", hasEndpoints,
|
||||
"endpointCount", len(endpoints))
|
||||
|
||||
// Find the service instance
|
||||
for _, inst := range *w.instances {
|
||||
if inst.ServiceSnapshot == nil {
|
||||
continue
|
||||
}
|
||||
if inst.ServiceSnapshot.UID == service.UID {
|
||||
// Update the network status for all clusters
|
||||
for _, cluster := range inst.Clusters {
|
||||
for i := range cluster.Network {
|
||||
cluster.Network[i].SetHasEndpoints(hasEndpoints)
|
||||
}
|
||||
}
|
||||
log.Debug("[wireguard] updated instance endpoint status",
|
||||
"service", service.Name,
|
||||
"hasEndpoints", hasEndpoints)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("[wireguard] instance not found for endpoint status update", "service", service.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func isIPv6Address(ip string) bool {
|
||||
// Strip CIDR notation if present before checking
|
||||
addr := utils.StripCIDR(ip)
|
||||
return utils.IsIPv6(addr)
|
||||
}
|
||||
@@ -131,3 +131,16 @@ func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint strin
|
||||
func (ep *Endpoints) GetLabel() string {
|
||||
return ep.label
|
||||
}
|
||||
|
||||
func (ep *Endpoints) ResolvePort(servicePort v1.ServicePort) int32 {
|
||||
return ResolvePortWithLookup(servicePort, func(name string) int32 {
|
||||
for _, subset := range ep.endpoints.Subsets {
|
||||
for _, p := range subset.Ports {
|
||||
if p.Name == name {
|
||||
return p.Port
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ type Endpointslices struct {
|
||||
label string
|
||||
endpointsv4 []discoveryv1.Endpoint
|
||||
endpointsv6 []discoveryv1.Endpoint
|
||||
ports []discoveryv1.EndpointPort
|
||||
}
|
||||
|
||||
func NewEndpointslices() Provider {
|
||||
@@ -64,6 +65,9 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
|
||||
ep.endpointsv4 = eps.Endpoints
|
||||
}
|
||||
|
||||
// Store ports for resolving named ports
|
||||
ep.ports = eps.Ports
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -146,3 +150,14 @@ func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint,
|
||||
func (ep *Endpointslices) GetLabel() string {
|
||||
return ep.label
|
||||
}
|
||||
|
||||
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
|
||||
return ResolvePortWithLookup(servicePort, func(name string) int32 {
|
||||
for _, p := range ep.ports {
|
||||
if p.Name != nil && *p.Name == name && p.Port != nil {
|
||||
return *p.Port
|
||||
}
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
@@ -18,4 +18,22 @@ type Provider interface {
|
||||
GetLabel() string
|
||||
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
|
||||
LoadObject(runtime.Object, context.CancelFunc) error
|
||||
// ResolvePort resolves a service port to the actual target port.
|
||||
// For named ports, it looks up the port number from the endpoint.
|
||||
// For numeric ports, it returns the port as-is.
|
||||
ResolvePort(servicePort v1.ServicePort) int32
|
||||
}
|
||||
|
||||
// ResolvePortWithLookup is a helper that resolves a service port using a lookup function
|
||||
// for named ports. This consolidates the common resolution logic.
|
||||
func ResolvePortWithLookup(servicePort v1.ServicePort, lookupNamedPort func(string) int32) int32 {
|
||||
if servicePort.TargetPort.IntVal != 0 {
|
||||
return servicePort.TargetPort.IntVal
|
||||
}
|
||||
if servicePort.TargetPort.StrVal != "" {
|
||||
if port := lookupNamedPort(servicePort.TargetPort.StrVal); port != 0 {
|
||||
return port
|
||||
}
|
||||
}
|
||||
return servicePort.Port
|
||||
}
|
||||
|
||||
@@ -4,30 +4,30 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/arp"
|
||||
"github.com/kube-vip/kube-vip/pkg/election"
|
||||
"github.com/kube-vip/kube-vip/pkg/iptables"
|
||||
"github.com/kube-vip/kube-vip/pkg/kubevip"
|
||||
"github.com/kube-vip/kube-vip/pkg/lease"
|
||||
"github.com/kube-vip/kube-vip/pkg/networkinterface"
|
||||
"github.com/kube-vip/kube-vip/pkg/nftables"
|
||||
"github.com/kube-vip/kube-vip/pkg/services"
|
||||
"github.com/kube-vip/kube-vip/pkg/sysctl"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
type WireGuard struct {
|
||||
Common
|
||||
wg *wireguard.WireGuard
|
||||
tunnelMgr *wireguard.TunnelManager
|
||||
kubeAPIHost string
|
||||
kubeAPIPort string
|
||||
}
|
||||
@@ -43,39 +43,23 @@ func NewWireGuard(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
|
||||
}
|
||||
|
||||
func (w *WireGuard) Configure(ctx context.Context) error {
|
||||
log.Info("reading wireguard peer configuration from Kubernetes secret")
|
||||
s, err := w.clientSet.CoreV1().Secrets(w.config.Namespace).Get(ctx, "wireguard", metav1.GetOptions{})
|
||||
log.Info("reading wireguard tunnel configurations from Kubernetes secret")
|
||||
tunnelMgr := wireguard.NewTunnelManager()
|
||||
err := tunnelMgr.LoadConfigurationsFromSecret(ctx, w.clientSet, w.config.Namespace, "wireguard")
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to load WireGuard tunnel configurations: %w", err)
|
||||
}
|
||||
// parse all the details needed for Wireguard
|
||||
peerPublicKey := string(s.Data["peerPublicKey"])
|
||||
peerEndpoint := string(s.Data["peerEndpoint"])
|
||||
privateKey := string(s.Data["privateKey"])
|
||||
allowedIPs := string(s.Data["allowedIPs"])
|
||||
listenPort := string(s.Data["listenPort"])
|
||||
if listenPort == "" {
|
||||
listenPort = "51820"
|
||||
|
||||
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/src_valid_mark"); err != nil {
|
||||
return fmt.Errorf("net.ipv4.conf.all.src_valid_mark is disabled and could not be enabled %w", err)
|
||||
}
|
||||
port, err := strconv.Atoi(listenPort)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert listenPort to integer: %w", err)
|
||||
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/route_localnet"); err != nil {
|
||||
return fmt.Errorf("net.ipv4.conf.all.route_localnet is disabled and could not be enabled %w", err)
|
||||
}
|
||||
IPs := make([]string, 0)
|
||||
for ip := range strings.SplitSeq(allowedIPs, ",") {
|
||||
IPs = append(IPs, strings.TrimSpace(ip))
|
||||
}
|
||||
cfg := wireguard.WGConfig{
|
||||
PrivateKey: privateKey,
|
||||
PeerPublicKey: peerPublicKey,
|
||||
PeerEndpoint: peerEndpoint,
|
||||
InterfaceName: "wg0",
|
||||
Address: w.config.VIP,
|
||||
KeepAlive: time.Duration(5) * time.Second,
|
||||
AllowedIPs: IPs,
|
||||
ListenPort: port,
|
||||
}
|
||||
w.wg = wireguard.NewWireGuard(cfg)
|
||||
|
||||
w.tunnelMgr = tunnelMgr
|
||||
configuredVIPs := tunnelMgr.ListConfiguredTunnels()
|
||||
log.Info("loaded WireGuard tunnel configurations", "vips", configuredVIPs)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -91,15 +75,28 @@ func (w *WireGuard) InitControlPlane() error {
|
||||
}
|
||||
|
||||
func (w *WireGuard) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
|
||||
if !w.tunnelMgr.HasConfigForVIP(w.config.VIP) {
|
||||
log.Error("no WireGuard tunnel configuration found for control plane VIP", "vip", w.config.VIP)
|
||||
return
|
||||
}
|
||||
w.runGlobalElection(ctx, w, w.config.LeaseName, w.config, electionManager)
|
||||
}
|
||||
|
||||
func (w *WireGuard) ConfigureServices() {
|
||||
// NOT IMPLEMENTED
|
||||
w.svcProcessor.TunnelMgr = w.tunnelMgr
|
||||
}
|
||||
|
||||
func (w *WireGuard) StartServices(ctx context.Context) error {
|
||||
// NOT IMPLEMENTED
|
||||
if w.config.EgressClean {
|
||||
vip.ClearIPTables(w.config.EgressWithNftables, w.config.ServiceNamespace, iptables.ProtocolIPv4)
|
||||
}
|
||||
if w.config.EnableServicesElection {
|
||||
log.Info("beginning watching services, leaderelection will happen for every service")
|
||||
err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,48 +105,65 @@ func (w *WireGuard) Name() string {
|
||||
}
|
||||
|
||||
func (w *WireGuard) OnStartedLeading(ctx context.Context) {
|
||||
log.Info("started leading", "id", w.config.NodeName)
|
||||
err := w.wg.Up()
|
||||
// Bring up the WireGuard tunnel for control plane VIP
|
||||
err := w.tunnelMgr.BringUpTunnelForVIP(w.config.VIP)
|
||||
if err != nil {
|
||||
log.Error("could not start wireguard", "err", err)
|
||||
_ = w.wg.Down()
|
||||
log.Error("could not start wireguard tunnel for control plane", "vip", w.config.VIP, "err", err)
|
||||
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Get the tunnel to access its configuration
|
||||
wg := w.tunnelMgr.GetTunnelForVIP(w.config.VIP)
|
||||
if wg == nil {
|
||||
log.Error("failed to get wireguard tunnel after bringing up", "vip", w.config.VIP)
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
tunnelConfig := w.tunnelMgr.GetConfigForVIP(w.config.VIP)
|
||||
if tunnelConfig == nil {
|
||||
log.Error("failed to get tunnel configuration", "vip", w.config.VIP)
|
||||
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Strip CIDR notation from VIP if present
|
||||
vipIP := w.config.VIP
|
||||
if strings.Contains(vipIP, "/") {
|
||||
ip, _, err := net.ParseCIDR(vipIP)
|
||||
if err != nil {
|
||||
log.Error("could not parse VIP CIDR", "err", err, "vip", vipIP)
|
||||
_ = w.wg.Down()
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
}
|
||||
vipIP = ip.String()
|
||||
}
|
||||
vipIP := utils.StripCIDR(w.config.VIP)
|
||||
|
||||
// Parse Kubernetes API port
|
||||
kubeAPIPortInt, err := strconv.ParseUint(w.kubeAPIPort, 10, 16)
|
||||
if err != nil {
|
||||
log.Error("could not parse KUBERNETES_SERVICE_PORT_HTTPS", "err", err, "port", w.kubeAPIPort)
|
||||
_ = w.wg.Down()
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
_ = wg.Down()
|
||||
panic("could not parse KUBERNETES_SERVICE_PORT_HTTPS")
|
||||
}
|
||||
|
||||
// Apply nftables DNAT rule to route traffic from wg0:6443 to Kubernetes API service
|
||||
log.Info("applying nftables DNAT rule", "interface", "wg0", "vip", vipIP, "sourcePort", 6443, "kubeAPIHost", w.kubeAPIHost, "kubeAPIPort", w.kubeAPIPort)
|
||||
err = nftables.ApplyAPIServerDNAT("wg0", vipIP, w.kubeAPIHost, 6443, uint16(kubeAPIPortInt), "controlplane", false)
|
||||
// Apply nftables DNAT rule to route traffic from wireguard interface:6443 to Kubernetes API service
|
||||
log.Info("applying nftables DNAT rule",
|
||||
"interface", tunnelConfig.InterfaceName,
|
||||
"vip", vipIP,
|
||||
"sourcePort", 6443,
|
||||
"kubeAPIHost", w.kubeAPIHost,
|
||||
"kubeAPIPort", w.kubeAPIPort)
|
||||
err = nftables.ApplyDNAT(tunnelConfig.InterfaceName, vipIP, w.kubeAPIHost, 6443, uint16(kubeAPIPortInt), "controlplane", false, "TCP")
|
||||
if err != nil {
|
||||
log.Error("could not apply nftables DNAT rule, restarting kube-vip", "err", err)
|
||||
_ = w.wg.Down()
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
log.Error("could not apply nftables DNAT rule", "err", err)
|
||||
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
|
||||
panic("could not apply nftables DNAT rule")
|
||||
}
|
||||
|
||||
if w.config.EnableServices && !w.config.EnableServicesElection {
|
||||
if err := w.svcProcessor.ServicesWatcher(ctx, w.svcProcessor.SyncServices); err != nil {
|
||||
log.Error("failed to start services watcher", "err", err)
|
||||
}
|
||||
}
|
||||
log.Info("nftables DNAT rule applied successfully")
|
||||
@@ -169,12 +183,15 @@ func (w *WireGuard) OnStoppedLeading() {
|
||||
log.Info("nftables DNAT chains deleted successfully")
|
||||
}
|
||||
|
||||
err = w.wg.Down()
|
||||
// Tear down all tunnels (control plane + services)
|
||||
err = w.tunnelMgr.TearDownAllTunnels()
|
||||
if err != nil {
|
||||
log.Error(err.Error(), "id", w.config.NodeName)
|
||||
log.Error("failed to tear down tunnels", "err", err)
|
||||
}
|
||||
|
||||
log.Error("lost leadership, restarting kube-vip")
|
||||
if w.config.EnableServices && !w.config.EnableServicesElection {
|
||||
w.svcProcessor.Stop()
|
||||
}
|
||||
log.Error("lost control plane leadership, restarting kube-vip")
|
||||
if !w.closing.Load() {
|
||||
w.signalChan <- syscall.SIGINT
|
||||
}
|
||||
@@ -186,7 +203,7 @@ func (w *WireGuard) OnNewLeader(identity string) {
|
||||
// I just got the lock
|
||||
return
|
||||
}
|
||||
// safety check
|
||||
_ = w.wg.Down()
|
||||
// safety check - tear down tunnel if we're not the leader
|
||||
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
|
||||
log.Info("new leader elected", "id", identity)
|
||||
}
|
||||
|
||||
@@ -514,7 +514,7 @@ func GetInputChain(IPv6 bool, service string) *nftables.Chain {
|
||||
}
|
||||
}
|
||||
|
||||
func ApplyAPIServerDNAT(
|
||||
func ApplyDNAT(
|
||||
wgIf string,
|
||||
vipIP string,
|
||||
targetIP string,
|
||||
@@ -522,6 +522,7 @@ func ApplyAPIServerDNAT(
|
||||
targetPort uint16,
|
||||
service string,
|
||||
IPv6 bool,
|
||||
protocol string,
|
||||
) error {
|
||||
|
||||
conn, err := nftables.New()
|
||||
@@ -561,6 +562,17 @@ func ApplyAPIServerDNAT(
|
||||
return fmt.Errorf("invalid vip or target ip")
|
||||
}
|
||||
|
||||
// Determine protocol number
|
||||
var protoNum byte
|
||||
switch protocol {
|
||||
case "TCP":
|
||||
protoNum = unix.IPPROTO_TCP
|
||||
case "UDP":
|
||||
protoNum = unix.IPPROTO_UDP
|
||||
default:
|
||||
return fmt.Errorf("unsupported protocol: %s", protocol)
|
||||
}
|
||||
|
||||
/* ---------------- DNAT RULE ---------------- */
|
||||
|
||||
dnatRule := &nftables.Rule{
|
||||
@@ -576,12 +588,12 @@ func ApplyAPIServerDNAT(
|
||||
Data: append([]byte(wgIf), 0),
|
||||
},
|
||||
|
||||
// tcp
|
||||
// protocol (tcp or udp)
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_TCP},
|
||||
Data: []byte{protoNum},
|
||||
},
|
||||
|
||||
// dport == sourcePort (incoming port, e.g., 6443)
|
||||
@@ -635,7 +647,7 @@ func ApplyAPIServerDNAT(
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_TCP},
|
||||
Data: []byte{protoNum},
|
||||
},
|
||||
|
||||
&expr.Payload{
|
||||
@@ -671,12 +683,12 @@ func ApplyAPIServerDNAT(
|
||||
Data: append([]byte(wgIf), 0),
|
||||
},
|
||||
|
||||
// tcp sport == target port
|
||||
// protocol (tcp or udp) sport == target port
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_TCP},
|
||||
Data: []byte{protoNum},
|
||||
},
|
||||
|
||||
&expr.Payload{
|
||||
@@ -734,12 +746,12 @@ func ApplyAPIServerDNAT(
|
||||
Data: ipToBytes(target, IPv6),
|
||||
},
|
||||
|
||||
// tcp dport == target port
|
||||
// protocol (tcp or udp) dport == target port
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{
|
||||
Op: expr.CmpOpEq,
|
||||
Register: 1,
|
||||
Data: []byte{unix.IPPROTO_TCP},
|
||||
Data: []byte{protoNum},
|
||||
},
|
||||
|
||||
&expr.Payload{
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/kube-vip/kube-vip/pkg/servicecontext"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"github.com/kube-vip/kube-vip/pkg/vip"
|
||||
"github.com/kube-vip/kube-vip/pkg/wireguard"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/vishvananda/netlink"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -57,6 +58,9 @@ type Processor struct {
|
||||
nodeLabelManager labelManager
|
||||
|
||||
electionMgr *election.Manager
|
||||
|
||||
// TunnelMgr manages multiple WireGuard tunnels (one per service VIP)
|
||||
TunnelMgr *wireguard.TunnelManager
|
||||
}
|
||||
|
||||
// labelManager is the interface for the node label manager to add/remove labels
|
||||
@@ -94,6 +98,7 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
|
||||
leaseMgr: leaseMgr,
|
||||
nodeLabelManager: nodeLabelManager,
|
||||
electionMgr: electionMgr,
|
||||
TunnelMgr: wireguard.NewTunnelManager(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,7 +213,39 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
|
||||
p.svcMap.Store(svc.UID, svcCtx)
|
||||
}
|
||||
|
||||
if p.config.EnableServicesElection || // Service Election
|
||||
// WireGuard services always need endpoint watching for DNAT rule updates
|
||||
// This is independent of leader election settings (which are for control plane HA)
|
||||
if p.config.EnableWireguard && !svcCtx.IsWatched {
|
||||
// Call serviceFunc first to set up the WireGuard tunnel
|
||||
err = serviceFunc(svcCtx, svc)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
if errors.Is(err, &utils.PanicError{}) {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
if svcCtx != nil {
|
||||
svcCtx.IsWatched = false
|
||||
}
|
||||
}()
|
||||
|
||||
// Start endpoint watcher for WireGuard services (uses EndpointSlices by default)
|
||||
var provider providers.Provider
|
||||
if p.config.EnableEndpoints {
|
||||
provider = providers.NewEndpoints()
|
||||
} else {
|
||||
provider = providers.NewEndpointslices()
|
||||
}
|
||||
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
svcCtx.IsWatched = true
|
||||
} else if p.config.EnableServicesElection || // Service Election
|
||||
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
|
||||
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
|
||||
|
||||
|
||||
@@ -317,6 +317,15 @@ func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Configure WireGuard DNAT rules if WireGuard is enabled
|
||||
if p.config.EnableWireguard {
|
||||
log.Debug("[service] configuring WireGuard DNAT rules", "service", svc.Name, "namespace", svc.Namespace)
|
||||
if err := p.addServiceWireguard(ctx, svc); err != nil {
|
||||
log.Warn("[service] failed to configure WireGuard DNAT", "service", svc.Name, "namespace", svc.Namespace, "err", err)
|
||||
// Don't fail the entire service if WireGuard config fails
|
||||
}
|
||||
}
|
||||
|
||||
finishTime := time.Since(startTime)
|
||||
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
|
||||
|
||||
@@ -422,6 +431,12 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
|
||||
// Update the service array
|
||||
p.ServiceInstances = updatedInstances
|
||||
|
||||
// Clean up WireGuard DNAT rules if WireGuard is enabled
|
||||
if p.config.EnableWireguard {
|
||||
log.Debug("[service] cleaning up WireGuard DNAT rules", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name)
|
||||
p.deleteServiceWireguard(ctx, serviceInstance.ServiceSnapshot)
|
||||
}
|
||||
|
||||
log.Info("Removed instance from manager", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name, "remaining advertised services", len(p.ServiceInstances))
|
||||
|
||||
return nil
|
||||
|
||||
170
pkg/services/services_wireguard.go
Normal file
170
pkg/services/services_wireguard.go
Normal file
@@ -0,0 +1,170 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/nftables"
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// addServiceWireguard configures a WireGuard tunnel for a service
|
||||
// The tunnel is brought up here, but DNAT rules are configured by the endpoint watcher
|
||||
// via wireguardWorker.processInstance() when endpoints become available
|
||||
func (p *Processor) addServiceWireguard(_ context.Context, svc *v1.Service) error {
|
||||
if !p.config.EnableWireguard {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get service VIPs
|
||||
serviceIPs, err := utils.FetchServiceIPs(svc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get service IPs for %s/%s: %w", svc.Namespace, svc.Name, err)
|
||||
}
|
||||
|
||||
if len(serviceIPs) == 0 {
|
||||
return fmt.Errorf("no service IPs found for service %s/%s", svc.Namespace, svc.Name)
|
||||
}
|
||||
|
||||
// For each VIP, bring up the WireGuard tunnel
|
||||
// DNAT rules will be configured by the endpoint watcher when endpoints are available
|
||||
var successCount int
|
||||
var lastErr error
|
||||
for _, vip := range serviceIPs {
|
||||
if err := p.setupServiceWireguardTunnel(svc, vip); err != nil {
|
||||
log.Error("[wireguard] failed to setup tunnel for VIP",
|
||||
"service", svc.Name,
|
||||
"namespace", svc.Namespace,
|
||||
"vip", vip,
|
||||
"err", err)
|
||||
lastErr = err
|
||||
// Continue with other VIPs even if one fails
|
||||
continue
|
||||
}
|
||||
successCount++
|
||||
}
|
||||
|
||||
if successCount == 0 {
|
||||
return fmt.Errorf("failed to setup WireGuard tunnel for any VIP in service %s/%s: %w", svc.Namespace, svc.Name, lastErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setupServiceWireguardTunnel brings up the WireGuard tunnel for a single VIP
|
||||
// DNAT rules are NOT configured here - they are handled by the endpoint watcher
|
||||
func (p *Processor) setupServiceWireguardTunnel(svc *v1.Service, vip string) error {
|
||||
// Check if we have a tunnel configuration for this VIP
|
||||
if !p.TunnelMgr.HasConfigForVIP(vip) {
|
||||
return fmt.Errorf("no WireGuard tunnel configuration found for VIP %s", vip)
|
||||
}
|
||||
|
||||
// Get the tunnel configuration to determine the interface name
|
||||
tunnelConfig := p.TunnelMgr.GetConfigForVIP(vip)
|
||||
if tunnelConfig == nil {
|
||||
return fmt.Errorf("failed to get tunnel configuration for VIP %s", vip)
|
||||
}
|
||||
|
||||
// Bring up the WireGuard tunnel for this VIP
|
||||
if err := p.TunnelMgr.BringUpTunnelForVIP(vip); err != nil {
|
||||
return fmt.Errorf("failed to bring up WireGuard tunnel for VIP %s: %w", vip, err)
|
||||
}
|
||||
|
||||
log.Info("[wireguard] brought up tunnel for service",
|
||||
"namespace", svc.Namespace,
|
||||
"name", svc.Name,
|
||||
"vip", vip,
|
||||
"interface", tunnelConfig.InterfaceName)
|
||||
|
||||
// DNAT rules will be configured by wireguardWorker.processInstance()
|
||||
// when the endpoint watcher detects available endpoints
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteServiceWireguard removes nftables DNAT rules and tears down WireGuard tunnel for a service
|
||||
func (p *Processor) deleteServiceWireguard(_ context.Context, svc *v1.Service) {
|
||||
if !p.config.EnableWireguard {
|
||||
return
|
||||
}
|
||||
|
||||
serviceID := fmt.Sprintf("%s_%s", svc.Namespace, svc.Name)
|
||||
serviceID = utils.SanitizeServiceID(serviceID)
|
||||
|
||||
log.Info("[wireguard] deleting DNAT rules and tunnel for service",
|
||||
"namespace", svc.Namespace,
|
||||
"name", svc.Name,
|
||||
"serviceID", serviceID)
|
||||
|
||||
// Get service IPs
|
||||
serviceIPs, _ := utils.FetchServiceIPs(svc)
|
||||
|
||||
// Delete DNAT chains for each port
|
||||
for _, port := range svc.Spec.Ports {
|
||||
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
|
||||
continue
|
||||
}
|
||||
|
||||
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
|
||||
|
||||
// Try to delete for both IPv4 and IPv6 if we have mixed IPs
|
||||
hasIPv4 := false
|
||||
hasIPv6 := false
|
||||
for _, vip := range serviceIPs {
|
||||
// Strip CIDR notation before checking IP version
|
||||
addr := utils.StripCIDR(vip)
|
||||
if utils.IsIPv6(addr) {
|
||||
hasIPv6 = true
|
||||
} else {
|
||||
hasIPv4 = true
|
||||
}
|
||||
}
|
||||
|
||||
if hasIPv4 {
|
||||
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
|
||||
log.Error("[wireguard] failed to delete IPv4 DNAT chains",
|
||||
"service", svc.Name,
|
||||
"port", port.Port,
|
||||
"err", err)
|
||||
} else {
|
||||
log.Debug("[wireguard] deleted IPv4 DNAT chains",
|
||||
"service", svc.Name,
|
||||
"port", port.Port)
|
||||
}
|
||||
}
|
||||
|
||||
if hasIPv6 {
|
||||
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
|
||||
log.Error("[wireguard] failed to delete IPv6 DNAT chains",
|
||||
"service", svc.Name,
|
||||
"port", port.Port,
|
||||
"err", err)
|
||||
} else {
|
||||
log.Debug("[wireguard] deleted IPv6 DNAT chains",
|
||||
"service", svc.Name,
|
||||
"port", port.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tear down the WireGuard tunnel for each VIP
|
||||
for _, vip := range serviceIPs {
|
||||
if err := p.TunnelMgr.TearDownTunnelForVIP(vip); err != nil {
|
||||
log.Error("[wireguard] failed to tear down tunnel",
|
||||
"service", svc.Name,
|
||||
"vip", vip,
|
||||
"err", err)
|
||||
} else {
|
||||
log.Info("[wireguard] tore down tunnel",
|
||||
"service", svc.Name,
|
||||
"vip", vip)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("[wireguard] DNAT rules deleted and tunnels torn down for service",
|
||||
"namespace", svc.Namespace,
|
||||
"name", svc.Name)
|
||||
}
|
||||
@@ -49,7 +49,7 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
|
||||
|
||||
ch := rw.ResultChan()
|
||||
|
||||
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances, p.leaseMgr)
|
||||
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances, p.leaseMgr, p.TunnelMgr)
|
||||
|
||||
var lastKnownGoodEndpoint string
|
||||
for event := range ch {
|
||||
|
||||
@@ -3,6 +3,9 @@ package utils
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
|
||||
@@ -56,3 +59,74 @@ func IsIPv6CIDR(cidr string) bool {
|
||||
}
|
||||
return ip.To4() == nil
|
||||
}
|
||||
|
||||
// StripCIDR removes the CIDR notation (e.g., "/24") from an IP address string.
|
||||
// If no CIDR notation is present, the original string is returned unchanged.
|
||||
func StripCIDR(ip string) string {
|
||||
if idx := strings.Index(ip, "/"); idx >= 0 {
|
||||
return ip[:idx]
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// SanitizeServiceID sanitizes a service ID to be valid for nftables chain names.
|
||||
// Only alphanumeric characters and underscores are allowed; other characters are replaced with underscores.
|
||||
// The result is truncated to 50 characters to respect nftables name length limits.
|
||||
func SanitizeServiceID(id string) string {
|
||||
var result strings.Builder
|
||||
for _, r := range id {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
result.WriteRune(r)
|
||||
} else {
|
||||
result.WriteRune('_')
|
||||
}
|
||||
}
|
||||
sanitized := result.String()
|
||||
|
||||
// Ensure it doesn't exceed nftables name length limit
|
||||
if len(sanitized) > 50 {
|
||||
sanitized = sanitized[:50]
|
||||
}
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
// FetchServiceIPs extracts IP addresses from a Kubernetes Service.
|
||||
// It checks the following sources in order:
|
||||
// 1. kube-vip.io/loadbalancerIPs annotation (comma-separated list)
|
||||
// 2. spec.LoadBalancerIP (deprecated but still used)
|
||||
// 3. status.loadBalancer.ingress
|
||||
// Returns an error if no IPs are found.
|
||||
func FetchServiceIPs(service *v1.Service) ([]string, error) {
|
||||
var ips []string
|
||||
|
||||
// Check for loadBalancerIPs annotation first (new style)
|
||||
if loadBalancerIPs, ok := service.Annotations["kube-vip.io/loadbalancerIPs"]; ok {
|
||||
for _, ip := range strings.Split(loadBalancerIPs, ",") {
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip != "" {
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to spec.LoadBalancerIP (deprecated but still used)
|
||||
if len(ips) == 0 && service.Spec.LoadBalancerIP != "" {
|
||||
ips = append(ips, service.Spec.LoadBalancerIP)
|
||||
}
|
||||
|
||||
// Check status.loadBalancer.ingress as well
|
||||
if len(ips) == 0 {
|
||||
for _, ingress := range service.Status.LoadBalancer.Ingress {
|
||||
if ingress.IP != "" {
|
||||
ips = append(ips, ingress.IP)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(ips) == 0 {
|
||||
return nil, fmt.Errorf("no IPs found for service")
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
323
pkg/wireguard/tunnel_manager.go
Normal file
323
pkg/wireguard/tunnel_manager.go
Normal file
@@ -0,0 +1,323 @@
|
||||
package wireguard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "log/slog"
|
||||
|
||||
"github.com/kube-vip/kube-vip/pkg/utils"
|
||||
"gopkg.in/yaml.v3"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// TunnelConfig represents a single WireGuard tunnel configuration
|
||||
type TunnelConfig struct {
|
||||
VIP string `yaml:"vip"` // The VIP this tunnel serves (e.g., "10.0.0.100/24")
|
||||
PrivateKey string `yaml:"privateKey"` // WireGuard private key
|
||||
PeerPublicKey string `yaml:"peerPublicKey"` // Peer's public key
|
||||
PeerEndpoint string `yaml:"peerEndpoint"` // Peer endpoint (IP:Port)
|
||||
AllowedIPs []string `yaml:"allowedIPs"` // Allowed IPs through tunnel
|
||||
ListenPort int `yaml:"listenPort"` // Local listen port
|
||||
|
||||
// Internal fields (not from YAML)
|
||||
Name string // Unique name for this tunnel (e.g., "tunnel1")
|
||||
InterfaceName string // Interface name (e.g., "wg0", "wg1")
|
||||
}
|
||||
|
||||
// TunnelManager manages multiple WireGuard tunnels
|
||||
type TunnelManager struct {
|
||||
mu sync.RWMutex
|
||||
tunnels map[string]*WireGuard // key: VIP (without CIDR), value: WireGuard instance
|
||||
configs map[string]*TunnelConfig // key: VIP (without CIDR), value: TunnelConfig
|
||||
refCount map[string]int // key: VIP (without CIDR), value: number of consumers using the tunnel
|
||||
}
|
||||
|
||||
// NewTunnelManager creates a new tunnel manager
|
||||
func NewTunnelManager() *TunnelManager {
|
||||
return &TunnelManager{
|
||||
tunnels: make(map[string]*WireGuard),
|
||||
configs: make(map[string]*TunnelConfig),
|
||||
refCount: make(map[string]int),
|
||||
}
|
||||
}
|
||||
|
||||
// LoadConfigurationsFromSecret loads WireGuard tunnel configurations from a Kubernetes secret
|
||||
// Secret format (YAML string in tunnels key):
|
||||
//
|
||||
// data:
|
||||
// tunnels: |
|
||||
// tunnel1:
|
||||
// vip: 10.0.0.100/24
|
||||
// privateKey: <key>
|
||||
// peerPublicKey: <key>
|
||||
// peerEndpoint: 203.0.113.1:51820
|
||||
// allowedIPs:
|
||||
// - 10.0.0.0/24
|
||||
// listenPort: 51820
|
||||
// tunnel2:
|
||||
// vip: 10.0.0.101/24
|
||||
// privateKey: <key>
|
||||
// peerPublicKey: <key>
|
||||
// peerEndpoint: 203.0.113.2:51821
|
||||
// allowedIPs:
|
||||
// - 10.0.0.0/24
|
||||
// listenPort: 51821
|
||||
func (tm *TunnelManager) LoadConfigurationsFromSecret(ctx context.Context, clientSet *kubernetes.Clientset, namespace, secretName string) error {
|
||||
secret, err := clientSet.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get secret %s/%s: %w", namespace, secretName, err)
|
||||
}
|
||||
|
||||
tunnelsData, ok := secret.Data["tunnels"]
|
||||
if !ok {
|
||||
return fmt.Errorf("secret %s/%s must contain 'tunnels' key", namespace, secretName)
|
||||
}
|
||||
|
||||
return tm.parseTunnelConfig(tunnelsData)
|
||||
}
|
||||
|
||||
// parseTunnelConfig parses the YAML tunnel configuration
|
||||
func (tm *TunnelManager) parseTunnelConfig(data []byte) error {
|
||||
tm.mu.Lock()
|
||||
defer tm.mu.Unlock()
|
||||
|
||||
// Parse YAML into a map of tunnel name -> TunnelConfig
|
||||
var tunnelsMap map[string]*TunnelConfig
|
||||
if err := yaml.Unmarshal(data, &tunnelsMap); err != nil {
|
||||
return fmt.Errorf("failed to parse tunnel configuration YAML: %w", err)
|
||||
}
|
||||
|
||||
if len(tunnelsMap) == 0 {
|
||||
return fmt.Errorf("no tunnel configurations found in secret")
|
||||
}
|
||||
|
||||
// Use the tunnel name from the YAML key as the interface name
|
||||
// This ensures deterministic interface naming across restarts
|
||||
for name, config := range tunnelsMap {
|
||||
config.Name = name
|
||||
config.InterfaceName = name
|
||||
|
||||
if err := tm.addTunnelConfig(config); err != nil {
|
||||
return fmt.Errorf("invalid tunnel config %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("loaded tunnel configurations", "count", len(tm.configs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// addTunnelConfig adds a tunnel configuration to the manager
|
||||
func (tm *TunnelManager) addTunnelConfig(config *TunnelConfig) error {
|
||||
// Validate required fields
|
||||
if config.VIP == "" {
|
||||
return fmt.Errorf("vip is required")
|
||||
}
|
||||
if config.PrivateKey == "" {
|
||||
return fmt.Errorf("privateKey is required")
|
||||
}
|
||||
if config.PeerPublicKey == "" {
|
||||
return fmt.Errorf("peerPublicKey is required")
|
||||
}
|
||||
if config.PeerEndpoint == "" {
|
||||
return fmt.Errorf("peerEndpoint is required")
|
||||
}
|
||||
if config.ListenPort < 1 || config.ListenPort > 65535 {
|
||||
return fmt.Errorf("listenPort must be between 1 and 65535")
|
||||
}
|
||||
|
||||
// Extract VIP without CIDR for indexing
|
||||
vipKey := utils.StripCIDR(config.VIP)
|
||||
|
||||
// Check for duplicate VIP
|
||||
if _, exists := tm.configs[vipKey]; exists {
|
||||
return fmt.Errorf("duplicate VIP configuration: %s", config.VIP)
|
||||
}
|
||||
|
||||
tm.configs[vipKey] = config
|
||||
log.Info("added tunnel configuration",
|
||||
"name", config.Name,
|
||||
"vip", config.VIP,
|
||||
"interface", config.InterfaceName,
|
||||
"listenPort", config.ListenPort)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTunnelForVIP returns the WireGuard tunnel instance for a given VIP
|
||||
// If the tunnel doesn't exist yet, it returns nil
|
||||
func (tm *TunnelManager) GetTunnelForVIP(vip string) *WireGuard {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
return tm.tunnels[vipKey]
|
||||
}
|
||||
|
||||
// GetConfigForVIP returns the tunnel configuration for a given VIP
|
||||
func (tm *TunnelManager) GetConfigForVIP(vip string) *TunnelConfig {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
return tm.configs[vipKey]
|
||||
}
|
||||
|
||||
// BringUpTunnelForVIP creates and brings up a WireGuard tunnel for the given VIP.
|
||||
// If the tunnel is already up, it increments the reference count.
|
||||
// Multiple consumers (control plane, services) can share the same VIP tunnel.
|
||||
func (tm *TunnelManager) BringUpTunnelForVIP(vip string) error {
|
||||
tm.mu.Lock()
|
||||
defer tm.mu.Unlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
|
||||
// Check if already up - increment reference count
|
||||
if _, exists := tm.tunnels[vipKey]; exists {
|
||||
tm.refCount[vipKey]++
|
||||
log.Debug("tunnel already up, incremented reference count", "vip", vip, "refCount", tm.refCount[vipKey])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get configuration
|
||||
config, exists := tm.configs[vipKey]
|
||||
if !exists {
|
||||
return fmt.Errorf("no tunnel configuration found for VIP %s", vip)
|
||||
}
|
||||
|
||||
// Create WireGuard configuration
|
||||
wgCfg := WGConfig{
|
||||
InterfaceName: config.InterfaceName,
|
||||
PrivateKey: config.PrivateKey,
|
||||
PeerPublicKey: config.PeerPublicKey,
|
||||
PeerEndpoint: config.PeerEndpoint,
|
||||
Address: config.VIP,
|
||||
AllowedIPs: config.AllowedIPs,
|
||||
ListenPort: config.ListenPort,
|
||||
KeepAlive: 5 * time.Second,
|
||||
}
|
||||
|
||||
// Create and bring up the tunnel
|
||||
wg := NewWireGuard(wgCfg)
|
||||
if err := wg.Up(); err != nil {
|
||||
return fmt.Errorf("failed to bring up tunnel for VIP %s: %w", vip, err)
|
||||
}
|
||||
|
||||
tm.tunnels[vipKey] = wg
|
||||
tm.refCount[vipKey] = 1
|
||||
log.Info("brought up WireGuard tunnel",
|
||||
"vip", vip,
|
||||
"interface", config.InterfaceName,
|
||||
"listenPort", config.ListenPort)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TearDownTunnelForVIP decrements the reference count for the given VIP tunnel.
|
||||
// The tunnel is only torn down when the reference count reaches zero.
|
||||
// This allows multiple consumers (control plane, services) to share the same VIP tunnel.
|
||||
func (tm *TunnelManager) TearDownTunnelForVIP(vip string) error {
|
||||
tm.mu.Lock()
|
||||
defer tm.mu.Unlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
|
||||
wg, exists := tm.tunnels[vipKey]
|
||||
if !exists {
|
||||
log.Debug("tunnel not found for teardown", "vip", vip)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decrement reference count
|
||||
tm.refCount[vipKey]--
|
||||
if tm.refCount[vipKey] > 0 {
|
||||
log.Debug("tunnel still in use, decremented reference count", "vip", vip, "refCount", tm.refCount[vipKey])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reference count is zero, tear down the tunnel
|
||||
if err := wg.Down(); err != nil {
|
||||
log.Error("failed to tear down tunnel", "vip", vip, "err", err)
|
||||
// Continue to remove from map even if teardown failed
|
||||
}
|
||||
|
||||
delete(tm.tunnels, vipKey)
|
||||
delete(tm.refCount, vipKey)
|
||||
log.Info("tore down WireGuard tunnel", "vip", vip)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TearDownAllTunnels tears down all active tunnels regardless of reference count.
|
||||
// This is typically called during shutdown.
|
||||
func (tm *TunnelManager) TearDownAllTunnels() error {
|
||||
tm.mu.Lock()
|
||||
defer tm.mu.Unlock()
|
||||
|
||||
var errors []error
|
||||
for vip, wg := range tm.tunnels {
|
||||
if err := wg.Down(); err != nil {
|
||||
log.Error("failed to tear down tunnel", "vip", vip, "err", err)
|
||||
errors = append(errors, err)
|
||||
}
|
||||
}
|
||||
|
||||
tm.tunnels = make(map[string]*WireGuard)
|
||||
tm.refCount = make(map[string]int)
|
||||
log.Info("tore down all WireGuard tunnels")
|
||||
|
||||
if len(errors) > 0 {
|
||||
return fmt.Errorf("errors occurred during teardown: %v", errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRefCount returns the current reference count for a VIP tunnel.
|
||||
// Returns 0 if the tunnel doesn't exist.
|
||||
func (tm *TunnelManager) GetRefCount(vip string) int {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
return tm.refCount[vipKey]
|
||||
}
|
||||
|
||||
// ListActiveTunnels returns a list of VIPs with active tunnels
|
||||
func (tm *TunnelManager) ListActiveTunnels() []string {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vips := make([]string, 0, len(tm.tunnels))
|
||||
for vip := range tm.tunnels {
|
||||
vips = append(vips, vip)
|
||||
}
|
||||
|
||||
return vips
|
||||
}
|
||||
|
||||
// ListConfiguredTunnels returns a list of VIPs with configurations
|
||||
func (tm *TunnelManager) ListConfiguredTunnels() []string {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vips := make([]string, 0, len(tm.configs))
|
||||
for vip := range tm.configs {
|
||||
vips = append(vips, vip)
|
||||
}
|
||||
|
||||
return vips
|
||||
}
|
||||
|
||||
// HasConfigForVIP checks if a tunnel configuration exists for the given VIP
|
||||
func (tm *TunnelManager) HasConfigForVIP(vip string) bool {
|
||||
tm.mu.RLock()
|
||||
defer tm.mu.RUnlock()
|
||||
|
||||
vipKey := utils.StripCIDR(vip)
|
||||
_, exists := tm.configs[vipKey]
|
||||
return exists
|
||||
}
|
||||
Reference in New Issue
Block a user