package manager import ( "context" "encoding/base64" "fmt" "regexp" "strconv" "strings" log "log/slog" "github.com/kube-vip/kube-vip/pkg/kubevip" "github.com/kube-vip/kube-vip/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" watchtools "k8s.io/client-go/tools/watch" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" ) // This file handles the watching of node annotations for configuration, it will exit once the annotations are // present func annotationsWatcher(ctx context.Context, clientSet, rwClientSet kubernetes.Interface, config *kubevip.Config) error { // Use a restartable watcher, as this should help in the event of etcd or timeout issues log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", config.Annotations) labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": config.NodeName}} listOptions := metav1.ListOptions{ LabelSelector: labels.Set(labelSelector.MatchLabels).String(), } // First we'll check the annotations for the node and if // they aren't what are expected, we'll drop into the watch until they are nodeList, err := clientSet.CoreV1().Nodes().List(ctx, listOptions) if err != nil { return err } // We'll assume there's only one node with the hostname annotation. If that's not true, // there's probably bigger problems node := nodeList.Items[0] bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, &node, config.Annotations) if err == nil { // No error, the annotations already exist config.BGPConfig = bgpConfig config.BGPPeerConfig = bgpPeer return nil } // We got an error with the annotations, falling back to the watch until // they're as needed log.Warn(err.Error()) rw, err := watchtools.NewRetryWatcherWithContext(ctx, node.ResourceVersion, &cache.ListWatch{ WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) { return rwClientSet.CoreV1().Nodes().Watch(ctx, listOptions) }, }) if err != nil { return fmt.Errorf("error creating annotations watcher: %s", err.Error()) } defer func() { rw.Stop() log.Debug("[annotations] watcher stopped") }() ch := rw.ResultChan() for event := range ch { // We need to inspect the event and get ResourceVersion out of it switch event.Type { case watch.Added, watch.Modified: node, ok := event.Object.(*v1.Node) if !ok { return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher") } bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, node, config.Annotations) if err != nil { log.Error(err.Error()) continue } config.BGPConfig = bgpConfig config.BGPPeerConfig = bgpPeer log.Info("[annotations] exiting Annotations watcher - annotations found") return nil case watch.Deleted: node, ok := event.Object.(*v1.Node) if !ok { return fmt.Errorf("unable to parse Kubernetes Node from Kubernetes watcher") } log.Info("Node has been deleted", "name", node.Name) case watch.Bookmark: // Un-used case watch.Error: log.Error("Error attempting to watch Kubernetes Nodes") log.Error("annotations watcher failed", "err", utils.WatchError(event.Object)) default: } } log.Info("[annotations] exiting annotations watcher") if ctx.Err() != nil { return nil } return utils.NewPanicError("annotations watcher channel closed unexpectedly") } // parseNodeAnnotations parses the annotations on the node and updates the configuration // returning an error if the annotations are not valid or missing; and nil if everything is OK // to continue // // Parsed annotation config overlays config in passed bgpConfig in order to preserve configs // set by other means with the exception that bgpConfig.Peers is overwritten. // // The regex expression for each annotation ensures (at least in terms of annotations) backwards // compatibility with the Equinix Metal annotation format changed in // https://github.com/equinix/cloud-provider-equinix-metal/releases/tag/v3.3.0 // // "metal.equinix.com/`" --> "metal.equinix.com/bgp-peers-{{n}}-`" // * `` is the relevant information, such as `node-asn` or `peer-ip` // * `{{n}}` is the number of the peer, always starting with `0` // * kube-vip is only designed to manage one peer, just look for {{n}} == 0 func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix string) (kubevip.BGPConfig, kubevip.BGPPeer, error) { bgpPeer := kubevip.BGPPeer{} prefix = regexp.QuoteMeta(prefix) nodeASN := "" regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?node-asn$", prefix)) for k, v := range node.Annotations { if regex.Match([]byte(k)) { nodeASN = v } } if nodeASN == "" { return bgpConfig, bgpPeer, fmt.Errorf("node-asn value missing or empty") } u64, err := strconv.ParseUint(nodeASN, 10, 32) if err != nil { return bgpConfig, bgpPeer, err } bgpConfig.AS = uint32(u64) srcIP := "" regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?src-ip$", prefix)) for k, v := range node.Annotations { if regex.Match([]byte(k)) { srcIP = v } } if srcIP == "" { return bgpConfig, bgpPeer, fmt.Errorf("src-ip value missing or empty") } // Set the routerID (Unique ID for BGP) to the source IP // Also set the BGP peering to the sourceIP bgpConfig.RouterID, bgpConfig.SourceIP = srcIP, srcIP peerASN := "" regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-asn$", prefix)) for k, v := range node.Annotations { if regex.Match([]byte(k)) { peerASN = v } } if peerASN == "" { return bgpConfig, bgpPeer, fmt.Errorf("peer-asn value missing or empty") } u64, err = strconv.ParseUint(peerASN, 10, 32) if err != nil { return bgpConfig, bgpPeer, err } bgpPeer.AS = uint32(u64) peerIPString := "" regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-[0-9]+-)?peer-ip$", prefix)) for k, v := range node.Annotations { if regex.Match([]byte(k)) { peerIPString += v + "," } } peerIPString = strings.TrimRight(peerIPString, ",") peerIPs := strings.Split(peerIPString, ",") if len(peerIPs) >= 1 && peerIPs[0] == "" || len(peerIPs) == 0 { return bgpConfig, bgpPeer, fmt.Errorf("peer-ip value missing or empty") } bgpConfig.Peers = make([]kubevip.BGPPeer, 0, len(peerIPs)) regexPass := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?bgp-pass$", prefix)) regexMultiHop := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-multi-hop$", prefix)) for _, peerIP := range peerIPs { ipAddr := strings.TrimSpace(peerIP) if ipAddr != "" { bgpPeer.Address = ipAddr // Check if we're also expecting a password for this peer base64BGPPassword := "" for k, v := range node.Annotations { if regexPass.Match([]byte(k)) { base64BGPPassword = v } } if base64BGPPassword != "" { // Decode base64 encoded string decodedPassword, err := base64.StdEncoding.DecodeString(base64BGPPassword) if err != nil { return bgpConfig, bgpPeer, err } // Set the password for each peer bgpPeer.Password = string(decodedPassword) } // Check if multi-hop is enabled. for k, v := range node.Annotations { if regexMultiHop.MatchString(k) { switch v { case "true": bgpPeer.MultiHop = true case "false": bgpPeer.MultiHop = false default: return bgpConfig, bgpPeer, fmt.Errorf("invalid %q annotation value: %q, must be \"true\" or \"false\"", k, v) } } } bgpConfig.Peers = append(bgpConfig.Peers, bgpPeer) } } //log.Debugf("BGPConfig: %v\n", bgpConfig) //log.Debugf("BGPPeerConfig: %v\n", bgpPeer) return bgpConfig, bgpPeer, nil }