kube-vip/pkg/manager/watch_annotations.go

package manager

import (
	"context"
	"encoding/base64"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	log "log/slog"

	"github.com/kube-vip/kube-vip/pkg/kubevip"
	"github.com/kube-vip/kube-vip/pkg/utils"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	watchtools "k8s.io/client-go/tools/watch"

	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
)

// This file handles the watching of node annotations for configuration, it will exit once the annotations are
// present
func annotationsWatcher(ctx context.Context, clientSet,
	rwClientSet kubernetes.Interface, config *kubevip.Config) error {
	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
	log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", config.Annotations)

	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": config.NodeName}}
	listOptions := metav1.ListOptions{
		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
	}

	// First we'll check the annotations for the node and if
	// they aren't what are expected, we'll drop into the watch until they are
	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, listOptions)
	if err != nil {
		return err
	}

	// We'll assume there's only one node with the hostname annotation. If that's not true,
	// there's probably bigger problems
	node := nodeList.Items[0]

	bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, &node, config.Annotations)
	if err == nil {
		// No error, the annotations already exist
		config.BGPConfig = bgpConfig
		config.BGPPeerConfig = bgpPeer
		return nil
	}

	// We got an error with the annotations, falling back to the watch until
	// they're as needed
	log.Warn(err.Error())

	rw, err := watchtools.NewRetryWatcherWithContext(ctx, node.ResourceVersion, &cache.ListWatch{
		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
			return rwClientSet.CoreV1().Nodes().Watch(ctx, listOptions)
		},
	})

	if err != nil {
		return fmt.Errorf("error creating annotations watcher: %s", err.Error())
	}

	defer func() {
		rw.Stop()
		log.Debug("[annotations] watcher stopped")
	}()

	ch := rw.ResultChan()

	for event := range ch {
		// We need to inspect the event and get ResourceVersion out of it
		switch event.Type {
		case watch.Added, watch.Modified:
			node, ok := event.Object.(*v1.Node)
			if !ok {
				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
			}

			bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, node, config.Annotations)
			if err != nil {
				log.Error(err.Error())
				continue
			}

			config.BGPConfig = bgpConfig
			config.BGPPeerConfig = bgpPeer

			log.Info("[annotations] exiting Annotations watcher - annotations found")
			return nil
		case watch.Deleted:
			node, ok := event.Object.(*v1.Node)
			if !ok {
				return fmt.Errorf("unable to parse Kubernetes Node from Kubernetes watcher")
			}

			log.Info("Node has been deleted", "name", node.Name)

		case watch.Bookmark:
			// Un-used
		case watch.Error:
			log.Error("Error attempting to watch Kubernetes Nodes")
			log.Error("annotations watcher failed", "err", utils.WatchError(event.Object))
		default:
		}
	}
	log.Info("[annotations] exiting annotations watcher")
	if ctx.Err() != nil {
		return nil
	}
	return utils.NewPanicError("annotations watcher channel closed unexpectedly")
}

// parseNodeAnnotations parses the annotations on the node and updates the configuration
// returning an error if the annotations are not valid or missing; and nil if everything is OK
// to continue
//
// Parsed annotation config overlays config in passed bgpConfig in order to preserve configs
// set by other means with the exception that bgpConfig.Peers is overwritten.
//
// The regex expression for each annotation ensures (at least in terms of annotations) backwards
// compatibility with the Equinix Metal annotation format changed in
// https://github.com/equinix/cloud-provider-equinix-metal/releases/tag/v3.3.0
//
// "metal.equinix.com/<info>`" --> "metal.equinix.com/bgp-peers-{{n}}-<info>`"
// * `<info>` is the relevant information, such as `node-asn` or `peer-ip`
// * `{{n}}` is the number of the peer, always starting with `0`
// * kube-vip is only designed to manage one peer, just look for {{n}} == 0
func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix string) (kubevip.BGPConfig, kubevip.BGPPeer, error) {
	bgpPeer := kubevip.BGPPeer{}

	prefix = regexp.QuoteMeta(prefix)

	nodeASN := ""
	regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?node-asn$", prefix))
	for k, v := range node.Annotations {
		if regex.Match([]byte(k)) {
			nodeASN = v
		}
	}
	if nodeASN == "" {
		return bgpConfig, bgpPeer, fmt.Errorf("node-asn value missing or empty")
	}

	u64, err := strconv.ParseUint(nodeASN, 10, 32)
	if err != nil {
		return bgpConfig, bgpPeer, err
	}

	bgpConfig.AS = uint32(u64)

	srcIP := ""
	regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?src-ip$", prefix))
	for k, v := range node.Annotations {
		if regex.Match([]byte(k)) {
			srcIP = v
		}
	}
	if srcIP == "" {
		return bgpConfig, bgpPeer, fmt.Errorf("src-ip value missing or empty")
	}

	// Set the routerID (Unique ID for BGP) to the source IP
	// Also set the BGP peering to the sourceIP
	bgpConfig.RouterID, bgpConfig.SourceIP = srcIP, srcIP

	peerASN := ""
	regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-asn$", prefix))
	for k, v := range node.Annotations {
		if regex.Match([]byte(k)) {
			peerASN = v
		}
	}
	if peerASN == "" {
		return bgpConfig, bgpPeer, fmt.Errorf("peer-asn value missing or empty")
	}

	u64, err = strconv.ParseUint(peerASN, 10, 32)
	if err != nil {
		return bgpConfig, bgpPeer, err
	}

	bgpPeer.AS = uint32(u64)

	peerIPString := ""
	regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-[0-9]+-)?peer-ip$", prefix))
	for k, v := range node.Annotations {
		if regex.Match([]byte(k)) {
			peerIPString += v + ","
		}
	}
	peerIPString = strings.TrimRight(peerIPString, ",")

	peerIPs := strings.Split(peerIPString, ",")
	if len(peerIPs) >= 1 && peerIPs[0] == "" || len(peerIPs) == 0 {
		return bgpConfig, bgpPeer, fmt.Errorf("peer-ip value missing or empty")
	}

	bgpConfig.Peers = make([]kubevip.BGPPeer, 0, len(peerIPs))
	regexPass := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?bgp-pass$", prefix))
	regexMultiHop := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-multi-hop$", prefix))
	for _, peerIP := range peerIPs {
		ipAddr := strings.TrimSpace(peerIP)

		if ipAddr != "" {
			bgpPeer.Address = ipAddr
			// Check if we're also expecting a password for this peer
			base64BGPPassword := ""
			for k, v := range node.Annotations {
				if regexPass.Match([]byte(k)) {
					base64BGPPassword = v
				}
			}
			if base64BGPPassword != "" {
				// Decode base64 encoded string
				decodedPassword, err := base64.StdEncoding.DecodeString(base64BGPPassword)
				if err != nil {
					return bgpConfig, bgpPeer, err
				}
				// Set the password for each peer
				bgpPeer.Password = string(decodedPassword)
			}

			// Check if multi-hop is enabled.
			for k, v := range node.Annotations {
				if regexMultiHop.MatchString(k) {
					switch v {
					case "true":
						bgpPeer.MultiHop = true
					case "false":
						bgpPeer.MultiHop = false
					default:
						return bgpConfig, bgpPeer,
							fmt.Errorf("invalid %q annotation value: %q, must be \"true\" or \"false\"", k, v)
					}
				}
			}
			bgpConfig.Peers = append(bgpConfig.Peers, bgpPeer)
		}
	}

	//log.Debugf("BGPConfig: %v\n", bgpConfig)
	//log.Debugf("BGPPeerConfig: %v\n", bgpPeer)

	return bgpConfig, bgpPeer, nil
}