Table mode is now added to kube-vip

This commit is contained in:
thebsdbox
2022-06-29 16:08:02 +01:00
parent c6f37f433a
commit 1bcd101b53
16 changed files with 252 additions and 157 deletions

View File

@@ -1,16 +1,12 @@
package cmd
import (
"context"
"fmt"
"os"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -91,39 +87,6 @@ var kubeKubeadmJoin = &cobra.Command{
log.Fatalf("Unable to find file [%s]", kubeConfigPath)
}
// We will use kubeconfig in order to find all the master nodes
// use the current context in kubeconfig
clientset, err := k8s.NewClientset(kubeConfigPath, false, "")
if err != nil {
log.Fatal(err.Error())
}
opts := metav1.ListOptions{}
opts.LabelSelector = "node-role.kubernetes.io/master"
nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
if err != nil {
log.Fatal(err.Error())
}
// Iterate over all nodes that are masters and find the details to build a peer list
for x := range nodes.Items {
// Get hostname and address
var nodeAddress, nodeHostname string
for y := range nodes.Items[x].Status.Addresses {
switch nodes.Items[x].Status.Addresses[y].Type {
case corev1.NodeHostName:
nodeHostname = nodes.Items[x].Status.Addresses[y].Address
case corev1.NodeInternalIP:
nodeAddress = nodes.Items[x].Status.Addresses[y].Address
}
}
newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
if err != nil {
panic(err.Error())
}
initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
}
// Generate manifest and print
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg)

View File

@@ -68,16 +68,20 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc..")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address") // todo: deprecate
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "Port for the VIP")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
// Clustering type (leaderElection)
@@ -86,7 +90,7 @@ func init() {
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
// Packet flags
// Equinix Metal flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
@@ -94,7 +98,6 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
// BGP flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
@@ -116,6 +119,9 @@ func init() {
// Service flags
kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
// Routing Table flags
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
// Behaviour flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPane, "controlplane", false, "Enable HA for control plane")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
@@ -205,11 +211,9 @@ var kubeVipManager = &cobra.Command{
// Set the logging level for all subsequent functions
log.SetLevel(log.Level(logLevel))
go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
log.Infof("Starting kube-vip.io [%s]", Release.Version)
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
@@ -230,12 +234,17 @@ var kubeVipManager = &cobra.Command{
if initConfig.EnableWireguard {
mode = "Wireguard"
}
if initConfig.EnableRoutingTable {
mode = "Routing Table"
}
// Provide configuration to output/logging
log.Infof("namespace [%s], Mode: [%s], Features(s): Control Plane:[%t], Services:[%t]", initConfig.Namespace, mode, initConfig.EnableControlPane, initConfig.EnableServices)
// End if nothing is enabled
if !initConfig.EnableServices && !initConfig.EnableControlPane {
log.Fatalln("no modes are enabled")
log.Fatalln("no features are enabled")
}
// If we're using wireguard then all traffic goes through the wg0 interface

View File

@@ -40,10 +40,11 @@ func startNetworking(c *kubevip.Config) (vip.Network, error) {
address = c.Address
}
network, err := vip.NewConfig(address, c.Interface, c.DDNS)
network, err := vip.NewConfig(address, c.Interface, c.VIPSubnet, c.DDNS, c.RoutingTableID)
if err != nil {
return nil, err
}
return network, nil
}

View File

@@ -174,12 +174,17 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
if err != nil {
log.Warnf("Attempted to clean existing VIP => %v", err)
}
err = cluster.Network.AddIP()
if err != nil {
log.Warnf("%v", err)
if c.EnableRoutingTable {
err = cluster.Network.AddRoute()
if err != nil {
log.Warnf("%v", err)
}
} else {
err = cluster.Network.AddIP()
if err != nil {
log.Warnf("%v", err)
}
}
if c.EnableARP {
//ctxArp, cancelArp = context.WithCancel(context.Background())

View File

@@ -184,6 +184,26 @@ func ParseEnvironment(c *Config) error {
c.EnableARP = b
}
// Wireguard Mode
env = os.Getenv(vipWireguard)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableWireguard = b
}
// Routing Table Mode
env = os.Getenv(vipRoutingTable)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableRoutingTable = b
}
// BGP Server options
env = os.Getenv(bgpEnable)
if env != "" {
@@ -194,16 +214,6 @@ func ParseEnvironment(c *Config) error {
c.EnableBGP = b
}
// BGP Server options
env = os.Getenv(vipWireguard)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableWireguard = b
}
// BGP Router interface determines an interface that we can use to find an address for
env = os.Getenv(bgpRouterInterface)
if env != "" {

View File

@@ -97,6 +97,9 @@ const (
//vipWireguard - defines if wireguard will be used for vips
vipWireguard = "vip_wireguard" //nolint
//vipRoutingTable - defines if table mode will be used for vips
vipRoutingTable = "vip_routingtable" //nolint
//cpNamespace defines the namespace the control plane pods will run in
cpNamespace = "cp_namespace"

View File

@@ -177,18 +177,29 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
newEnvironment = append(newEnvironment, packet...)
}
// If BGP, but we're not using packet
// Detect and enable wireguard mode
if c.EnableWireguard {
bgp := []corev1.EnvVar{
wireguard := []corev1.EnvVar{
{
Name: vipWireguard,
Value: strconv.FormatBool(c.EnableWireguard),
},
}
newEnvironment = append(newEnvironment, bgp...)
newEnvironment = append(newEnvironment, wireguard...)
}
// If BGP, but we're not using packet
// Detect and enable routing table mode
if c.EnableRoutingTable {
routingtable := []corev1.EnvVar{
{
Name: vipWireguard,
Value: strconv.FormatBool(c.EnableRoutingTable),
},
}
newEnvironment = append(newEnvironment, routingtable...)
}
// If BGP, but we're not using Equinix Metal
if c.EnableBGP {
bgp := []corev1.EnvVar{
{
@@ -198,7 +209,7 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, bgp...)
}
// If BGP, but we're not using packet
// If BGP, but we're not using Equinix Metal
if c.EnableBGP && !c.EnableMetal {
bgpConfig := []corev1.EnvVar{
{

View File

@@ -25,19 +25,6 @@ func ParseBackendConfig(ep string) (*BackEnd, error) {
return &BackEnd{Address: endpoint[0], Port: p}, nil
}
//ParsePeerConfig -
func ParsePeerConfig(ep string) (*RaftPeer, error) {
endpoint := strings.Split(ep, ":")
if len(endpoint) != 3 {
return nil, fmt.Errorf("ensure a peer is in in the format id:address:port, e.g. server1:10.0.0.1:8080")
}
p, err := strconv.Atoi(endpoint[2])
if err != nil {
return nil, err
}
return &RaftPeer{ID: endpoint[0], Address: endpoint[1], Port: p}, nil
}
//OpenConfig will attempt to read a file and parse it's contents into a configuration
func OpenConfig(path string) (*Config, error) {
if path == "" {
@@ -73,61 +60,12 @@ func (c *Config) PrintConfig() {
fmt.Print(string(b))
}
//ParseFlags will write the current configuration to a specified [path]
func (c *Config) ParseFlags(localPeer string, remotePeers, backends []string) error {
// Parse localPeer
p, err := ParsePeerConfig(localPeer)
if err != nil {
return err
}
c.LocalPeer = *p
// Parse remotePeers
//Iterate backends
for i := range remotePeers {
p, err := ParsePeerConfig(remotePeers[i])
if err != nil {
return err
}
c.RemotePeers = append(c.RemotePeers, *p)
}
//Iterate backends
for i := range backends {
b, err := ParseBackendConfig(backends[i])
if err != nil {
return err
}
c.LoadBalancers[0].Backends = append(c.LoadBalancers[0].Backends, *b)
}
return nil
}
//SampleConfig will create an example configuration and write it to the specified [path]
func SampleConfig() {
// Generate Sample configuration
c := &Config{
// Generate sample peers
RemotePeers: []RaftPeer{
{
ID: "server2",
Address: "192.168.0.2",
Port: 10000,
},
{
ID: "server3",
Address: "192.168.0.3",
Port: 10000,
},
},
LocalPeer: RaftPeer{
ID: "server1",
Address: "192.168.0.1",
Port: 10000,
},
// Virtual IP address
VIP: "192.168.0.100",
// Interface to bind to

View File

@@ -18,6 +18,9 @@ type Config struct {
// EnableWireguard, will use wireguard to advertise the VIP address
EnableWireguard bool `yaml:"enableWireguard"`
// EnableRoutingTable, will use the routing table to advertise the VIP address
EnableRoutingTable bool `yaml:"enableRoutingTable"`
// EnableControlPane, will enable the control plane functionality (used for hybrid behaviour)
EnableControlPane bool `yaml:"enableControlPane"`
@@ -33,18 +36,15 @@ type Config struct {
// LeaderElection defines the settings around Kubernetes LeaderElection
LeaderElection
// LocalPeer is the configuration of this host
LocalPeer RaftPeer `yaml:"localPeer"`
// Peers are all of the peers within the RAFT cluster
RemotePeers []RaftPeer `yaml:"remotePeers"`
// AddPeersAsBackends, this will automatically add RAFT peers as backends to a loadbalancer
AddPeersAsBackends bool `yaml:"addPeersAsBackends"`
// VIP is the Virtual IP address exposed for the cluster (TODO: deprecate)
VIP string `yaml:"vip"`
// VipSubnet is the Subnet that is applied to the VIP
VIPSubnet string `yaml:"vipSubnet"`
// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
VIPCIDR string `yaml:"vipCidr"`
@@ -81,6 +81,9 @@ type Config struct {
// Forwarding method for the IPVS Service
LoadBalancerForwardingMethod string `yaml:"lbForwardingMethod"`
// Routing Table ID for when using routing table mode
RoutingTableID int `yaml:"routingTableID"`
// BGP Configuration
BGPConfig bgp.Config
BGPPeerConfig bgp.Peer
@@ -124,18 +127,6 @@ type LeaderElection struct {
RetryPeriod int
}
// RaftPeer details the configuration of all cluster peers
type RaftPeer struct {
// ID is the unique identifier a peer instance
ID string `yaml:"id"`
// IP Address of a peer instance
Address string `yaml:"address"`
// Listening port of this peer instance
Port int `yaml:"port"`
}
// LoadBalancer contains the configuration of a load balancing instance
type LoadBalancer struct {
// Name of a LoadBalancer

View File

@@ -56,12 +56,15 @@ func NewInstance(service *v1.Service, config *kubevip.Config) (*Instance, error)
// Generate new Virtual IP configuration
newVip := &kubevip.Config{
VIP: instanceAddress, //TODO support more than one vip?
Interface: serviceInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPCIDR: config.VIPCIDR,
VIP: instanceAddress, //TODO support more than one vip?
Interface: serviceInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPCIDR: config.VIPCIDR,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
}
// Create new service

View File

@@ -137,6 +137,11 @@ func (sm *Manager) Start() error {
return sm.startWireguard()
}
if sm.config.EnableRoutingTable {
log.Infoln("Starting Kube-vip Manager with the Routing Table engine")
return sm.startTableMode()
}
log.Errorln("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
return nil
}

View File

@@ -71,7 +71,7 @@ func (sm *Manager) startARP() error {
} else {
ns, err = returnNameSpace()
if err != nil {
log.Errorf("Unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}
}

View File

@@ -0,0 +1,110 @@
package manager
import (
"context"
"os"
"time"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startTableMode() error {
var ns string
var err error
id, err := os.Hostname()
if err != nil {
return err
}
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Infof("all routing table entries will exist in table [%d]", sm.config.RoutingTableID)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
ns, err = returnNameSpace()
if err != nil {
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Infof("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Infof("beginning services leadership, namespace [%s], lock name [%s], id [%s]", ns, plunderLock, id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: plunderLock,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: 10 * time.Second,
RenewDeadline: 5 * time.Second,
RetryPeriod: 1 * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error(err)
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Infof("leader lost: %s", id)
for x := range sm.serviceInstances {
sm.serviceInstances[x].cluster.Stop()
}
log.Fatal("lost leadership, restarting kube-vip")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == id {
// I just got the lock
return
}
log.Infof("new leader elected: %s", identity)
},
},
})
}
return nil
}

View File

@@ -55,7 +55,7 @@ func (sm *Manager) startWireguard() error {
ns, err = returnNameSpace()
if err != nil {
log.Errorf("Unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
log.Warnf("unable to auto-detect namespace, dropping to [%s]", sm.config.Namespace)
ns = sm.config.Namespace
}

View File

@@ -27,6 +27,10 @@ func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) erro
return err
}
for x := range sm.serviceInstances {
sm.serviceInstances[x].cluster.Stop()
}
log.Infof("Shutting down kube-Vip")
return nil

View File

@@ -15,7 +15,9 @@ const (
// Network is an interface that enable managing operations for a given IP
type Network interface {
AddIP() error
AddRoute() error
DeleteIP() error
DeleteRoute() error
IsSet() (bool, error)
IP() string
SetIP(ip string) error
@@ -35,6 +37,8 @@ type network struct {
dnsName string
isDDNS bool
routeTable int
}
func netlinkParse(addr string) (*netlink.Addr, error) {
@@ -46,19 +50,29 @@ func netlinkParse(addr string) (*netlink.Addr, error) {
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, isDDNS bool) (Network, error) {
func NewConfig(address string, iface string, subnet string, isDDNS bool, tableID int) (Network, error) {
result := &network{}
link, err := netlink.LinkByName(iface)
if err != nil {
return result, errors.Wrapf(err, "could not get link for interface '%s'", iface)
}
result.link = link
result.routeTable = tableID
if IsIP(address) {
result.address, err = netlinkParse(address)
if err != nil {
return result, errors.Wrapf(err, "could not parse address '%s'", address)
// Check if the subnet needs overriding
if subnet != "" {
result.address, err = netlink.ParseAddr(address + subnet)
if err != nil {
return result, errors.Wrapf(err, "could not parse address '%s'", address)
}
} else {
result.address, err = netlinkParse(address)
if err != nil {
return result, errors.Wrapf(err, "could not parse address '%s'", address)
}
}
// Ensure we don't have a global address on loopback
if iface == "lo" {
@@ -91,6 +105,34 @@ func NewConfig(address string, iface string, isDDNS bool) (Network, error) {
return result, err
}
// AddRoute - Add an IP address to a route table
func (configurator *network) AddRoute() error {
route := &netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Dst: configurator.address.IPNet,
LinkIndex: configurator.link.Attrs().Index,
Table: configurator.routeTable,
}
if err := netlink.RouteAdd(route); err != nil {
return err
}
return nil
}
// AddRoute - Add an IP address to a route table
func (configurator *network) DeleteRoute() error {
route := &netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Dst: configurator.address.IPNet,
LinkIndex: configurator.link.Attrs().Index,
Table: configurator.routeTable,
}
if err := netlink.RouteDel(route); err != nil {
return err
}
return nil
}
// AddIP - Add an IP address to the interface
func (configurator *network) AddIP() error {
if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {