Multihop_and_use CCM Secret

cmd/kube-vip-manifests.go

@@ -14,10 +14,11 @@ import (
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
 
 //var inCluster bool
+var taint bool
 
 func init() {
-	kubeManifestPod.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeManifestDaemon.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
@@ -78,7 +79,7 @@ var kubeManifestDaemon = &cobra.Command{
 			cmd.Help()
 			log.Fatalln("No address is specified for kube-vip to expose services on")
 		}
-		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version, inCluster)
+		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
 
 		fmt.Println(cfg)
 	},

cmd/kube-vip.go

@@ -82,7 +82,7 @@ func init() {
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnablePacket, "packet", false, "This will use the Packet API (requires the token ENV) to update the EIP <-> VIP")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PacketAPIKey, "packetKey", "", "The API token for authenticating with the Packet API")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PacketProject, "packetProject", "", "The name of project already created within Packet")
-	kubeVipCmd.PersistentFlags().StringVar(&providerConfig, "provider-config", "", "The path to a provider configuration")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
 
 	// Load Balancer flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")

docs/manifests/kube-vip-em.yaml

@@ -0,0 +1,97 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services", "services/status"]
+    verbs: ["list","get","watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: kube-vip-ds
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        name: kube-vip-ds
+    spec:
+      containers:
+      - args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "false"
+        - name: vip_interface
+          value: lo
+        - name: port
+          value: "6443"
+        - name: vip_cidr
+          value: "32"
+        - name: svc_enable
+          value: "true"
+        - name: provider_config
+          value: /etc/cloud-sa/cloud-sa.json
+        - name: vip_packet
+          value: "true"
+        - name: bgp_enable
+          value: "true"
+        image: plndr/kube-vip:0.2.3
+        imagePullPolicy: Always
+        name: kube-vip
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - SYS_TIME
+        volumeMounts:
+        - mountPath: /etc/ssl/certs
+          name: ca-certs
+          readOnly: true
+        - mountPath: /etc/cloud-sa
+          name: cloud-sa-volume
+          readOnly: true
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      volumes:
+      - hostPath:
+          path: /etc/ssl/certs
+        name: ca-certs
+      - name: cloud-sa-volume
+        secret:
+          secretName: packet-cloud-config
+  updateStrategy: {}
+status:
+  currentNumberScheduled: 0
+  desiredNumberScheduled: 0
+  numberMisscheduled: 0
+  numberReady: 0

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect
 	github.com/mitchellh/mapstructure v1.3.3 // indirect
 	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.5.0
+	github.com/packethost/packngo v0.5.1
 	github.com/pelletier/go-toml v1.8.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.6.0

go.sum

@@ -381,6 +381,7 @@ github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0 h1:sqWev+JAbQevLXYorfyT
 github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0/go.mod h1:IVw8wEHROhX0qrmI8c6j3N8EDXZSC4YkktSzkX/JZ8Q=
 github.com/packethost/packngo v0.5.0 h1:WGpfeRMstPqgyXGUXl6b9xFsbUudXU3p0+JlYri290U=
 github.com/packethost/packngo v0.5.0/go.mod h1:aRxUEV1TprXVcWr35v8tNYgZMjv7FHaInXx224vF2fc=
+github.com/packethost/packngo v0.5.1/go.mod h1:aRxUEV1TprXVcWr35v8tNYgZMjv7FHaInXx224vF2fc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=

pkg/cluster/clusterLeader.go

@@ -161,7 +161,7 @@ func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager, bgpSe
 		// We're using Packet with BGP, popuplate the Peer information from the API
 		if c.EnableBGP {
 			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
+			err = packet.BGPLookup(packetClient, c, "") //TODO: This will need looking at in the future
 			if err != nil {
 				log.Error(err)
 			}

pkg/kubevip/config_generator.go

@@ -74,12 +74,15 @@ const (
 	//vipAddPeersToLB defines that RAFT peers should be added to the load-balancer
 	vipAddPeersToLB = "vip_addpeerstolb"
 
-	//vipPacket defines that the packet API will be used tor EIP
+	//vipPacket defines that the packet API will be used for EIP
 	vipPacket = "vip_packet"
 
 	//vipPacket defines which project within Packet to use
 	vipPacketProject = "vip_packetproject"
 
+	//providerConfig defines a path to a configuration that should be parsed
+	providerConfig = "provider_config"
+
 	//bgpEnable defines if BGP should be enabled
 	bgpEnable = "bgp_enable"
 	//bgpRouterID defines the routerID for the BGP server
@@ -150,8 +153,13 @@ func ParseEnvironment(c *Config) error {
 		c.Interface = env
 	}
 
-	// Find Kubernetes Leader Election configuration
+	// Find provider configuration
+	env = os.Getenv(providerConfig)
+	if env != "" {
+		c.ProviderConfig = env
+	}
 
+	// Find Kubernetes Leader Election configuration
 	env = os.Getenv(vipLeaderElection)
 	if env != "" {
 		b, err := strconv.ParseBool(env)
@@ -622,6 +630,17 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		newEnvironment = append(newEnvironment, raft...)
 	}
 
+	// If we're specifying a configuration
+	if c.ProviderConfig != "" {
+		provider := []corev1.EnvVar{
+			{
+				Name:  providerConfig,
+				Value: c.ProviderConfig,
+			},
+		}
+		newEnvironment = append(newEnvironment, provider...)
+	}
+
 	// If Packet is enabled then add it to the manifest
 	if c.EnablePacket {
 		packet := []corev1.EnvVar{
@@ -639,16 +658,21 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 			},
 		}
 		newEnvironment = append(newEnvironment, packet...)
-
 	}
 
-	// If BGP is enabled then add it to the manifest
+	// If BGP, but we're not using packet
 	if c.EnableBGP {
 		bgp := []corev1.EnvVar{
 			{
 				Name:  bgpEnable,
 				Value: strconv.FormatBool(c.EnableBGP),
 			},
+		}
+		newEnvironment = append(newEnvironment, bgp...)
+	}
+	// If BGP, but we're not using packet
+	if c.EnableBGP && !c.EnablePacket {
+		bgpConfig := []corev1.EnvVar{
 			{
 				Name:  bgpRouterID,
 				Value: c.BGPConfig.RouterID,
@@ -661,10 +685,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 				Name:  bgpPeerAddress,
 				Value: c.BGPPeerConfig.Address,
 			},
-			{
-				Name:  bgpMultiHop,
-				Value: strconv.FormatBool(c.BGPPeerConfig.MultiHop),
-			},
 			{
 				Name:  bgpPeerPassword,
 				Value: c.BGPPeerConfig.Password,
@@ -674,7 +694,7 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 				Value: fmt.Sprintf("%d", c.BGPPeerConfig.AS),
 			},
 		}
-		newEnvironment = append(newEnvironment, bgp...)
+		newEnvironment = append(newEnvironment, bgpConfig...)
 
 	}
 
@@ -807,6 +827,27 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		}
 		newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, adminConfVolume)
 	}
+
+	if c.ProviderConfig != "" {
+		providerConfigMount := corev1.VolumeMount{
+			Name:      "cloud-sa-volume",
+			MountPath: "/etc/cloud-sa",
+			ReadOnly:  true,
+		}
+		newManifest.Spec.Containers[0].VolumeMounts = append(newManifest.Spec.Containers[0].VolumeMounts, providerConfigMount)
+
+		providerConfigVolume := corev1.Volume{
+			Name: "cloud-sa-volume",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: "packet-cloud-config",
+				},
+			},
+		}
+		newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, providerConfigVolume)
+
+	}
+
 	return newManifest
 
 }
@@ -819,7 +860,7 @@ func GeneratePodManifestFromConfig(c *Config, imageVersion string, inCluster boo
 }
 
 // GenerateDeamonsetManifestFromConfig will take a kube-vip config and generate a manifest
-func GenerateDeamonsetManifestFromConfig(c *Config, imageVersion string, inCluster bool) string {
+func GenerateDeamonsetManifestFromConfig(c *Config, imageVersion string, inCluster, taint bool) string {
 
 	podSpec := generatePodSpec(c, imageVersion, inCluster).Spec
 	newManifest := &appv1.DaemonSet{
@@ -847,17 +888,17 @@ func GenerateDeamonsetManifestFromConfig(c *Config, imageVersion string, inClust
 			},
 		},
 	}
-
-	newManifest.Spec.Template.Spec.Tolerations = []corev1.Toleration{
-		{
-			Key:    "node-role.kubernetes.io/master",
-			Effect: corev1.TaintEffectNoSchedule,
-		},
+	if taint {
+		newManifest.Spec.Template.Spec.Tolerations = []corev1.Toleration{
+			{
+				Key:    "node-role.kubernetes.io/master",
+				Effect: corev1.TaintEffectNoSchedule,
+			},
+		}
+		newManifest.Spec.Template.Spec.NodeSelector = map[string]string{
+			"node-role.kubernetes.io/master": "true",
+		}
 	}
-	newManifest.Spec.Template.Spec.NodeSelector = map[string]string{
-		"node-role.kubernetes.io/master": "true",
-	}
-
 	b, _ := yaml.Marshal(newManifest)
 	return string(b)
 }

pkg/kubevip/config_types.go

@@ -76,6 +76,9 @@ type Config struct {
 	// PacketProject, is the name of a particular defined project
 	PacketProject string
 
+	// ProviderConfig, is the path to a provider configuration file
+	ProviderConfig string
+
 	// LoadBalancers are the various services we can load balance over
 	LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
 }

pkg/manager/manager_bgp.go

@@ -2,6 +2,7 @@ package manager
 
 import (
 	"context"
+	"os"
 	"syscall"
 
 	"github.com/packethost/packngo"
@@ -20,6 +21,18 @@ func (sm *Manager) startBGP() error {
 	// If Packet is enabled then we can begin our preperation work
 	var packetClient *packngo.Client
 	if sm.config.EnablePacket {
+		var projectID string
+		if sm.config.ProviderConfig != "" {
+			key, project, err := packet.GetPacketConfig(sm.config.ProviderConfig)
+			if err != nil {
+				log.Error(err)
+			} else {
+				// Set the environment variable with the key for the project
+				os.Setenv("PACKET_AUTH_TOKEN", key)
+				// Update the configuration with the project key
+				projectID = project
+			}
+		}
 		packetClient, err = packngo.NewClient()
 		if err != nil {
 			log.Error(err)
@@ -28,7 +41,7 @@ func (sm *Manager) startBGP() error {
 		// We're using Packet with BGP, popuplate the Peer information from the API
 		if sm.config.EnableBGP {
 			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, sm.config)
+			err = packet.BGPLookup(packetClient, sm.config, projectID)
 			if err != nil {
 				log.Error(err)
 			}

pkg/packet/bgp.go

@@ -10,13 +10,17 @@ import (
 )
 
 // BGPLookup will use the Packet API functions to populate the BGP information
-func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
-
-	proj := findProject(k.PacketProject, c)
-	if proj == nil {
-		return fmt.Errorf("Unable to find Project [%s]", k.PacketProject)
+func BGPLookup(c *packngo.Client, k *kubevip.Config, projectID string) error {
+	var thisDevice *packngo.Device
+	if projectID == "" {
+		proj := findProject(k.PacketProject, c)
+		if proj == nil {
+			return fmt.Errorf("Unable to find Project [%s]", k.PacketProject)
+		}
+		thisDevice = findSelf(c, proj.ID)
+	} else {
+		thisDevice = findSelf(c, projectID)
 	}
-	thisDevice := findSelf(c, proj.ID)
 	if thisDevice == nil {
 		return fmt.Errorf("Unable to find local/this device in packet API")
 	}
@@ -47,8 +51,9 @@ func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
 	// Add the peer(s)
 	for x := range neighbours[0].PeerIps {
 		peer := bgp.Peer{
-			Address: neighbours[0].PeerIps[x],
-			AS:      uint32(neighbours[0].PeerAs),
+			Address:  neighbours[0].PeerIps[x],
+			AS:       uint32(neighbours[0].PeerAs),
+			MultiHop: neighbours[0].Multihop,
 		}
 		k.BGPConfig.Peers = append(k.BGPConfig.Peers, peer)
 	}

pkg/service/manager_bgp.go

@@ -30,7 +30,7 @@ func (sm *Manager) startBGP() error {
 		// We're using Packet with BGP, popuplate the Peer information from the API
 		if sm.config.EnableBGP {
 			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, sm.config)
+			err = packet.BGPLookup(packetClient, sm.config, "") // TODO: This will need looking at in the future
 			if err != nil {
 				log.Error(err)
 			}