Add leader election using etcd as a backend

This adds a new leader election mode that instead of using the kube api server to obtain a lease, it relies solely on etcd. By removing the dependency on the api server, we can use it even before the api server is up, before cluster creation. In an external etcd topology, this way we can make the api server point to the VIPs instead of to the etcd machines IPs. This configuration isolates the CP configuration from the etcd machine IPS. It's is super convenient to avoid having to roll new control plane nodes when the etcd nodes change. Signed-off-by: Guillermo Gaston <gaslor@amazon.com>
2026-09-20 08:03:47 +08:00 · 2023-09-30 19:08:41 +00:00
parent cbcc68e69d
commit ae307f8cad
31 changed files with 1829 additions and 219 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -33,6 +33,18 @@ jobs:
          go-version: ${{ env.GO_VERSION }}
      - name: Run tests
        run: make unit-tests
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set up Go ${{ env.GO_VERSION }}
+      uses: actions/setup-go@v4
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Run tests
+      run: make integration-tests
  e2e-tests:
    runs-on: ubuntu-latest
    name: E2E ARP tests
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,7 @@
 .idea
 kube-vip
 .vscode
+bin
+testing/e2e/etcd/certs
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data
--- a/7
+++ b/7
@@ -15,7 +15,7 @@ TARGETOS=linux
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= plndr

 .PHONY: all build clean install uninstall fmt simplify check run e2e-tests

@@ -119,8 +119,11 @@ manifests:
 unit-tests:
 	go test ./...

+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
 e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd

 service-tests:
 	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/e2e/services -Services
--- a/cmd/kube-vip-start.go
+++ b/cmd/kube-vip-start.go
@@ -9,11 +9,13 @@ import (
 )

 // Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
+var (
+	startConfig                         kubevip.Config
+	startConfigLB                       kubevip.LoadBalancer
+	startLocalPeer, startKubeConfigPath string
+	startRemotePeers, startBackends     []string
+	inCluster                           bool
+)

 func init() {
 	// Get the configuration file
@@ -65,6 +67,10 @@ var kubeVipStart = &cobra.Command{
 			log.Fatalln(err)
 		}

+		if startConfig.LeaderElectionType == "etcd" {
+			log.Fatalln("Leader election with etcd not supported in start command, use manager")
+		}
+
 		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
 		if err != nil {
 			log.Fatalf("%v", err)
@@ -109,6 +115,5 @@ var kubeVipStart = &cobra.Command{
 				}
 			}
 		}
-
 	},
 }
--- a/cmd/kube-vip.go
+++ b/cmd/kube-vip.go
@@ -87,6 +87,7 @@ func init() {

 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
@@ -139,6 +140,12 @@ func init() {
 	// Prometheus HTTP Server
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")

+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
 	kubeVipCmd.AddCommand(kubeVipManager)
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/jpillora/backoff v1.0.0
 	github.com/kamhlos/upnp v0.0.0-20210324072331-5661950dff08
 	github.com/mdlayher/ndp v1.0.1
-	github.com/onsi/ginkgo/v2 v2.11.0
+	github.com/onsi/ginkgo/v2 v2.12.1
 	github.com/onsi/gomega v1.27.10
 	github.com/osrg/gobgp/v3 v3.17.0
 	github.com/packethost/packngo v0.30.0
@@ -21,7 +21,12 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vishvananda/netlink v1.2.1-beta.2
-	golang.org/x/sys v0.10.0
+	go.etcd.io/etcd/api/v3 v3.5.9
+	go.etcd.io/etcd/client/pkg/v3 v3.5.9
+	go.etcd.io/etcd/client/v3 v3.5.9
+	go.uber.org/zap v1.21.0
+	golang.org/x/exp v0.0.0-20231005195138-3e424a577f31
+	golang.org/x/sys v0.12.0
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
 	k8s.io/api v0.27.4
 	k8s.io/apimachinery v0.27.4
@@ -36,6 +41,8 @@ require (
 	github.com/alessio/shellescape v1.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/eapache/channels v1.1.0 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -88,14 +95,16 @@ require (
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.1.0 // indirect
-	golang.org/x/tools v0.9.3 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
--- a/go.sum
+++ b/go.sum
@@ -44,6 +44,8 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -60,6 +62,10 @@ github.com/cloudflare/ipvs v0.9.1/go.mod h1:5H4icNJZ8T4H7bg0/THRew5BbNOkDgl2RC1t
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -107,6 +113,7 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -281,8 +288,8 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
-github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA=
+github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/osrg/gobgp/v3 v3.17.0 h1:Rh4hLzirQJnAz//57bxpiiDTkhPtsDuJ6oy0aeciJDA=
@@ -359,13 +366,30 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs=
+go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k=
+go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE=
+go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4=
+go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E=
+go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -374,8 +398,8 @@ golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -386,6 +410,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 h1:9k5exFQKQglLo+RoP+4zMjOFE14P6+vyR0baDAi0Rcs=
+golang.org/x/exp v0.0.0-20231005195138-3e424a577f31/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -411,7 +437,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -449,6 +475,7 @@ golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -456,8 +483,8 @@ golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -527,8 +554,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -541,11 +570,11 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -554,8 +583,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -610,9 +639,10 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
-golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
-golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -737,6 +767,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
--- a/pkg/cluster/clusterLeaderElection.go
+++ b/pkg/cluster/clusterLeaderElection.go
@@ -12,6 +12,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/equinixmetal"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
 	"github.com/kube-vip/kube-vip/pkg/k8s"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
@@ -19,6 +20,7 @@ import (
 	"github.com/packethost/packngo"

 	log "github.com/sirupsen/logrus"
+	clientv3 "go.etcd.io/etcd/client/v3"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +37,8 @@ type Manager struct {
 	KubernetesClient *kubernetes.Clientset
 	// This channel is used to signal a shutdown
 	SignalChan chan os.Signal
+
+	EtcdClient *clientv3.Client
 }

 // NewManager will create a new managing object
@@ -79,20 +83,6 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *

 	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, c.LeaseName, id)

-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:        c.LeaseName,
-			Namespace:   c.Namespace,
-			Annotations: c.LeaseAnnotations,
-		},
-		Client: sm.KubernetesClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
 	// use a Go context so we can tell the leaderelection code when we
 	// want to step down
 	ctx, cancel := context.WithCancel(context.Background())
@@ -177,6 +167,89 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		}
 	}

+	run := &runConfig{
+		config:  c,
+		leaseID: id,
+		sm:      sm,
+		onStartedLeading: func(ctx context.Context) {
+			// As we're leading lets start the vip service
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient)
+			if err != nil {
+				log.Errorf("Error starting the VIP service on the leader [%s]", err)
+			}
+		},
+		onStoppedLeading: func() {
+			// we can do cleanup here
+			log.Info("This node is becoming a follower within the cluster")
+
+			// Stop the dns context
+			cancelDNS()
+			// Stop the Arp context if it is running
+			cancelArp()
+
+			// Stop the BGP server
+			if bgpServer != nil {
+				err := bgpServer.Close()
+				if err != nil {
+					log.Warnf("%v", err)
+				}
+			}
+
+			err := cluster.Network.DeleteIP()
+			if err != nil {
+				log.Warnf("%v", err)
+			}
+
+			log.Fatal("lost leadership, restarting kube-vip")
+		},
+		onNewLeader: func(identity string) {
+			// we're notified when new leader elected
+			log.Infof("Node [%s] is assuming leadership of the cluster", identity)
+		},
+	}
+
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		cluster.runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		cluster.runEtcdLeaderElectionOrDie(ctx, run)
+	default:
+		log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
+	}
+
+	return nil
+}
+
+type runConfig struct {
+	config  *kubevip.Config
+	leaseID string
+	sm      *Manager
+
+	// onStartedLeading is called when this member starts leading.
+	onStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	onStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	onNewLeader func(identity string)
+}
+
+func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:        run.config.LeaseName,
+			Namespace:   run.config.Namespace,
+			Annotations: run.config.LeaseAnnotations,
+		},
+		Client: run.sm.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.leaseID,
+		},
+	}
+
 	// start the leader election code loop
 	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
 		Lock: lock,
@@ -187,49 +260,29 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		// get elected before your background loop finished, violating
 		// the stated goal of the lease.
 		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
+		LeaseDuration:   time.Duration(run.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.config.RetryPeriod) * time.Second,
 		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// As we're leading lets start the vip service
-				err = cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient)
-				if err != nil {
-					log.Errorf("Error starting the VIP service on the leader [%s]", err)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the dns context
-				cancelDNS()
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				log.Fatal("lost leadership, restarting kube-vip")
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-			},
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
 		},
 	})
+}

-	return nil
+func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.sm.EtcdClient},
+		Name:                 run.config.LeaseName,
+		MemberID:             run.leaseID,
+		LeaseDurationSeconds: int64(run.config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
 }

 func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) error {
--- a/pkg/etcd/client.go
+++ b/pkg/etcd/client.go
@@ -0,0 +1,26 @@
+package etcd
+
+import (
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func NewClient(c *kubevip.Config) (*clientv3.Client, error) {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: c.Etcd.CAFile,
+		CertFile:      c.Etcd.ClientCertFile,
+		KeyFile:       c.Etcd.ClientKeyFile,
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return clientv3.New(clientv3.Config{
+		Endpoints: c.Etcd.Endpoints,
+		TLS:       clientTLS,
+	})
+}
--- a/pkg/etcd/election.go
+++ b/pkg/etcd/election.go
@@ -0,0 +1,226 @@
+package etcd
+
+import (
+	"context"
+	"hash/fnv"
+	"time"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+// LeaderElectionConfig allows to configure the leader election params.
+type LeaderElectionConfig struct {
+	// EtcdConfig contains the client to connect to the etcd cluster.
+	EtcdConfig ClientConfig
+
+	// Name uniquely identifies this leader election. All members of the same election
+	// should use the same value here.
+	Name string
+
+	// MemberID identifies uniquely this contestant from other in the leader election.
+	// It will be converted to an int64 using a hash, so theoretically collisions are possible
+	// when using a string. If you want to guarantee safety, us MemberUniqueID to specify a unique
+	// int64 directly.
+	// If two processes start a leader election using the same MemberID, one of them will
+	// fail.
+	MemberID string
+
+	// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
+	// from string to int using hashing.
+	MemberUniqueID *int64
+
+	// LeaseDurationSeconds is the duration that non-leader candidates will
+	// wait to force acquire leadership.
+	// This is just a request to the etcd server but it's not guaranteed, the server
+	// might decide to make the duration longer.
+	LeaseDurationSeconds int64
+
+	// Callbacks are callbacks that are triggered during certain lifecycle
+	// events of the LeaderElector
+	Callbacks LeaderCallbacks
+}
+
+// LeaderCallbacks are callbacks that are triggered during certain
+// lifecycle events of the election.
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+// ClientConfig contains the client to connect to the etcd cluster.
+type ClientConfig struct {
+	Client *clientv3.Client
+}
+
+// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
+func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
+	if err := RunElection(ctx, config); err != nil {
+		panic(err)
+	}
+}
+
+// RunElection starts a client with the provided config or panics.
+// RunElection blocks until leader election loop is
+// stopped by ctx or it has stopped holding the leader lease.
+func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
+	var memberID int64
+	if config.MemberUniqueID != nil {
+		memberID = *config.MemberUniqueID
+	} else {
+		h := fnv.New64a()
+		if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
+			return err
+		}
+		memberID = int64(h.Sum64())
+	}
+
+	ttl := config.LeaseDurationSeconds
+	r := &pb.LeaseGrantRequest{TTL: ttl, ID: memberID}
+	lease, err := clientv3.RetryLeaseClient(
+		config.EtcdConfig.Client,
+	).LeaseGrant(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "creating lease")
+	}
+
+	leaseID := clientv3.LeaseID(lease.ID)
+
+	s, err := concurrency.NewSession(
+		config.EtcdConfig.Client,
+		concurrency.WithTTL(int(lease.TTL)),
+		concurrency.WithLease(leaseID),
+	)
+	if err != nil {
+		return err
+	}
+
+	election := concurrency.NewElection(s, config.Name)
+
+	m := &member{
+		client:         config.EtcdConfig.Client,
+		election:       election,
+		callbacks:      config.Callbacks,
+		memberID:       config.MemberID,
+		weAreTheLeader: make(chan struct{}, 1),
+		leaseTTL:       lease.TTL,
+	}
+
+	go m.tryToBeLeader(ctx)
+	m.watchLeaderChanges(ctx)
+
+	return nil
+}
+
+type member struct {
+	key              string
+	client           *clientv3.Client
+	election         *concurrency.Election
+	isLeader         bool
+	currentLeaderKey string
+	callbacks        LeaderCallbacks
+	memberID         string
+	weAreTheLeader   chan struct{}
+	leaseTTL         int64
+}
+
+func (m *member) watchLeaderChanges(ctx context.Context) {
+	observeCtx, observeCancel := context.WithCancel(ctx)
+	defer observeCancel()
+	changes := m.election.Observe(observeCtx)
+
+watcher:
+	for {
+		select {
+		case <-ctx.Done():
+			break watcher
+		case <-m.weAreTheLeader:
+
+			m.isLeader = true
+			m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
+			log.Debugf("[%s] Marking self as leader with key %s\n", m.memberID, m.key)
+		case response := <-changes:
+			log.Debugf("[%s] Leader Changes: %+v\n", m.memberID, response)
+			if len(response.Kvs) == 0 {
+				// There is a race condition where just after we stop being the leader
+				// if there are no more leaders, we might get a response with no key-values
+				// just before the response channel is closed or the context is cancel
+				// In that case, just continue and let one of those two things happen
+				continue
+			}
+			newLeaderKey := response.Kvs[0].Key
+			if m.isLeader && m.key != string(newLeaderKey) {
+				// We stopped being leaders
+
+				// exit the loop, so we cancel the observe context so we stop watching
+				// for new leaders. That will close the channel and make this function exit,
+				// which also makes the routine to finish and RunElection returns
+				break watcher
+			}
+
+			if m.currentLeaderKey != string(newLeaderKey) {
+				// we observed a leader, this could be us or someone else
+				m.currentLeaderKey = string(newLeaderKey)
+				m.callbacks.OnNewLeader(string(response.Kvs[0].Value))
+			}
+		}
+	}
+
+	// If we are here, either we have stopped being leaders or we lost the watcher
+	// Make sure we call OnStoppedLeading if we were the leader.
+	if m.isLeader {
+		m.callbacks.OnStoppedLeading()
+	}
+
+	log.Debugf("[%s] Exiting watcher\n", m.memberID)
+}
+
+func (m *member) tryToBeLeader(ctx context.Context) {
+	if err := m.election.Campaign(ctx, m.memberID); err != nil {
+		log.Errorf("Failed trying to become the leader: %s", err)
+		// Resign just in case we acquired leadership just before failing
+		if err := m.election.Resign(m.client.Ctx()); err != nil {
+			log.Warnf("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader: %s", err)
+		}
+		return
+		// TODO: what to do here?
+		// We probably want watchLeaderChanges to exit as well, since Run
+		// is expecting us to try to become the leader, but if we are here,
+		// we won't. So if we don't panic, we need to signal it somehow
+	}
+
+	// Inform the observer that we are the leader as soon as possible,
+	// so it can detect if we stop being it
+	m.weAreTheLeader <- struct{}{}
+
+	// Once we are the leader, start the routine to resign if context is canceled
+	go m.resignOnCancel(ctx)
+
+	// After becoming the leader, we wait for at least a lease TTL to wait for
+	// the previous leader to detect the new leadership (if there was one) and
+	// stop its processes
+	// TODO: is this too cautious?
+	log.Debugf("[%s] Waiting %d seconds before running OnStartedLeading", m.memberID, m.leaseTTL)
+	time.Sleep(time.Second * time.Duration(m.leaseTTL))
+
+	// We are the leader, execute our code
+	m.callbacks.OnStartedLeading(ctx)
+
+	// Here the routine dies if OnStartedLeading doesn't block, there is nothing else to do
+}
+
+func (m *member) resignOnCancel(ctx context.Context) {
+	<-ctx.Done()
+	if err := m.election.Resign(m.client.Ctx()); err != nil {
+		log.Errorf("Failed to resign after the context was canceled: %s", err)
+	}
+}
--- a/pkg/etcd/election_test.go
+++ b/pkg/etcd/election_test.go
@@ -0,0 +1,169 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	. "github.com/onsi/gomega"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+func TestRunElectionWithMemberIDCollision(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	electionName := randomElectionNameForTest("memberIDConflict")
+	log.Printf("Election name %s\n", electionName)
+	memberCtx, cancelMember1 := context.WithCancel(ctx)
+	config := &etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		MemberID:             "my-host",
+		LeaseDurationSeconds: 1,
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				log.Println("I'm the leader!!!!")
+				log.Println("Renouncing as leader by canceling context")
+				cancelMember1()
+			},
+			OnNewLeader: func(identity string) {
+				log.Printf("New leader: %s\n", identity)
+			},
+			OnStoppedLeading: func() {
+				log.Println("I'm not the leader anymore")
+			},
+		},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(memberCtx, config)).To(Succeed())
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
+		g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
+	}()
+
+	wg.Wait()
+}
+
+func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	cliMember1 := client(g)
+	defer cliMember1.Close()
+
+	electionName := randomElectionNameForTest("steppingDown")
+	configBase := etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		LeaseDurationSeconds: 1,
+	}
+
+	member1Ctx, _ := context.WithCancel(ctx)
+	member2Ctx, cancelMember2 := context.WithCancel(ctx)
+
+	config1 := configBase
+	config1.EtcdConfig.Client = cliMember1
+	config1.MemberID = "my-host"
+	uniqueID := rand.Int63()
+	config1.MemberUniqueID = &uniqueID
+	config1.Callbacks = baseCallbacksForName(config1.MemberID)
+	config1.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-host, the new leader!!!!")
+		log.Println("Loosing the leadership on purpose by stopping renewing the lease")
+		g.Expect(cliMember1.Lease.Close()).To(Succeed())
+		log.Println("Member1 leases closed")
+	}
+
+	config2 := configBase
+	config2.MemberID = "my-other-host"
+	config2.Callbacks = baseCallbacksForName(config2.MemberID)
+	config2.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-other-host, the new leader!!!!")
+		log.Println("Renouncing as leader by canceling context")
+		cancelMember2()
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
+		log.Println("Member1 routine done")
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
+		g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
+		log.Println("Member2 routine done")
+	}()
+
+	wg.Wait()
+}
+
+func baseCallbacksForName(name string) etcd.LeaderCallbacks {
+	return etcd.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			log.Printf("[%s] I'm the new leader!!!!\n", name)
+		},
+		OnNewLeader: func(identity string) {
+			log.Printf("[%s] New leader: %s\n", name, identity)
+		},
+		OnStoppedLeading: func() {
+			log.Printf("[%s] I'm not the leader anymore\n", name)
+		},
+	}
+}
+
+func randomElectionNameForTest(name string) string {
+	return name + "-" + randomString(6)
+}
+
+const charSet = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func randomString(n int) string {
+	result := make([]byte, n)
+	for i := range result {
+		result[i] = charSet[rnd.Intn(len(charSet))]
+	}
+	return string(result)
+}
+
+func client(g Gomega) *clientv3.Client {
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{"localhost:2379"},
+		Logger:    zap.NewNop(),
+	})
+	g.Expect(err).NotTo(HaveOccurred())
+	return c
+}
--- a/pkg/etcd/etcd_suite_test.go
+++ b/pkg/etcd/etcd_suite_test.go
@@ -0,0 +1,153 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	etcdVersion     = "v3.5.9"
+	etcdBinDir      = "bin"
+	etcdBinPath     = etcdBinDir + "/etcd"
+	downloadURL     = "https://storage.googleapis.com/etcd"
+	tmpDownloadFile = "etcd.tar.gz"
+	pidFile         = "etcd.pid"
+)
+
+func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.DebugLevel)
+	ctx := context.Background()
+	expectSuccess(startEtcd(ctx), "starting etcd")
+
+	os.Exit(runTestsWithCleanup(m, func() {
+		expectSuccess(stopEtcd(), "stopping etcd")
+	}))
+}
+
+func runTestsWithCleanup(m *testing.M, cleanup func()) int {
+	defer cleanup()
+	return m.Run()
+}
+
+func expectSuccess(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s: %s\n", msg, err)
+	}
+}
+
+func startEtcd(ctx context.Context) error {
+	if _, err := os.Stat(pidFile); err == nil {
+		log.Println("Etcd already running, reusing")
+		return nil
+	}
+
+	etcdPath, err := installEtcd(ctx)
+	if err != nil {
+		errors.Wrap(err, "installing etcd for tests")
+	}
+
+	etcdCmd := exec.Command(etcdPath, "--data-dir", "./etcd-data")
+	if os.Getenv("ETCD_SERVER_LOGS") == "true" {
+		log.Println("Enabling etcd server logs")
+		etcdCmd.Stdout = os.Stdout
+		etcdCmd.Stderr = os.Stderr
+	}
+	log.Println("Starting etcd")
+	if err := etcdCmd.Start(); err != nil {
+		errors.Wrap(err, "starting etcd for tests")
+	}
+
+	if err := os.WriteFile(pidFile, []byte(strconv.Itoa(etcdCmd.Process.Pid)), 0o600); err != nil {
+		return err
+	}
+
+	log.Println("Waiting for etcd to be up")
+	time.Sleep(time.Second)
+
+	return nil
+}
+
+func installEtcd(ctx context.Context) (string, error) {
+	projectRoot, err := filepath.Abs("../../")
+	if err != nil {
+		return "", err
+	}
+	binDir := filepath.Join(projectRoot, etcdBinDir)
+	etcdPath := filepath.Join(projectRoot, etcdBinPath)
+
+	if _, err := os.Stat(etcdPath); err == nil {
+		log.Println("Etcd already installed, skipping")
+		return etcdPath, nil
+	}
+
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", err
+	}
+
+	download := fmt.Sprintf("%s/%s/etcd-%s-linux-amd64.tar.gz", downloadURL, etcdVersion, etcdVersion)
+
+	// Hacky to run bash, but simplifies this code a lot
+	cmd := fmt.Sprintf("curl -sL %s | tar -xzvf - -C %s --strip-components=1", download, binDir)
+	out, err := exec.CommandContext(ctx, "bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return "", errors.Wrapf(err, "downloading etcd: %s", string(out))
+	}
+
+	return etcdPath, nil
+}
+
+func stopEtcd() error {
+	if os.Getenv("REUSE_ETCD") == "true" {
+		log.Println("REUSE_ETCD=true, leaving etcd running")
+		return nil
+	}
+
+	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+		log.Println("Etcd pid file doesn't exit, skipping cleanup")
+		return nil
+	}
+
+	dat, err := os.ReadFile(pidFile)
+	if err != nil {
+		return err
+	}
+	pid, err := strconv.Atoi(string(dat))
+	if err != nil {
+		return err
+	}
+
+	etcdProcess, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Stopping etcd")
+	if err := etcdProcess.Kill(); err != nil {
+		return errors.Wrap(err, "Failed stopping etcd")
+	}
+
+	log.Println("Deleting etcd data")
+	if err := os.RemoveAll("./etcd-data"); err != nil {
+		return errors.Wrap(err, "deleting etcd data")
+	}
+
+	log.Println("Deleting etcd pid file")
+	if err := os.RemoveAll(pidFile); err != nil {
+		return errors.Wrap(err, "deleting pid file")
+	}
+
+	return nil
+}
--- a/pkg/kubevip/config_types.go
+++ b/pkg/kubevip/config_types.go
@@ -50,8 +50,15 @@ type Config struct {
 	// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
 	Annotations string

-	// LeaderElection defines the settings around Kubernetes LeaderElection
-	LeaderElection
+	// LeaderElectionType defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.
+	// Etcd doesn't support load balancer mode (EnableLoadBalancer=true) or any other feature that depends on the kube-api server.
+	LeaderElectionType string `yaml:"leaderElectionType"`
+
+	// KubernetesLeaderElection defines the settings around Kubernetes KubernetesLeaderElection
+	KubernetesLeaderElection
+
+	// Etcd defines all the settings for the etcd client.
+	Etcd Etcd

 	// AddPeersAsBackends, this will automatically add RAFT peers as backends to a loadbalancer
 	AddPeersAsBackends bool `yaml:"addPeersAsBackends"`
@@ -145,8 +152,8 @@ type Config struct {
 	ServicesLeaseName string `yaml:"servicesLeaseName"`
 }

-// LeaderElection defines all of the settings for Kubernetes LeaderElection
-type LeaderElection struct {
+// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection
+type KubernetesLeaderElection struct {
 	// EnableLeaderElection will use the Kubernetes leader election algorithm
 	EnableLeaderElection bool `yaml:"enableLeaderElection"`

@@ -166,6 +173,14 @@ type LeaderElection struct {
 	LeaseAnnotations map[string]string
 }

+// Etcd defines all the settings for the etcd client.
+type Etcd struct {
+	CAFile         string
+	ClientCertFile string
+	ClientKeyFile  string
+	Endpoints      []string
+}
+
 // LoadBalancer contains the configuration of a load balancing instance
 type LoadBalancer struct {
 	// Name of a LoadBalancer
--- a/pkg/manager/cluster.go
+++ b/pkg/manager/cluster.go
@@ -0,0 +1,29 @@
+package manager
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+)
+
+func initClusterManager(sm *Manager) (*cluster.Manager, error) {
+	m := &cluster.Manager{
+		SignalChan: sm.signalChan,
+	}
+
+	switch sm.config.LeaderElectionType {
+	case "kubernetes", "":
+		m.KubernetesClient = sm.clientSet
+	case "etcd":
+		client, err := etcd.NewClient(sm.config)
+		if err != nil {
+			return nil, err
+		}
+		m.EtcdClient = client
+	default:
+		return nil, errors.Errorf("invalid LeaderElectionMode %s not supported", sm.config.LeaderElectionType)
+	}
+
+	return m, nil
+}
--- a/pkg/manager/manager.go
+++ b/pkg/manager/manager.go
@@ -67,6 +67,8 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
 	homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")

 	switch {
+	case config.LeaderElectionType == "etcd":
+		// Do nothing, we don't construct a k8s client for etcd leader election
 	case fileExists(adminConfigPath):
 		if config.EnableControlPlane {
 			// If this is a control plane host it will likely have started as a static pod or won't have the
--- a/pkg/manager/manager_arp.go
+++ b/pkg/manager/manager_arp.go
@@ -47,9 +47,9 @@ func (sm *Manager) startARP() error {
 			return err
 		}

-		clusterManager := &cluster.Manager{
-			KubernetesClient: sm.clientSet,
-			SignalChan:       sm.signalChan,
+		clusterManager, err := initClusterManager(sm)
+		if err != nil {
+			return err
 		}

 		go func() {
--- a/pkg/manager/manager_bgp.go
+++ b/pkg/manager/manager_bgp.go
@@ -103,9 +103,9 @@ func (sm *Manager) startBGP() error {
 			return err
 		}

-		clusterManager := &cluster.Manager{
-			KubernetesClient: sm.clientSet,
-			SignalChan:       sm.signalChan,
+		clusterManager, err := initClusterManager(sm)
+		if err != nil {
+			return err
 		}

 		go func() {
--- a/testing/e2e/e2e_test.go
+++ b/testing/e2e/e2e_test.go
@@ -4,14 +4,10 @@
 package e2e_test

 import (
-	"bufio"
 	"bytes"
 	"crypto/tls"
-	"encoding/binary"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -24,20 +20,15 @@ import (
 	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster"
-	"sigs.k8s.io/kind/pkg/cmd"
-	load "sigs.k8s.io/kind/pkg/cmd/kind/load/docker-image"
 	"sigs.k8s.io/kind/pkg/log"

 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/format"
 	"github.com/onsi/gomega/gexec"
-)

-type kubevipManifestValues struct {
-	ControlPlaneVIP string
-	ImagePath       string
-}
+	"github.com/kube-vip/kube-vip/testing/e2e"
+)

 var _ = Describe("kube-vip broadcast neighbor", func() {
 	var (
@@ -50,7 +41,7 @@ var _ = Describe("kube-vip broadcast neighbor", func() {

 	BeforeEach(func() {
 		klog.SetOutput(GinkgoWriter)
-		logger = TestLogger{}
+		logger = e2e.TestLogger{}

 		imagePath = os.Getenv("E2E_IMAGE_PATH")

@@ -115,9 +106,9 @@ var _ = Describe("kube-vip broadcast neighbor", func() {

 			defer manifestFile.Close()

-			ipv4VIP = generateIPv4VIP()
+			ipv4VIP = e2e.GenerateIPv4VIP()

-			Expect(kubeVIPManifestTemplate.Execute(manifestFile, kubevipManifestValues{
+			Expect(kubeVIPManifestTemplate.Execute(manifestFile, e2e.KubevipManifestValues{
 				ControlPlaneVIP: ipv4VIP,
 				ImagePath:       imagePath,
 			})).To(Succeed())
@@ -128,7 +119,7 @@ var _ = Describe("kube-vip broadcast neighbor", func() {
 			createKindCluster(logger, &clusterConfig, clusterName)

 			By(withTimestamp("loading local docker image to kind cluster"))
-			loadDockerImageToKind(logger, imagePath, clusterName)
+			e2e.LoadDockerImageToKind(logger, imagePath, clusterName)

 			By(withTimestamp("checking that the Kubernetes control plane nodes are accessible via the assigned IPv4 VIP"))
 			// Allow enough time for control plane nodes to load the docker image and
@@ -173,14 +164,14 @@ var _ = Describe("kube-vip broadcast neighbor", func() {
 				})
 			}

-			ipv6VIP = generateIPv6VIP()
+			ipv6VIP = e2e.GenerateIPv6VIP()

 			manifestFile, err := os.Create(manifestPath)
 			Expect(err).NotTo(HaveOccurred())

 			defer manifestFile.Close()

-			Expect(kubeVIPManifestTemplate.Execute(manifestFile, kubevipManifestValues{
+			Expect(kubeVIPManifestTemplate.Execute(manifestFile, e2e.KubevipManifestValues{
 				ControlPlaneVIP: ipv6VIP,
 				ImagePath:       imagePath,
 			})).To(Succeed())
@@ -191,7 +182,7 @@ var _ = Describe("kube-vip broadcast neighbor", func() {
 			createKindCluster(logger, &clusterConfig, clusterName)

 			By(withTimestamp("loading local docker image to kind cluster"))
-			loadDockerImageToKind(logger, imagePath, clusterName)
+			e2e.LoadDockerImageToKind(logger, imagePath, clusterName)

 			By(withTimestamp("checking that the Kubernetes control plane nodes are accessible via the assigned IPv6 VIP"))
 			// Allow enough time for control plane nodes to load the docker image and
@@ -221,12 +212,6 @@ func createKindCluster(logger log.Logger, config *v1alpha4.Cluster, clusterName
 	)).To(Succeed())
 }

-func loadDockerImageToKind(logger log.Logger, imagePath string, clusterName string) {
-	loadImageCmd := load.NewCommand(logger, cmd.StandardIOStreams())
-	loadImageCmd.SetArgs([]string{"--name", clusterName, imagePath})
-	Expect(loadImageCmd.Execute()).To(Succeed())
-}
-
 func assertControlPlaneIsRoutable(controlPlaneVIP string, transportTimeout, eventuallyTimeout time.Duration) {
 	if strings.Contains(controlPlaneVIP, ":") {
 		controlPlaneVIP = fmt.Sprintf("[%s]", controlPlaneVIP)
@@ -281,104 +266,3 @@ func killLeader(leaderIPAddr string, clusterName string) {
 func withTimestamp(text string) string {
 	return fmt.Sprintf("%s: %s", time.Now(), text)
 }
-
-func getKindNetworkSubnetCIDRs() []string {
-	cmd := exec.Command(
-		"docker", "inspect", "kind",
-		"--format", `{{ range $i, $a := .IPAM.Config }}{{ println .Subnet }}{{ end }}`,
-	)
-	cmdOut := new(bytes.Buffer)
-	cmd.Stdout = cmdOut
-	Expect(cmd.Run()).To(Succeed(), "The Docker \"kind\" network was not found.")
-	reader := bufio.NewReader(cmdOut)
-
-	cidrs := []string{}
-	for {
-		line, readErr := reader.ReadString('\n')
-		if readErr != nil && readErr != io.EOF {
-			Expect(readErr).NotTo(HaveOccurred(), "Error finding subnet CIDRs in the Docker \"kind\" network")
-		}
-
-		cidrs = append(cidrs, strings.TrimSpace(line))
-		if readErr == io.EOF {
-			break
-		}
-	}
-
-	return cidrs
-}
-
-func generateIPv4VIP() string {
-	cidrs := getKindNetworkSubnetCIDRs()
-
-	for _, cidr := range cidrs {
-		ip, ipNet, parseErr := net.ParseCIDR(cidr)
-		Expect(parseErr).NotTo(HaveOccurred())
-
-		if ip.To4() != nil {
-			mask := binary.BigEndian.Uint32(ipNet.Mask)
-			start := binary.BigEndian.Uint32(ipNet.IP)
-			end := (start & mask) | (^mask)
-
-			chosenVIP := make([]byte, 4)
-			binary.BigEndian.PutUint32(chosenVIP, end-5)
-			return net.IP(chosenVIP).String()
-		}
-	}
-	Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
-	return ""
-}
-
-func generateIPv6VIP() string {
-	cidrs := getKindNetworkSubnetCIDRs()
-
-	for _, cidr := range cidrs {
-		ip, ipNet, parseErr := net.ParseCIDR(cidr)
-		Expect(parseErr).NotTo(HaveOccurred())
-
-		if ip.To4() == nil {
-			lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
-			lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
-			lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
-
-			chosenVIP := make([]byte, 16)
-			// Copy upper half into chosenVIP
-			copy(chosenVIP, ipNet.IP[0:8])
-			// Copy lower half into chosenVIP
-			binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
-			return net.IP(chosenVIP).String()
-		}
-	}
-	Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
-	return ""
-}
-
-type TestLogger struct{}
-
-func (t TestLogger) Warnf(format string, args ...interface{}) {
-	klog.Warningf(format, args...)
-}
-
-func (t TestLogger) Warn(message string) {
-	klog.Warning(message)
-}
-
-func (t TestLogger) Error(message string) {
-	klog.Error(message)
-}
-
-func (t TestLogger) Errorf(format string, args ...interface{}) {
-	klog.Errorf(format, args...)
-}
-
-func (t TestLogger) V(level log.Level) log.InfoLogger {
-	return TestInfoLogger{Verbose: klog.V(klog.Level(level))}
-}
-
-type TestInfoLogger struct {
-	klog.Verbose
-}
-
-func (t TestInfoLogger) Info(message string) {
-	t.Verbose.Info(message)
-}
--- a/testing/e2e/etcd/README.md
+++ b/testing/e2e/etcd/README.md
@@ -0,0 +1,32 @@
+# Running etcd Tests
+## Prerequisites:
+* Docker
+
+If you want to use an image that only exists in your local docker cache, use this env var (modify registry and tag accordingly):
+```sh
+export E2E_IMAGE_PATH=plndr/kube-vip:v0.6.2
+```
+
+If you want to preserve the etcd nodes after a test run, use the following:
+```sh
+export E2E_PRESERVE_CLUSTER=true
+```
+
+Note that you'll need to delete them before being able to run the test again, this is only for debugging. You can use `kind delete cluster` or just `docker rm` the containers.
+
+Tu run the tests:
+```sh
+ginkgo -vv --tags=e2e testing/e2e/etcd
+
+```
+
+The E2E tests:
+1. Start 3 kind nodes (using docker)
+2. Load the local docker image into kind 
+3. Init the etcd cluster and join all nodes
+4. Verify the etcd API can be accessed through the VIP
+   1. This proves leader election through etcd in kube-vip is working.
+5. Removes the first node (which is probably the VIP leader)
+4. Verify the etcd API can be accessed through the VIP
+
+> Note: this has only been tested on Linux but it might work on Mac
--- a/testing/e2e/etcd/cluster.go
+++ b/testing/e2e/etcd/cluster.go
@@ -0,0 +1,341 @@
+//go:build e2e
+// +build e2e
+
+package etcd
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"golang.org/x/exp/slices"
+	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
+	"sigs.k8s.io/kind/pkg/cluster"
+	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
+
+	"github.com/kube-vip/kube-vip/testing/e2e"
+)
+
+type ClusterSpec struct {
+	Nodes                int
+	Name                 string
+	VIP                  string
+	KubeVIPImage         string
+	KubeVIPpManifestPath string
+	KubeletManifestPath  string
+	KubeletFlagsPath     string
+	EtcdCertsFolder      string
+	Logger               e2e.TestLogger
+}
+
+type Cluster struct {
+	*ClusterSpec
+	Nodes []nodes.Node
+
+	provider *cluster.Provider
+}
+
+func CreateCluster(ctx context.Context, spec *ClusterSpec) *Cluster {
+	c := &Cluster{
+		ClusterSpec: spec,
+	}
+
+	c.provider = cluster.NewProvider(
+		cluster.ProviderWithLogger(spec.Logger),
+		cluster.ProviderWithDocker(),
+	)
+
+	c.Logger.Printf("Creating kind nodes")
+	c.initKindCluster()
+
+	c.Logger.Printf("Loading kube-vip image into nodes")
+	e2e.LoadDockerImageToKind(spec.Logger, spec.KubeVIPImage, spec.Name)
+
+	c.Logger.Printf("Starting etcd cluster")
+	c.initEtcd(ctx)
+
+	c.Logger.Printf("Checking 1 node etcd is available through VIP")
+	c.VerifyEtcdThroughVIP(ctx, 15*time.Second)
+
+	c.Logger.Printf("Adding the rest of the nodes to the etcd cluster")
+	c.joinRestOfNodes(ctx)
+
+	c.Logger.Printf("Checking health for all nodes")
+	for _, node := range c.Nodes {
+		c.expectEtcdNodeHealthy(ctx, node, 15*time.Second)
+	}
+
+	c.Logger.Printf("Checking %d nodes etcd is available through VIP", c.ClusterSpec.Nodes)
+	c.VerifyEtcdThroughVIP(ctx, 15*time.Second)
+
+	return c
+}
+
+func (c *Cluster) initKindCluster() {
+	kindCluster := &kindconfigv1alpha4.Cluster{
+		Networking: kindconfigv1alpha4.Networking{
+			IPFamily: kindconfigv1alpha4.IPv4Family,
+		},
+	}
+
+	for i := 0; i < c.ClusterSpec.Nodes; i++ {
+		kindCluster.Nodes = append(kindCluster.Nodes, kindconfigv1alpha4.Node{
+			Role: kindconfigv1alpha4.ControlPlaneRole,
+			ExtraMounts: []kindconfigv1alpha4.Mount{
+				{
+					HostPath:      c.ClusterSpec.KubeVIPpManifestPath,
+					ContainerPath: "/etc/kubernetes/manifests/kube-vip.yaml",
+				},
+				{
+					HostPath:      c.ClusterSpec.KubeletManifestPath,
+					ContainerPath: "/var/lib/kubelet/config.yaml",
+				},
+				{
+					HostPath:      c.ClusterSpec.KubeletFlagsPath,
+					ContainerPath: "/etc/default/kubelet",
+				},
+			},
+		})
+	}
+
+	Expect(c.provider.Create(
+		c.Name,
+		cluster.CreateWithV1Alpha4Config(kindCluster),
+		cluster.CreateWithRetain(true),
+		cluster.CreateWithStopBeforeSettingUpKubernetes(true),
+		cluster.CreateWithWaitForReady(2*time.Minute),
+		cluster.CreateWithNodeImage("public.ecr.aws/eks-anywhere/kubernetes-sigs/kind/node:v1.26.7-eks-d-1-26-16-eks-a-47"),
+	)).To(Succeed())
+}
+
+func (c *Cluster) initEtcd(ctx context.Context) {
+	var err error
+	c.Nodes, err = c.provider.ListInternalNodes(c.Name)
+	slices.SortFunc(c.Nodes, func(a, b nodes.Node) int {
+		aName := a.String()
+		bName := b.String()
+		if aName < bName {
+			return 1
+		} else if aName > bName {
+			return -1
+		}
+
+		return 0
+	})
+
+	Expect(err).NotTo(HaveOccurred())
+	firstNode := c.Nodes[0]
+
+	createCerts(firstNode)
+
+	// We need to run all phases individually to be able to re-run the health phase
+	// In CI it can take longer than the 30 seconds timeout that is hardcoded in etcdadm
+	// If etcdadm added the option to configure this timeout, we could change this to just
+	// call etcdadm init to run all phases.
+
+	flags := []string{
+		"--init-system", "kubelet",
+		"--certs-dir", "/etc/kubernetes/pki/etcd",
+		"--server-cert-extra-sans", strings.Join([]string{"etcd", c.VIP, e2e.NodeIPv4(firstNode)}, ","),
+		"--version", "3.5.8-eks-1-26-16",
+		"--image-repository", "public.ecr.aws/eks-distro/etcd-io/etcd",
+	}
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "install", flags)...,
+	)
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "certificates", flags)...,
+	)
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "snapshot", flags)...,
+	)
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "configure", flags)...,
+	)
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "start", flags)...,
+	)
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "etcdctl", flags)...,
+	)
+
+	Eventually(func() error {
+		return runInNode(firstNode,
+			"etcdadm",
+			initArgsForPhase("init", "health", flags)...,
+		)
+	}, 3*time.Minute).Should(Succeed(), "etcd should become healthy in node in less than 3 minutes")
+
+	e2e.RunInNode(firstNode,
+		"etcdadm",
+		initArgsForPhase("init", "post-init-instructions", flags)...,
+	)
+
+	bindEtcdListenerToAllIPs(firstNode)
+
+	e2e.CopyFolderFromNodeToDisk(firstNode, "/etc/kubernetes/pki/etcd", c.EtcdCertsFolder)
+
+	c.expectEtcdNodeHealthy(ctx, firstNode, 15*time.Second)
+}
+
+func runInNode(node nodes.Node, command string, args ...string) error {
+	return e2e.PrintCommandOutputIfErr(node.Command(command, args...).Run())
+}
+
+func initArgsForPhase(command, phaseName string, flags []string) []string {
+	c := make([]string, 0, 3+len(flags))
+	c = append(c, command, "phase", phaseName)
+	c = append(c, flags...)
+	return c
+}
+
+func (c *Cluster) joinRestOfNodes(ctx context.Context) {
+	for _, node := range c.Nodes[1:] {
+		c.joinNode(ctx, c.Nodes[0], node)
+	}
+}
+
+func (c *Cluster) joinNode(ctx context.Context, firstNode, node nodes.Node) {
+	nodeutils.CopyNodeToNode(firstNode, node, "/etc/kubernetes/pki/ca.crt")
+	nodeutils.CopyNodeToNode(firstNode, node, "/etc/kubernetes/pki/ca.key")
+	nodeutils.CopyNodeToNode(firstNode, node, "/etc/kubernetes/pki/etcd/ca.crt")
+	nodeutils.CopyNodeToNode(firstNode, node, "/etc/kubernetes/pki/etcd/ca.key")
+
+	e2e.RunInNode(node,
+		"etcdadm",
+		"join",
+		"https://"+e2e.NodeIPv4(firstNode)+":2379",
+		"--init-system", "kubelet",
+		"--certs-dir", "/etc/kubernetes/pki/etcd",
+		"--server-cert-extra-sans", strings.Join([]string{"etcd", c.VIP, e2e.NodeIPv4(node)}, ","),
+		"--version", "3.5.8-eks-1-26-16",
+		"--image-repository", "public.ecr.aws/eks-distro/etcd-io/etcd",
+	)
+
+	bindEtcdListenerToAllIPs(node)
+
+	c.expectEtcdNodeHealthy(ctx, node, 30*time.Second)
+}
+
+func (c *Cluster) DeleteEtcdMember(ctx context.Context, toDelete, toKeep nodes.Node) {
+	// point client to the node we are keeping because we are going to use it to remove the other node
+	// and vip is possibly pointing to that node
+	client := c.newEtcdClient(e2e.NodeIPv4(toDelete))
+	defer client.Close()
+	members, err := client.MemberList(ctx)
+	Expect(err).NotTo(HaveOccurred())
+	c.Logger.Printf("Members: %v", members.Members)
+
+	nodeName := toDelete.String()
+	for _, m := range members.Members {
+		if m.Name == nodeName {
+			c.Logger.Printf("Removing node %s with memberID %d", m.Name, m.ID)
+			// We need to retry this request because etcd will reject it if the
+			// server doesn't have recent connections to enough active members
+			// to protect the quorum. (active - 1) >= 1+((members-1)/2)
+			Eventually(func() error {
+				_, err := client.MemberRemove(ctx, m.ID)
+				return err
+			}).WithPolling(time.Second).WithTimeout(10*time.Second).Should(
+				Succeed(), "removing member should succeed once all members have connections to each other",
+			)
+
+			break
+		}
+	}
+
+	e2e.DeleteNodes(toDelete)
+}
+
+func (c *Cluster) Delete() {
+	Expect(c.provider.Delete(c.Name, "")).To(Succeed())
+}
+
+func startKubeletForEtcd(node nodes.Node) {
+	e2e.RunInNode(node,
+		"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
+	)
+	e2e.RunInNode(node,
+		"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
+	)
+}
+
+func createCerts(node nodes.Node) {
+	e2e.RunInNode(node,
+		"kubeadm",
+		"init",
+		"phase", "certs", "ca", "--config", "/kind/kubeadm.conf",
+	)
+
+	e2e.RunInNode(node,
+		"kubeadm",
+		"init",
+		"phase", "certs", "etcd-ca", "--config", "/kind/kubeadm.conf",
+	)
+}
+
+func bindEtcdListenerToAllIPs(node nodes.Node) {
+	// There is no easy way to make etcdadm configure etcd to bind to 0.0.0.0
+	// so we just manually update the manifest after it's created and restart it
+	// We want to listen in 0.0.0.0 so our kube-vip can connect to it.
+	e2e.RunInNode(node,
+		"sed", "-i", `s/https:\/\/.*,https:\/\/127.0.0.1:2379/https:\/\/0.0.0.0:2379/g`, "/etc/kubernetes/manifests/etcd.manifest",
+	)
+
+	e2e.StopPodInNode(node, "etcd")
+
+	e2e.RunInNode(node,
+		"systemctl", "restart", "kubelet",
+	)
+}
+
+func (c *Cluster) newEtcdClient(serverIPs ...string) *clientv3.Client {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: filepath.Join(c.EtcdCertsFolder, "ca.crt"),
+		CertFile:      filepath.Join(c.EtcdCertsFolder, "etcdctl-etcd-client.crt"),
+		KeyFile:       filepath.Join(c.EtcdCertsFolder, "etcdctl-etcd-client.key"),
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	Expect(err).NotTo(HaveOccurred())
+
+	endpoints := make([]string, 0, len(serverIPs))
+	for _, ip := range serverIPs {
+		endpoints = append(endpoints, ip+":2379")
+	}
+
+	client, err := clientv3.New(clientv3.Config{
+		Endpoints:   endpoints,
+		TLS:         clientTLS,
+		DialTimeout: time.Second,
+	})
+	Expect(err).NotTo(HaveOccurred())
+
+	return client
+}
+
+func (c *Cluster) VerifyEtcdThroughVIP(ctx context.Context, timeout time.Duration) {
+	etcdClient := c.newEtcdClient(c.VIP)
+	defer etcdClient.Close()
+	rCtx, cancel := context.WithTimeout(ctx, timeout)
+	_, err := etcdClient.MemberList(rCtx)
+	Expect(err).NotTo(HaveOccurred())
+	cancel()
+}
--- a/testing/e2e/etcd/election_test.go
+++ b/testing/e2e/etcd/election_test.go
@@ -0,0 +1,114 @@
+//go:build e2e
+// +build e2e
+
+package etcd_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"text/template"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"k8s.io/klog/v2"
+
+	"github.com/kube-vip/kube-vip/testing/e2e"
+	"github.com/kube-vip/kube-vip/testing/e2e/etcd"
+)
+
+type testConfig struct {
+	logger              e2e.TestLogger
+	kubeVipImage        string
+	kubeVipManifestPath string
+	clusterName         string
+	tempDirPath         string
+	vip                 string
+	etcdCertsFolder     string
+	currentDir          string
+	cluster             *etcd.Cluster
+}
+
+func (t *testConfig) cleanup() {
+	if os.Getenv("E2E_PRESERVE_CLUSTER") == "true" {
+		return
+	}
+
+	t.cluster.Delete()
+	Expect(os.RemoveAll(t.tempDirPath)).To(Succeed())
+	Expect(os.RemoveAll(t.etcdCertsFolder)).To(Succeed())
+}
+
+var _ = Describe("kube-vip with etcd leader election", func() {
+	ctx := context.Background()
+	test := &testConfig{}
+
+	AfterEach(func() {
+		test.cleanup()
+	})
+
+	BeforeEach(func() {
+		By("configuring test", func() {
+			var err error
+			format.UseStringerRepresentation = true // Otherwise error stacks have binary format.
+			klog.SetOutput(GinkgoWriter)
+
+			test.clusterName = "kube-vip-etcd-test" // this needs to unique per it block
+			test.logger = e2e.TestLogger{}
+			test.etcdCertsFolder = "certs"
+
+			test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
+
+			test.vip = e2e.GenerateIPv4VIP()
+			test.logger.Printf("Selected VIP %s", test.vip)
+
+			test.currentDir, err = os.Getwd()
+			Expect(err).NotTo(HaveOccurred())
+
+			test.tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
+			Expect(err).NotTo(HaveOccurred())
+
+			test.kubeVipManifestPath = filepath.Join(test.tempDirPath, "etcd-vip-ipv4.yaml")
+			manifestFile, err := os.Create(test.kubeVipManifestPath)
+			Expect(err).NotTo(HaveOccurred())
+			defer manifestFile.Close()
+
+			templatePath := filepath.Join(test.currentDir, "kube-etcd-vip.yaml.tmpl")
+			kubeVIPManifestTemplate, err := template.New("kube-etcd-vip.yaml.tmpl").ParseFiles(templatePath)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(kubeVIPManifestTemplate.Execute(manifestFile, e2e.KubevipManifestValues{
+				ControlPlaneVIP: test.vip,
+				ImagePath:       test.kubeVipImage,
+			})).To(Succeed())
+		})
+
+		By("creating etcd cluster", func() {
+			spec := &etcd.ClusterSpec{
+				Name:                 test.clusterName,
+				Nodes:                2,
+				VIP:                  test.vip,
+				KubeVIPImage:         test.kubeVipImage,
+				KubeVIPpManifestPath: test.kubeVipManifestPath,
+				KubeletManifestPath:  filepath.Join(test.currentDir, "kubelet.yaml"),
+				KubeletFlagsPath:     filepath.Join(test.currentDir, "kubelet-flags.env"),
+				EtcdCertsFolder:      filepath.Join(test.currentDir, test.etcdCertsFolder),
+				Logger:               test.logger,
+			}
+
+			test.cluster = etcd.CreateCluster(ctx, spec)
+		})
+	})
+
+	When("an etcd node is removed", func() {
+		It("elects a new kube-vip leader and provides a VIP to the second node", func() {
+			By("removing as member and killing the first node", func() {
+				test.cluster.DeleteEtcdMember(ctx, test.cluster.Nodes[0], test.cluster.Nodes[1])
+			})
+			By("verifying etcd is up and accessible through the vip", func() {
+				test.cluster.VerifyEtcdThroughVIP(ctx, 40*time.Second)
+			})
+		})
+	})
+})
--- a/testing/e2e/etcd/etcd_suite_test.go
+++ b/testing/e2e/etcd/etcd_suite_test.go
@@ -0,0 +1,16 @@
+//go:build e2e
+// +build e2e
+
+package etcd_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestEtcd(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Etcd Suite")
+}
--- a/testing/e2e/etcd/health.go
+++ b/testing/e2e/etcd/health.go
@@ -0,0 +1,121 @@
+//go:build e2e
+// +build e2e
+
+package etcd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/kube-vip/kube-vip/testing/e2e"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	"sigs.k8s.io/kind/pkg/cluster/nodes"
+)
+
+func (c *Cluster) expectEtcdNodeHealthy(ctx context.Context, node nodes.Node, timeout time.Duration) {
+	httpClient := c.newEtcdHTTPClient()
+	client := c.newEtcdClient(e2e.NodeIPv4(node))
+	nodeEtcdEndpoint := etcdEndpointForNode(node)
+	Eventually(func(g Gomega) error {
+		health, err := getEtcdHealth(httpClient, node)
+		g.Expect(err).NotTo(HaveOccurred())
+		if !health.Healthy() {
+			c.Logger.Printf("Member %s is not healthy with reason: %s", node.String(), health.Reason)
+		}
+		g.Expect(health.Healthy()).To(BeTrue(), "member is not healthy with reason: %s", health.Reason)
+		statusCtx, statusCancel := context.WithTimeout(ctx, 2*time.Second)
+		defer statusCancel()
+		status, err := client.Status(statusCtx, nodeEtcdEndpoint)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		g.Expect(status.Errors).To(BeEmpty(), "member should not have any errors in status")
+		g.Expect(status.IsLearner).To(BeFalse(), "member should not be a learner")
+
+		alarmsCtx, alarmsCancel := context.WithTimeout(ctx, 2*time.Second)
+		defer alarmsCancel()
+		alarms, err := client.AlarmList(alarmsCtx)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(alarms.Alarms).To(BeEmpty(), "cluster should not have any alarms")
+
+		return nil
+	}, timeout).Should(Succeed(), "node %s should eventually be healthy", node.String())
+}
+
+func (c *Cluster) newEtcdHTTPClient() *http.Client {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: filepath.Join(c.EtcdCertsFolder, "ca.crt"),
+		CertFile:      filepath.Join(c.EtcdCertsFolder, "etcdctl-etcd-client.crt"),
+		KeyFile:       filepath.Join(c.EtcdCertsFolder, "etcdctl-etcd-client.key"),
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	Expect(err).NotTo(HaveOccurred())
+
+	return &http.Client{
+		Timeout: 2 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: clientTLS,
+		},
+	}
+}
+
+type etcdHealthCheckResponse struct {
+	Health string `json:"health"`
+	Reason string `json:"reason"`
+}
+
+func (h *etcdHealthCheckResponse) Healthy() bool {
+	return h.Health == "true"
+}
+
+func getEtcdHealth(c *http.Client, node nodes.Node) (*etcdHealthCheckResponse, error) {
+	req, err := http.NewRequest("GET", etcdHealthEndpoint(node), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Wrapf(err, "etcd member not ready, returned http status %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	health, err := parseEtcdHealthResponse(body)
+	if err != nil {
+		return nil, err
+	}
+
+	return health, nil
+}
+
+func etcdEndpointForNode(node nodes.Node) string {
+	return e2e.NodeIPv4(node) + ":2379"
+}
+
+func etcdHealthEndpoint(node nodes.Node) string {
+	return fmt.Sprintf("https://%s:2379/health", e2e.NodeIPv4(node))
+}
+
+func parseEtcdHealthResponse(data []byte) (*etcdHealthCheckResponse, error) {
+	obj := &etcdHealthCheckResponse{}
+	if err := json.Unmarshal(data, obj); err != nil {
+		return nil, err
+	}
+	return obj, nil
+}
--- a/testing/e2e/etcd/kube-etcd-vip.yaml.tmpl
+++ b/testing/e2e/etcd/kube-etcd-vip.yaml.tmpl
@@ -0,0 +1,51 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+  - name: kube-vip
+    args:
+    - manager
+    - --leaderElectionType=etcd
+    - --etcdCACert=/etc/kubernetes/pki/etcd/ca.crt
+    - --etcdCert=/etc/kubernetes/pki/etcd/server.crt
+    - --etcdKey=/etc/kubernetes/pki/etcd/server.key
+    - --etcdEndpoints=127.0.0.1:2379
+    env:
+    - name: vip_arp
+      value: "true"
+    - name: vip_interface
+      value: eth0
+    - name: vip_leaderelection
+      value: "true"
+    - name: address
+      value: "{{ .ControlPlaneVIP }}"
+    - name: vip_leaseduration
+      value: "2"
+    - name: vip_renewdeadline
+      value: "3"
+    - name: vip_retryperiod
+      value: "1"
+    - name: cp_enable
+      value: "true"
+    - name: vip_loglevel
+      value: "5"
+    image: "{{ .ImagePath }}"
+    imagePullPolicy: Never
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+    volumeMounts:
+    # force kube-vip to use CP ip from admin.conf instead of localhost
+    - mountPath: /etc/kubernetes/pki/etcd
+      name: etcd-certs
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd
+    name: etcd-certs
--- a/testing/e2e/etcd/kubelet-flags.env
+++ b/testing/e2e/etcd/kubelet-flags.env
@@ -0,0 +1 @@
+KUBELET_EXTRA_ARGS="--kubeconfig='' --bootstrap-kubeconfig='' --container-runtime-endpoint=unix:///run/containerd/containerd.sock --node-labels= --pod-infra-container-image=registry.k8s.io/pause:3.9"
--- a/testing/e2e/etcd/kubelet.yaml
+++ b/testing/e2e/etcd/kubelet.yaml
@@ -0,0 +1,20 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+authentication:
+  anonymous:
+    enabled: true
+  webhook:
+    enabled: false
+authorization:
+  mode: AlwaysAllow
+enableServer: false
+logging:
+  flushFrequency: 0
+  options:
+    json:
+      infoBufferSize: "0"
+  verbosity: 0
+podCIDR: 10.241.1.0/24
+staticPodPath: /etc/kubernetes/manifests
+cgroupDriver: systemd
+cgroupRoot: /kubelet
--- a/testing/e2e/ip.go
+++ b/testing/e2e/ip.go
@@ -0,0 +1,88 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"io"
+	"net"
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func GenerateIPv6VIP() string {
+	cidrs := getKindNetworkSubnetCIDRs()
+
+	for _, cidr := range cidrs {
+		ip, ipNet, parseErr := net.ParseCIDR(cidr)
+		Expect(parseErr).NotTo(HaveOccurred())
+
+		if ip.To4() == nil {
+			lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
+			lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
+			lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
+
+			chosenVIP := make([]byte, 16)
+			// Copy upper half into chosenVIP
+			copy(chosenVIP, ipNet.IP[0:8])
+			// Copy lower half into chosenVIP
+			binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
+			return net.IP(chosenVIP).String()
+		}
+	}
+	Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
+	return ""
+}
+
+func GenerateIPv4VIP() string {
+	cidrs := getKindNetworkSubnetCIDRs()
+
+	for _, cidr := range cidrs {
+		ip, ipNet, parseErr := net.ParseCIDR(cidr)
+		Expect(parseErr).NotTo(HaveOccurred())
+
+		if ip.To4() != nil {
+			mask := binary.BigEndian.Uint32(ipNet.Mask)
+			start := binary.BigEndian.Uint32(ipNet.IP)
+			end := (start & mask) | (^mask)
+
+			chosenVIP := make([]byte, 4)
+			binary.BigEndian.PutUint32(chosenVIP, end-5)
+			return net.IP(chosenVIP).String()
+		}
+	}
+	Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
+	return ""
+}
+
+func getKindNetworkSubnetCIDRs() []string {
+	cmd := exec.Command(
+		"docker", "inspect", "kind",
+		"--format", `{{ range $i, $a := .IPAM.Config }}{{ println .Subnet }}{{ end }}`,
+	)
+	cmdOut := new(bytes.Buffer)
+	cmd.Stdout = cmdOut
+	Expect(cmd.Run()).To(Succeed(), "The Docker \"kind\" network was not found.")
+	reader := bufio.NewReader(cmdOut)
+
+	cidrs := []string{}
+	for {
+		line, readErr := reader.ReadString('\n')
+		if readErr != nil && readErr != io.EOF {
+			Expect(readErr).NotTo(HaveOccurred(), "Error finding subnet CIDRs in the Docker \"kind\" network")
+		}
+
+		cidrs = append(cidrs, strings.TrimSpace(line))
+		if readErr == io.EOF {
+			break
+		}
+	}
+
+	return cidrs
+}
--- a/testing/e2e/kind.go
+++ b/testing/e2e/kind.go
@@ -0,0 +1,147 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
+	"sigs.k8s.io/kind/pkg/cmd"
+	load "sigs.k8s.io/kind/pkg/cmd/kind/load/docker-image"
+	"sigs.k8s.io/kind/pkg/exec"
+	kindlog "sigs.k8s.io/kind/pkg/log"
+)
+
+func DeleteNodes(n ...nodes.Node) {
+	Expect(deleteNodes(n...)).To(Succeed())
+}
+
+func deleteNodes(n ...nodes.Node) error {
+	if len(n) == 0 {
+		return nil
+	}
+	const command = "docker"
+	args := make([]string, 0, len(n)+3) // allocate once
+	args = append(args,
+		"rm",
+		"-f", // force the container to be delete now
+		"-v", // delete volumes
+	)
+	for _, node := range n {
+		args = append(args, node.String())
+	}
+	if err := exec.Command(command, args...).Run(); err != nil {
+		return errors.Wrap(err, "failed to delete nodes")
+	}
+	return nil
+}
+
+func NodeIPv4(node nodes.Node) string {
+	ip, _, err := node.IP()
+	Expect(err).NotTo(HaveOccurred())
+	return ip
+}
+
+func LoadDockerImageToKind(logger kindlog.Logger, imagePath, clusterName string) {
+	loadImageCmd := load.NewCommand(logger, cmd.StandardIOStreams())
+	loadImageCmd.SetArgs([]string{"--name", clusterName, imagePath})
+	Expect(loadImageCmd.Execute()).To(Succeed())
+}
+
+func RunInNode(node nodes.Node, command string, args ...string) {
+	Expect(PrintCommandOutputIfErr(
+		node.Command(command, args...).Run(),
+	)).To(Succeed())
+}
+
+func StopPodInNode(node nodes.Node, containerName string) {
+	RunInNode(node,
+		"bash", "-c",
+		fmt.Sprintf(
+			"crictl pods  --output json --name %s-%s | jq -r \".items[0].id\" | xargs crictl stopp",
+			containerName,
+			node.String(),
+		),
+	)
+}
+
+func CopyFromNodeToDisk(node nodes.Node, org, dst string) {
+	dstFile, err := os.Create(dst)
+	Expect(err).NotTo(HaveOccurred())
+	defer dstFile.Close()
+
+	Expect(node.Command("cat", org).SetStdout(dstFile).Run()).To(Succeed())
+}
+
+func CopyFolderFromNodeToDisk(node nodes.Node, org, dst string) {
+	Expect(os.MkdirAll(dst, 0o755)).To(Succeed())
+
+	for _, file := range filesInNodeFolder(node, org) {
+		CopyFromNodeToDisk(node, file, filepath.Join(dst, filepath.Base(file)))
+	}
+}
+
+func CopyFolderFromNodeToNode(org, dst nodes.Node, folder string) {
+	for _, folder := range foldersInNodeFolder(org, folder) {
+		CopyFolderFromNodeToNode(org, dst, folder)
+	}
+
+	for _, file := range filesInNodeFolder(org, folder) {
+		Expect(nodeutils.CopyNodeToNode(org, dst, file)).To(Succeed())
+	}
+}
+
+func filesInNodeFolder(node nodes.Node, folder string) []string {
+	return commandOutputInLines(
+		node,
+		"find", folder, "-maxdepth", "1", "-mindepth", "1", "-type", "f",
+	)
+}
+
+func foldersInNodeFolder(node nodes.Node, folder string) []string {
+	return commandOutputInLines(
+		node,
+		"find", folder, "-maxdepth", "1", "-mindepth", "1", "-type", "d",
+	)
+}
+
+func commandOutputInLines(node nodes.Node, command string, args ...string) []string {
+	var linesB bytes.Buffer
+	Expect(node.Command(
+		command, args...,
+	).SetStdout(&linesB).Run()).To(Succeed())
+
+	var lines []string
+	scanner := bufio.NewScanner(&linesB)
+	for scanner.Scan() {
+		if l := scanner.Text(); l != "" {
+			lines = append(lines, l)
+		}
+	}
+	Expect(scanner.Err()).To(Succeed())
+
+	return lines
+}
+
+func PrintCommandOutputIfErr(err error) error {
+	tErr := err
+	for tErr != nil {
+		runErrP := &exec.RunError{}
+		runErr := &runErrP
+		if errors.As(tErr, runErr) {
+			klog.Errorf("Command failed %s:\n%s", (*runErr).Command, string((*runErr).Output))
+			break
+		}
+	}
+
+	return tErr
+}
--- a/testing/e2e/logger.go
+++ b/testing/e2e/logger.go
@@ -0,0 +1,43 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kind/pkg/log"
+)
+
+type TestLogger struct{}
+
+func (t TestLogger) Warnf(format string, args ...interface{}) {
+	klog.Warningf(format, args...)
+}
+
+func (t TestLogger) Warn(message string) {
+	klog.Warning(message)
+}
+
+func (t TestLogger) Error(message string) {
+	klog.Error(message)
+}
+
+func (t TestLogger) Errorf(format string, args ...interface{}) {
+	klog.Errorf(format, args...)
+}
+
+func (t TestLogger) Printf(format string, args ...interface{}) {
+	klog.Infof(format, args...)
+}
+
+func (t TestLogger) V(level log.Level) log.InfoLogger {
+	return TestInfoLogger{Verbose: klog.V(klog.Level(level))}
+}
+
+type TestInfoLogger struct {
+	klog.Verbose
+}
+
+func (t TestInfoLogger) Info(message string) {
+	t.Verbose.Info(message)
+}
--- a/testing/e2e/services/services.go
+++ b/testing/e2e/services/services.go
@@ -23,7 +23,6 @@ import (

 // 1. Create a deployment
 // 2. Expose the deployment
-
 func (config *testConfig) startServiceTest(ctx context.Context, clientset *kubernetes.Clientset) {
 	nodeTolerate := os.Getenv("NODE_TOLERATE")

--- a/testing/e2e/template.go
+++ b/testing/e2e/template.go
@@ -0,0 +1,9 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+type KubevipManifestValues struct {
+	ControlPlaneVIP string
+	ImagePath       string
+}
				`@@ -0,0 +1 @@`
				`KUBELET_EXTRA_ARGS="--kubeconfig='' --bootstrap-kubeconfig='' --container-runtime-endpoint=unix:///run/containerd/containerd.sock --node-labels= --pod-infra-container-image=registry.k8s.io/pause:3.9"`