fix(e2e): harden nightly etcd readiness

Signed-off-by: Maximilian Rink <maximilian.rink@telekom.de>

.github/workflows/nightly-e2e.yaml

@@ -53,32 +53,59 @@ jobs:
           go-version-file: go.mod
       - name: Build image locally
         run: make dockerx86Local
+      - name: Prepare Etcd artifacts
+        if: always()
+        run: |
+          mkdir -p /tmp/kube-vip-etcd-artifacts
+          : > /tmp/kube-vip-etcd-artifacts/suite.log
+          printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
       - name: Run Etcd tests
         id: etcd
         # Scheduled failures are tolerated only during the initial stabilization window.
         # The enforcement step below makes manual runs and later schedules blocking.
         continue-on-error: true
-        run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--json-report=/tmp/kube-vip-test-report-etcd.json --output-dir=/tmp" make e2e-tests-etcd
+        shell: bash
+        run: |
+          set +e
+          set -o pipefail
+          DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
+            GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
+            make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
+          exit_code=${PIPESTATUS[0]}
+          echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+          exit "$exit_code"
       - name: Summarize Etcd suite
         if: always()
         env:
           OUTCOME: ${{ steps.etcd.outcome }}
-        run: 'echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"'
+          EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
+        run: |
+          echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
+          printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
+            "${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
+            > /tmp/kube-vip-etcd-artifacts/result.json
       - name: Save logs
         uses: actions/upload-artifact@v7
+        continue-on-error: true
         with:
           name: etcd-e2e-logs
-          path: /tmp/kube-vip-test*
-          if-no-files-found: error
+          path: |
+            /tmp/kube-vip-etcd-artifacts
+            /tmp/kube-vip-test*
+          if-no-files-found: warn
         if: always()
       - name: Enforce Etcd result
-        if: steps.etcd.outcome == 'failure'
+        if: always()
         env:
           EVENT_NAME: ${{ github.event_name }}
+          OUTCOME: ${{ steps.etcd.outcome }}
         run: |
-          if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
-            echo "::warning::etcd e2e suite failed during stabilization through 2026-09-30"
+          if test "$OUTCOME" = success; then
             exit 0
           fi
-          echo "::error::etcd e2e suite failed; see the etcd-e2e-logs artifact"
+          if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
+            echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
+            exit 0
+          fi
+          echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
           exit 1

testing/e2e/etcd/cluster.go

@@ -5,6 +5,7 @@ package etcd
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"strings"
 	"time"
@@ -66,18 +67,18 @@ func (c *Cluster) Create(ctx context.Context) {
 	c.initEtcd(ctx)
 
 	c.Logger.Printf("Checking 1 node etcd is available through VIP")
-	c.VerifyEtcdThroughVIP(ctx, 15*time.Second)
+	c.VerifyEtcdThroughVIP(ctx, time.Minute)
 
 	c.Logger.Printf("Adding the rest of the nodes to the etcd cluster")
 	c.joinRestOfNodes(ctx)
 
 	c.Logger.Printf("Checking health for all nodes")
 	for _, node := range c.Nodes {
-		c.expectEtcdNodeHealthy(ctx, node, 15*time.Second)
+		c.expectEtcdNodeHealthy(ctx, node, time.Minute)
 	}
 
 	c.Logger.Printf("Checking %d nodes etcd is available through VIP", c.ClusterSpec.Nodes)
-	c.VerifyEtcdThroughVIP(ctx, 15*time.Second)
+	c.VerifyEtcdThroughVIP(ctx, time.Minute)
 }
 
 func (c *Cluster) initKindCluster() {
@@ -196,7 +197,7 @@ func (c *Cluster) initEtcd(ctx context.Context) {
 
 	e2e.CopyFolderFromNodeToDisk(firstNode, "/etc/kubernetes/pki/etcd", c.EtcdCertsFolder)
 
-	c.expectEtcdNodeHealthy(ctx, firstNode, 15*time.Second)
+	c.expectEtcdNodeHealthy(ctx, firstNode, time.Minute)
 }
 
 func runInNode(node nodes.Node, command string, args ...string) error {
@@ -235,7 +236,7 @@ func (c *Cluster) joinNode(ctx context.Context, firstNode, node nodes.Node) {
 
 	bindEtcdListenerToAllIPs(node)
 
-	c.expectEtcdNodeHealthy(ctx, node, 30*time.Second)
+	c.expectEtcdNodeHealthy(ctx, node, time.Minute)
 }
 
 func (c *Cluster) DeleteEtcdMember(ctx context.Context, toDelete, toKeep nodes.Node) {
@@ -329,8 +330,12 @@ func (c *Cluster) newEtcdClient(serverIPs ...string) *clientv3.Client {
 func (c *Cluster) VerifyEtcdThroughVIP(ctx context.Context, timeout time.Duration) {
 	etcdClient := c.newEtcdClient(c.VIP)
 	defer etcdClient.Close()
-	rCtx, cancel := context.WithTimeout(ctx, timeout)
-	_, err := etcdClient.MemberList(rCtx)
-	Expect(err).NotTo(HaveOccurred())
-	cancel()
+	err := waitForEtcdHealth(ctx, timeout, time.Second, func(probeCtx context.Context) error {
+		_, err := etcdClient.MemberList(probeCtx)
+		if err != nil {
+			return fmt.Errorf("listing members through VIP: %w", err)
+		}
+		return nil
+	})
+	Expect(err).NotTo(HaveOccurred(), "etcd should eventually be available through VIP %s", c.VIP)
 }

testing/e2e/etcd/election_test.go

@@ -4,7 +4,6 @@
 package etcd_test
 
 import (
-	"context"
 	"os"
 	"path/filepath"
 	"text/template"
@@ -44,16 +43,13 @@ func (t *testConfig) cleanup() {
 }
 
 var _ = Describe("kube-vip with etcd leader election", func() {
-	ctx, cancel := context.WithCancel(context.TODO())
-	defer cancel()
-
 	test := &testConfig{}
 
 	AfterEach(func() {
 		test.cleanup()
 	})
 
-	BeforeEach(func() {
+	BeforeEach(func(ctx SpecContext) {
 		By("configuring test", func() {
 			var err error
 			format.UseStringerRepresentation = true // Otherwise error stacks have binary format.
@@ -107,7 +103,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
 	})
 
 	When("an etcd node is removed", func() {
-		It("elects a new kube-vip leader and provides a VIP to the second node", func() {
+		It("elects a new kube-vip leader and provides a VIP to the second node", func(ctx SpecContext) {
 			By("removing as member and killing the first node", func() {
 				test.cluster.DeleteEtcdMember(ctx, test.cluster.Nodes[0], test.cluster.Nodes[1])
 			})

testing/e2e/etcd/health.go

@@ -22,30 +22,77 @@ import (
 func (c *Cluster) expectEtcdNodeHealthy(ctx context.Context, node nodes.Node, timeout time.Duration) {
 	httpClient := c.newEtcdHTTPClient()
 	client := c.newEtcdClient(e2e.NodeIPv4(node))
+	defer client.Close()
 	nodeEtcdEndpoint := etcdEndpointForNode(node)
-	Eventually(func(g Gomega) error {
-		health, err := getEtcdHealth(httpClient, node)
-		g.Expect(err).NotTo(HaveOccurred())
+	err := waitForEtcdHealth(ctx, timeout, time.Second, func(probeCtx context.Context) error {
+		health, err := getEtcdHealth(probeCtx, httpClient, node)
+		if err != nil {
+			return fmt.Errorf("checking member health: %w", err)
+		}
 		if !health.Healthy() {
 			c.Logger.Printf("Member %s is not healthy with reason: %s", node.String(), health.Reason)
+			return fmt.Errorf("member is not healthy: %s", health.Reason)
 		}
-		g.Expect(health.Healthy()).To(BeTrue(), "member is not healthy with reason: %s", health.Reason)
-		statusCtx, statusCancel := context.WithTimeout(ctx, 2*time.Second)
-		defer statusCancel()
-		status, err := client.Status(statusCtx, nodeEtcdEndpoint)
-		g.Expect(err).NotTo(HaveOccurred())
 
-		g.Expect(status.Errors).To(BeEmpty(), "member should not have any errors in status")
-		g.Expect(status.IsLearner).To(BeFalse(), "member should not be a learner")
+		status, err := client.Status(probeCtx, nodeEtcdEndpoint)
+		if err != nil {
+			return fmt.Errorf("checking member status: %w", err)
+		}
+		if len(status.Errors) != 0 {
+			return fmt.Errorf("member status contains errors: %v", status.Errors)
+		}
+		if status.IsLearner {
+			return errors.New("member is still a learner")
+		}
 
-		alarmsCtx, alarmsCancel := context.WithTimeout(ctx, 2*time.Second)
-		defer alarmsCancel()
-		alarms, err := client.AlarmList(alarmsCtx)
-		g.Expect(err).NotTo(HaveOccurred())
-		g.Expect(alarms.Alarms).To(BeEmpty(), "cluster should not have any alarms")
+		alarms, err := client.AlarmList(probeCtx)
+		if err != nil {
+			return fmt.Errorf("listing cluster alarms: %w", err)
+		}
+		if len(alarms.Alarms) != 0 {
+			return fmt.Errorf("cluster has alarms: %v", alarms.Alarms)
+		}
 
 		return nil
-	}, timeout).Should(Succeed(), "node %s should eventually be healthy", node.String())
+	})
+	Expect(err).NotTo(HaveOccurred(), "node %s should eventually be healthy", node.String())
+}
+
+func waitForEtcdHealth(ctx context.Context, timeout, interval time.Duration, check func(context.Context) error) error {
+	deadlineCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	var lastErr error
+	for {
+		if err := deadlineCtx.Err(); err != nil {
+			return etcdHealthWaitError(ctx, err, lastErr)
+		}
+
+		probeCtx, probeCancel := context.WithTimeout(deadlineCtx, 2*time.Second)
+		lastErr = check(probeCtx)
+		probeCancel()
+		if lastErr == nil {
+			return nil
+		}
+
+		timer := time.NewTimer(interval)
+		select {
+		case <-deadlineCtx.Done():
+			timer.Stop()
+			return etcdHealthWaitError(ctx, deadlineCtx.Err(), lastErr)
+		case <-timer.C:
+		}
+	}
+}
+
+func etcdHealthWaitError(ctx context.Context, deadlineErr, lastErr error) error {
+	if ctx.Err() != nil {
+		return fmt.Errorf("waiting for etcd health: %w", ctx.Err())
+	}
+	if lastErr != nil {
+		return fmt.Errorf("waiting for etcd health: %w (last probe: %v)", deadlineErr, lastErr)
+	}
+	return fmt.Errorf("waiting for etcd health: %w", deadlineErr)
 }
 
 func (c *Cluster) newEtcdHTTPClient() *http.Client {
@@ -75,8 +122,8 @@ func (h *etcdHealthCheckResponse) Healthy() bool {
 	return h.Health == "true"
 }
 
-func getEtcdHealth(c *http.Client, node nodes.Node) (*etcdHealthCheckResponse, error) {
-	req, err := http.NewRequest("GET", etcdHealthEndpoint(node), nil)
+func getEtcdHealth(ctx context.Context, c *http.Client, node nodes.Node) (*etcdHealthCheckResponse, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, etcdHealthEndpoint(node), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -87,7 +134,7 @@ func getEtcdHealth(c *http.Client, node nodes.Node) (*etcdHealthCheckResponse, e
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, errors.Wrapf(err, "etcd member not ready, returned http status %d", resp.StatusCode)
+		return nil, fmt.Errorf("etcd member not ready, returned HTTP status %d", resp.StatusCode)
 	}
 
 	body, err := io.ReadAll(resp.Body)

testing/e2e/etcd/health_test.go

@@ -0,0 +1,77 @@
+//go:build e2e
+// +build e2e
+
+package etcd
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestWaitForEtcdHealth(t *testing.T) {
+	tests := []struct {
+		name    string
+		context func() context.Context
+		check   func(context.Context) error
+		wantErr string
+	}{
+		{
+			name:    "retries transient errors",
+			context: context.Background,
+			check: func() func(context.Context) error {
+				attempts := 0
+				return func(context.Context) error {
+					attempts++
+					if attempts < 3 {
+						return errors.New("not ready")
+					}
+					return nil
+				}
+			}(),
+		},
+		{
+			name: "preserves parent cancellation",
+			context: func() context.Context {
+				ctx, cancel := context.WithCancel(context.Background())
+				cancel()
+				return ctx
+			},
+			check:   func(context.Context) error { return errors.New("probe failed") },
+			wantErr: "context canceled",
+		},
+		{
+			name: "does not probe after parent cancellation",
+			context: func() context.Context {
+				ctx, cancel := context.WithCancel(context.Background())
+				cancel()
+				return ctx
+			},
+			check: func(context.Context) error {
+				t.Fatal("check called with a canceled parent context")
+				return nil
+			},
+			wantErr: "context canceled",
+		},
+		{
+			name:    "reports last probe on timeout",
+			context: context.Background,
+			check:   func(context.Context) error { return errors.New("member is still a learner") },
+			wantErr: "last probe: member is still a learner",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := waitForEtcdHealth(tt.context(), 20*time.Millisecond, time.Millisecond, tt.check)
+			if tt.wantErr == "" && err != nil {
+				t.Fatalf("waitForEtcdHealth() error = %v", err)
+			}
+			if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) {
+				t.Fatalf("waitForEtcdHealth() error = %v, want substring %q", err, tt.wantErr)
+			}
+		})
+	}
+}