From d3c2319a79315d715c4ef47a4e09d67a0a9ef052 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Fri, 12 Jun 2026 16:18:31 +0400 Subject: [PATCH 1/2] fix(controllers): replace a persistently crash-looping member MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A non-bootstrap PVC member whose etcd cannot start — classically because its data dir was lost while the cluster membership moved on, leaving its frozen --initial-cluster stale (etcd refuses to boot: "error validating peerURLs ... member count is unequal") — crash-loops forever with no recovery path. The existing self-heal covers only memory-medium members (pod lost => data lost); for PVC members the PVC survives pod restarts, so "pod lost" != "data lost" and the stale --initial-cluster is never escaped. Detect such a member (etcd container not ready and restarted past a threshold, excluding OOMKilled) and delete it for replacement — but only when the rest of the cluster still has quorum, so a cluster-wide outage never cascades into mass deletion (the finalizer's MemberRemove is quorum-gated too). The cluster controller then gap-fills a fresh member with a current --initial-cluster. Also extend the Kamaji e2e to wait for readyMembers==3, then churn all three members one at a time and re-verify the tenant API still roundtrips through the fully replaced (hash-named) member set — guarding that member naming/replacement stays transparent to a Kamaji DataStore (stable -client Service + wildcard server-cert SAN). Signed-off-by: Andrey Kolkov --- controllers/etcdmember_controller.go | 77 ++++++++++++ controllers/etcdmember_controller_test.go | 143 ++++++++++++++++++++++ test/e2e/kamaji_datastore_test.go | 106 ++++++++++++++++ 3 files changed, 326 insertions(+) diff --git a/controllers/etcdmember_controller.go b/controllers/etcdmember_controller.go index 677fed0d..9aea6871 100644 --- a/controllers/etcdmember_controller.go +++ b/controllers/etcdmember_controller.go @@ -898,9 +898,66 @@ func restoreInitContainer(member *lll.EtcdMember, peerAddr, operatorImage string }, vols } +// dataLossRestartThreshold is how many times the etcd container must have +// restarted before we treat a non-bootstrap PVC member as unrecoverable and +// replace it. High enough to ride out transient join churn and slow restores; +// CrashLoopBackOff caps its backoff at 5m, so this many restarts means a +// member that has been unable to start for several minutes. +const dataLossRestartThreshold = 5 + +// etcdContainerStuck reports whether the pod's etcd container is persistently +// failing to start: not ready and restarted at least dataLossRestartThreshold +// times, excluding OOMKilled (a resource problem that re-creating the member +// would not fix). This is the signature of an unrecoverable member — most +// commonly a data dir lost while the cluster membership moved on, so the +// member's frozen --initial-cluster no longer matches the live cluster and +// etcd refuses to boot ("error validating peerURLs ... member count is +// unequal"). etcd ignores --initial-cluster for an initialised data dir, so a +// healthy member that merely restarts is unaffected. +func etcdContainerStuck(pod *corev1.Pod) bool { + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name != "etcd" { + continue + } + if cs.Ready || cs.RestartCount < dataLossRestartThreshold { + return false + } + if t := cs.LastTerminationState.Terminated; t != nil && t.Reason == "OOMKilled" { + return false + } + return true + } + return false +} + +// clusterHasQuorumWithout reports whether the member's cluster still has quorum +// among its OTHER members — i.e. losing this one is a minority failure, so +// replacing it cannot break the cluster. ReadyMembers already excludes this +// (not-ready) member, so it is the live healthy count. Used to gate self-heal: +// we never delete a member during a cluster-wide outage (where many members +// crash at once), only an isolated stuck member backed by a healthy majority. +func (r *EtcdMemberReconciler) clusterHasQuorumWithout(ctx context.Context, member *lll.EtcdMember) bool { + cluster, err := r.clusterFor(ctx, member) + if err != nil { + return false + } + desired := 0 + if cluster.Status.Observed != nil { + desired = int(cluster.Status.Observed.Replicas) + } + if desired == 0 && cluster.Spec.Replicas != nil { + desired = int(*cluster.Spec.Replicas) + } + if desired == 0 { + return false + } + return int(cluster.Status.ReadyMembers) >= desired/2+1 +} + // ── Status ─────────────────────────────────────────────────────────────── func (r *EtcdMemberReconciler) updateStatus(ctx context.Context, member *lll.EtcdMember) (ctrl.Result, error) { + log := log.FromContext(ctx) pod := &corev1.Pod{} if err := r.Get(ctx, types.NamespacedName{Namespace: member.Namespace, Name: member.Name}, pod); err != nil { if errors.IsNotFound(err) { @@ -950,6 +1007,26 @@ func (r *EtcdMemberReconciler) updateStatus(ctx context.Context, member *lll.Etc switch { case !podReady: + // Self-heal an unrecoverable member. A non-bootstrap PVC member whose + // etcd cannot start — classically because its data dir was lost while + // the cluster membership moved on, leaving its frozen --initial-cluster + // stale (etcd: "member count is unequal") — crash-loops forever on its + // own. Replace it: delete the CR so the finalizer does a clean + // MemberRemove and the cluster controller gap-fills a fresh member with + // a current --initial-cluster. Gate on the rest of the cluster having + // quorum so a cluster-wide outage never cascades into mass deletion + // (the finalizer's MemberRemove is quorum-gated too — belt and braces). + if !member.Spec.Bootstrap && + member.Spec.Storage.Medium != lll.StorageMediumMemory && + etcdContainerStuck(pod) && + r.clusterHasQuorumWithout(ctx, member) { + log.Info("etcd member is persistently crash-looping while the rest of the cluster is healthy; deleting it for replacement", + "restartThreshold", dataLossRestartThreshold) + if err := r.Delete(ctx, member); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } if setMemberCondition(member, lll.MemberReady, metav1.ConditionFalse, "PodNotReady", fmt.Sprintf("pod phase: %s", pod.Status.Phase)) { changed = true diff --git a/controllers/etcdmember_controller_test.go b/controllers/etcdmember_controller_test.go index 6a824f44..0c5cbd21 100644 --- a/controllers/etcdmember_controller_test.go +++ b/controllers/etcdmember_controller_test.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" lll "github.com/cozystack/etcd-operator/api/v1alpha2" ) @@ -691,6 +692,148 @@ func TestUpdateStatus_PopulatesMemberIDAndFlipsReady(t *testing.T) { } } +// TestEtcdContainerStuck pins the self-heal detection: an etcd container is +// "stuck" only when it is not ready, has restarted at least the threshold, and +// was not OOMKilled. +func TestEtcdContainerStuck(t *testing.T) { + mk := func(name string, ready bool, restarts int32, lastReason string) *corev1.Pod { + cs := corev1.ContainerStatus{Name: name, Ready: ready, RestartCount: restarts} + if lastReason != "" { + cs.LastTerminationState.Terminated = &corev1.ContainerStateTerminated{Reason: lastReason, ExitCode: 1} + } + return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{cs}}} + } + cases := []struct { + name string + pod *corev1.Pod + want bool + }{ + {"stuck: not ready, at threshold, Error exit", mk("etcd", false, dataLossRestartThreshold, "Error"), true}, + {"stuck: no last-termination recorded yet", mk("etcd", false, dataLossRestartThreshold+1, ""), true}, + {"ready", mk("etcd", true, dataLossRestartThreshold+4, "Error"), false}, + {"below restart threshold", mk("etcd", false, dataLossRestartThreshold-1, "Error"), false}, + {"OOMKilled is excluded", mk("etcd", false, dataLossRestartThreshold+4, "OOMKilled"), false}, + {"no etcd container", mk("other", false, dataLossRestartThreshold+4, "Error"), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := etcdContainerStuck(tc.pod); got != tc.want { + t.Fatalf("etcdContainerStuck = %v, want %v", got, tc.want) + } + }) + } +} + +// crashLoopPod builds a Pod whose etcd container is persistently crash-looping +// (not ready, restarted past the threshold with an Error exit) — the data-loss +// signature. +func crashLoopPod(name, ns string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + Ready: false, + RestartCount: dataLossRestartThreshold + 2, + LastTerminationState: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 1}, + }, + }}, + }, + } +} + +// clusterWithReady builds a 3-replica EtcdCluster and persists ready as its +// status.readyMembers (status is a subresource on the fake client). +func clusterWithReady(t *testing.T, c client.Client, name, ns string, ready int32) { + t.Helper() + got := mustGet(t, c, name, ns, &lll.EtcdCluster{}) + got.Status.ReadyMembers = ready + if err := c.Status().Update(context.Background(), got); err != nil { + t.Fatalf("seed cluster status: %v", err) + } +} + +// TestUpdateStatus_ReplacesStuckMember: a persistently crash-looping +// non-bootstrap PVC member is deleted for replacement when the rest of the +// cluster still has quorum. +func TestUpdateStatus_ReplacesStuckMember(t *testing.T) { + ctx := context.Background() + cluster := &lll.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "ns"}, + Spec: lll.EtcdClusterSpec{Replicas: ptrInt32(3)}, + } + member := &lll.EtcdMember{ + ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: "ns", Labels: memberLabels("test", "test-1")}, + Spec: lll.EtcdMemberSpec{ClusterName: "test", Version: "3.5.17", Storage: lll.StorageSpec{Size: quickQty(t, "1Gi")}, InitialCluster: "x", ClusterToken: "test"}, + } + c, _ := newTestClient(t, cluster, member, crashLoopPod("test-1", "ns")) + clusterWithReady(t, c, "test", "ns", 2) // 2/3 ready → quorum without test-1 + + r := &EtcdMemberReconciler{Client: c, Scheme: testScheme(t)} + if _, err := r.updateStatus(ctx, member); err != nil { + t.Fatalf("updateStatus: %v", err) + } + + err := c.Get(ctx, types.NamespacedName{Name: "test-1", Namespace: "ns"}, &lll.EtcdMember{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected member deleted for replacement; Get err = %v", err) + } +} + +// TestUpdateStatus_KeepsStuckMemberWithoutQuorum: the same crash-looping member +// is NOT deleted when the rest of the cluster lacks quorum — self-heal must +// never cascade a cluster-wide outage into mass deletion. +func TestUpdateStatus_KeepsStuckMemberWithoutQuorum(t *testing.T) { + ctx := context.Background() + cluster := &lll.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "ns"}, + Spec: lll.EtcdClusterSpec{Replicas: ptrInt32(3)}, + } + member := &lll.EtcdMember{ + ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: "ns", Labels: memberLabels("test", "test-1")}, + Spec: lll.EtcdMemberSpec{ClusterName: "test", Version: "3.5.17", Storage: lll.StorageSpec{Size: quickQty(t, "1Gi")}, InitialCluster: "x", ClusterToken: "test"}, + } + c, _ := newTestClient(t, cluster, member, crashLoopPod("test-1", "ns")) + clusterWithReady(t, c, "test", "ns", 1) // only 1/3 ready → no quorum + + r := &EtcdMemberReconciler{Client: c, Scheme: testScheme(t)} + if _, err := r.updateStatus(ctx, member); err != nil { + t.Fatalf("updateStatus: %v", err) + } + + if err := c.Get(ctx, types.NamespacedName{Name: "test-1", Namespace: "ns"}, &lll.EtcdMember{}); err != nil { + t.Fatalf("member must NOT be deleted without quorum; Get err = %v", err) + } +} + +// TestUpdateStatus_KeepsStuckBootstrapMember: the bootstrap seed is never +// self-healed by deletion — there is nothing to replace it from yet. +func TestUpdateStatus_KeepsStuckBootstrapMember(t *testing.T) { + ctx := context.Background() + cluster := &lll.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "ns"}, + Spec: lll.EtcdClusterSpec{Replicas: ptrInt32(3)}, + } + member := &lll.EtcdMember{ + ObjectMeta: metav1.ObjectMeta{Name: "test-0", Namespace: "ns", Labels: memberLabels("test", "test-0")}, + Spec: lll.EtcdMemberSpec{ClusterName: "test", Bootstrap: true, Version: "3.5.17", Storage: lll.StorageSpec{Size: quickQty(t, "1Gi")}, InitialCluster: "x", ClusterToken: "test"}, + } + c, _ := newTestClient(t, cluster, member, crashLoopPod("test-0", "ns")) + clusterWithReady(t, c, "test", "ns", 2) + + r := &EtcdMemberReconciler{Client: c, Scheme: testScheme(t)} + if _, err := r.updateStatus(ctx, member); err != nil { + t.Fatalf("updateStatus: %v", err) + } + + if err := c.Get(ctx, types.NamespacedName{Name: "test-0", Namespace: "ns"}, &lll.EtcdMember{}); err != nil { + t.Fatalf("bootstrap member must NOT be self-deleted; Get err = %v", err) + } +} + // TestRemoveMemberFromEtcd_LastMemberIsNoOp: if no other members exist (the // cluster is being torn down or this is genuinely the last member), the // finalizer can't reach a peer to call MemberRemove. Don't block — return diff --git a/test/e2e/kamaji_datastore_test.go b/test/e2e/kamaji_datastore_test.go index 08ee8a85..af51594e 100644 --- a/test/e2e/kamaji_datastore_test.go +++ b/test/e2e/kamaji_datastore_test.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/url" + "sort" "strings" "testing" "time" @@ -134,6 +135,94 @@ func TestKamajiDataStore(t *testing.T) { } t.Logf("found %q among etcd keys", proofName) + // ── Member churn: replace EVERY original member one at a time and prove + // Kamaji keeps working through the new, GenerateName-hashed members. + // Native members are named via GenerateName ("-"), so the + // DataStore can only address the cluster through the stable + // -client Service (its sole endpoint) and the operator's server + // cert SAN is a wildcard (*...svc). Member names therefore + // never reach Kamaji — this churns the entire member set out from under a + // live tenant control plane and guards that contract end to end. + // + // Gate on a fully-formed cluster first: Available latches on quorum, not on + // the full replica count, so without this wait we might delete a member + // while the third is still a freshly-promoted/learner member — collapsing + // the cluster into the fragile 2-node window mid-bootstrap. + waitFor(ctx, t, 5*time.Minute, "all 3 members Ready before churn", func(ctx context.Context) error { + ec := &etcdv1alpha2.EtcdCluster{} + if err := kube.Get(ctx, client.ObjectKey{Namespace: e2eNamespace, Name: clusterName}, ec); err != nil { + return err + } + if ec.Status.ReadyMembers != 3 { + return fmt.Errorf("readyMembers=%d, want 3", ec.Status.ReadyMembers) + } + return nil + }) + original := memberNames(ctx, t) + if len(original) != 3 { + t.Fatalf("expected 3 members before churn, got %d: %v", len(original), original) + } + t.Logf("original members: %v", original) + for _, victim := range original { + t.Logf("deleting EtcdMember %q (operator does MemberRemove + a GenerateName replacement)", victim) + m := &etcdv1alpha2.EtcdMember{ObjectMeta: metav1.ObjectMeta{Namespace: e2eNamespace, Name: victim}} + if err := kube.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("delete member %s: %v", victim, err) + } + waitFor(ctx, t, 5*time.Minute, fmt.Sprintf("%q removed and the cluster back to 3 ready members", victim), + func(ctx context.Context) error { + err := kube.Get(ctx, client.ObjectKey{Namespace: e2eNamespace, Name: victim}, &etcdv1alpha2.EtcdMember{}) + if err == nil { + return fmt.Errorf("victim %s still present (MemberRemove in flight)", victim) + } + if !apierrors.IsNotFound(err) { + return err + } + if names := memberNames(ctx, t); len(names) != 3 { + return fmt.Errorf("have %d members, want 3: %v", len(names), names) + } + ec := &etcdv1alpha2.EtcdCluster{} + if err := kube.Get(ctx, client.ObjectKey{Namespace: e2eNamespace, Name: clusterName}, ec); err != nil { + return err + } + if ec.Status.ReadyMembers != 3 { + return fmt.Errorf("readyMembers=%d, want 3", ec.Status.ReadyMembers) + } + return nil + }) + } + + // The cluster is now a wholly fresh member set — no original name remains. + final := memberNames(ctx, t) + for _, o := range original { + for _, f := range final { + if o == f { + t.Fatalf("original member %q still present after full churn: %v", o, final) + } + } + } + t.Logf("fully churned member set: %v -> %v", original, final) + + // Kamaji must still work through the new members: the original proof key + // survived all three replacements, and a fresh write still lands in etcd — + // all without any change to the DataStore (it still points at the same + // -client Service). + if keys := etcdKeys(ctx, t); !strings.Contains(keys, proofName) { + t.Fatalf("proof key %q lost after member churn", proofName) + } + const churnProof = "e2e-proof-postchurn" + cm2 := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: churnProof, Namespace: "default"}, + Data: map[string]string{"written-by": "etcd-operator-e2e-postchurn"}, + } + if _, err := tenantSet.CoreV1().ConfigMaps("default").Create(ctx, cm2, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("create post-churn ConfigMap via tenant API: %v", err) + } + if keys := etcdKeys(ctx, t); !strings.Contains(keys, churnProof) { + t.Fatalf("post-churn write %q did not reach etcd through the fresh members", churnProof) + } + t.Log("Kamaji tenant API still roundtrips through the fully churned (hash-named) member set") + // Teardown — reverse order, waiting where deletion is asynchronous. deleteAndWait(ctx, t, "kamaji.clastix.io/v1alpha1", "TenantControlPlane", e2eNamespace, tcpName, 5*time.Minute) deleteAndWait(ctx, t, "kamaji.clastix.io/v1alpha1", "DataStore", "", "kamaji-e2e", 2*time.Minute) @@ -307,6 +396,23 @@ func etcdKeys(ctx context.Context, t *testing.T) string { return stdout } +// memberNames returns the sorted names of the cluster's EtcdMembers, selected +// by the cluster label (member pods/CRs carry GenerateName-hashed names). +func memberNames(ctx context.Context, t *testing.T) []string { + t.Helper() + list := &etcdv1alpha2.EtcdMemberList{} + if err := kube.List(ctx, list, client.InNamespace(e2eNamespace), + client.MatchingLabels{"etcd-operator.cozystack.io/cluster": clusterName}); err != nil { + t.Fatalf("list etcd members: %v", err) + } + names := make([]string, 0, len(list.Items)) + for i := range list.Items { + names = append(names, list.Items[i].Name) + } + sort.Strings(names) + return names +} + func podExec(ctx context.Context, namespace, pod, container string, command []string) (string, string, error) { req := clientset.CoreV1().RESTClient().Post(). Resource("pods").Namespace(namespace).Name(pod).SubResource("exec"). From 4d3f7030f301a5bc920dd8a7f88b9291d18f0959 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 25 Jun 2026 14:16:43 +0400 Subject: [PATCH 2/2] review fixes Signed-off-by: Andrey Kolkov --- Makefile | 2 +- README.md | 2 +- controllers/etcdmember_controller.go | 35 ++- controllers/etcdmember_controller_test.go | 54 +++- docs/concepts.md | 15 +- docs/operations.md | 12 + test/e2e/kamaji_datastore_test.go | 21 +- test/e2e/member_selfheal_test.go | 289 ++++++++++++++++++++++ 8 files changed, 416 insertions(+), 14 deletions(-) create mode 100644 test/e2e/member_selfheal_test.go diff --git a/Makefile b/Makefile index 80060fa4..f7829447 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ test: manifests generate fmt vet envtest ## Run tests. .PHONY: test-e2e test-e2e: ## Run the e2e suite against the current kubeconfig context (expects cert-manager, Kamaji and the operator installed; see hack/e2e.sh). - go test -tags e2e -count=1 ./test/e2e/ -v -timeout 30m + go test -tags e2e -count=1 ./test/e2e/ -v -timeout 45m .PHONY: e2e e2e: ## Provision a kind cluster with cert-manager and Kamaji, deploy the operator, run the e2e suite. KEEP_CLUSTER=1 keeps the cluster for debugging. diff --git a/README.md b/README.md index aa9774d0..64669805 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The full design rationale is in [docs/concepts.md](docs/concepts.md). ## What's not supported (yet) -No multi-user / per-tenant RBAC inside etcd — single-user `root` auth is available via `spec.auth.enabled` (BYO credentials Secret; see [docs/concepts.md](docs/concepts.md#authentication)), but every authenticated client is `root`. No in-place version upgrades (changing `spec.version` only affects newly-created members). No PVC resizing — see [#2](https://github.com/lllamnyp/etcd-operator/issues/2). No automatic broken-member replacement for PVC-backed clusters (memory-backed members do auto-replace on Pod loss; `status.brokenMembers` reads 0 in practice — see [docs/concepts.md](docs/concepts.md#storage)). One-shot snapshots and restore-on-bootstrap are supported (see above), but there is no *scheduled* snapshot CRD. No defragmentation scheduling. PodAntiAffinity is supported via `spec.affinity` but not applied by default (defaulting tracked in [#16](https://github.com/lllamnyp/etcd-operator/issues/16)). See the [issue tracker](https://github.com/lllamnyp/etcd-operator/issues) for the running follow-up list. +No multi-user / per-tenant RBAC inside etcd — single-user `root` auth is available via `spec.auth.enabled` (BYO credentials Secret; see [docs/concepts.md](docs/concepts.md#authentication)), but every authenticated client is `root`. No in-place version upgrades (changing `spec.version` only affects newly-created members). No PVC resizing — see [#2](https://github.com/lllamnyp/etcd-operator/issues/2). PVC-backed members auto-replace only on a *persistent crash-loop* (a lost or corrupt data dir whose etcd cannot boot) — quorum-gated, and far slower than the seconds-fast Pod-loss path memory-backed members get (tens of minutes, at the CrashLoopBackOff cap); a member that is merely slow or flapping is left alone. `status.brokenMembers` still reads 0 in practice — see [docs/concepts.md](docs/concepts.md#storage). One-shot snapshots and restore-on-bootstrap are supported (see above), but there is no *scheduled* snapshot CRD. No defragmentation scheduling. PodAntiAffinity is supported via `spec.affinity` but not applied by default (defaulting tracked in [#16](https://github.com/lllamnyp/etcd-operator/issues/16)). See the [issue tracker](https://github.com/lllamnyp/etcd-operator/issues) for the running follow-up list. ## Quick start diff --git a/controllers/etcdmember_controller.go b/controllers/etcdmember_controller.go index 9aea6871..fe2274ac 100644 --- a/controllers/etcdmember_controller.go +++ b/controllers/etcdmember_controller.go @@ -914,7 +914,17 @@ const dataLossRestartThreshold = 5 // etcd refuses to boot ("error validating peerURLs ... member count is // unequal"). etcd ignores --initial-cluster for an initialised data dir, so a // healthy member that merely restarts is unaffected. +// +// A Pod already being deleted (DeletionTimestamp set — manual restart, node +// drain, eviction) is never "stuck": its containers are terminating on the way +// to a clean reschedule, and treating that window as unrecoverable would +// trigger a needless member replacement. OOMKilled is excluded whether it is +// the last termination or the current one — a just-OOMKilled container sits in +// State.Terminated before it transitions to Waiting/CrashLoopBackOff. func etcdContainerStuck(pod *corev1.Pod) bool { + if pod.DeletionTimestamp != nil { + return false + } for _, cs := range pod.Status.ContainerStatuses { if cs.Name != "etcd" { continue @@ -922,6 +932,9 @@ func etcdContainerStuck(pod *corev1.Pod) bool { if cs.Ready || cs.RestartCount < dataLossRestartThreshold { return false } + if t := cs.State.Terminated; t != nil && t.Reason == "OOMKilled" { + return false + } if t := cs.LastTerminationState.Terminated; t != nil && t.Reason == "OOMKilled" { return false } @@ -932,10 +945,15 @@ func etcdContainerStuck(pod *corev1.Pod) bool { // clusterHasQuorumWithout reports whether the member's cluster still has quorum // among its OTHER members — i.e. losing this one is a minority failure, so -// replacing it cannot break the cluster. ReadyMembers already excludes this -// (not-ready) member, so it is the live healthy count. Used to gate self-heal: -// we never delete a member during a cluster-wide outage (where many members -// crash at once), only an isolated stuck member backed by a healthy majority. +// replacing it cannot break the cluster. Used to gate self-heal: we never +// delete a member during a cluster-wide outage (where many members crash at +// once), only an isolated stuck member backed by a healthy majority. +// +// ReadyMembers is the cluster controller's count and can lag: a member that has +// just started crash-looping may still be counted ready until that controller +// reconciles. So if THIS member's own status still records it as ready, subtract +// it — otherwise a second concurrent failure could read a stale-high count, pass +// the gate, and let one member be deleted when quorum is in fact already gone. func (r *EtcdMemberReconciler) clusterHasQuorumWithout(ctx context.Context, member *lll.EtcdMember) bool { cluster, err := r.clusterFor(ctx, member) if err != nil { @@ -951,7 +969,14 @@ func (r *EtcdMemberReconciler) clusterHasQuorumWithout(ctx context.Context, memb if desired == 0 { return false } - return int(cluster.Status.ReadyMembers) >= desired/2+1 + readyOthers := int(cluster.Status.ReadyMembers) + for _, cond := range member.Status.Conditions { + if cond.Type == lll.MemberReady && cond.Status == metav1.ConditionTrue { + readyOthers-- + break + } + } + return readyOthers >= desired/2+1 } // ── Status ─────────────────────────────────────────────────────────────── diff --git a/controllers/etcdmember_controller_test.go b/controllers/etcdmember_controller_test.go index 0c5cbd21..69fd0565 100644 --- a/controllers/etcdmember_controller_test.go +++ b/controllers/etcdmember_controller_test.go @@ -694,7 +694,7 @@ func TestUpdateStatus_PopulatesMemberIDAndFlipsReady(t *testing.T) { // TestEtcdContainerStuck pins the self-heal detection: an etcd container is // "stuck" only when it is not ready, has restarted at least the threshold, and -// was not OOMKilled. +// was not OOMKilled — and never while the Pod is itself being deleted. func TestEtcdContainerStuck(t *testing.T) { mk := func(name string, ready bool, restarts int32, lastReason string) *corev1.Pod { cs := corev1.ContainerStatus{Name: name, Ready: ready, RestartCount: restarts} @@ -703,6 +703,20 @@ func TestEtcdContainerStuck(t *testing.T) { } return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{cs}}} } + // currentlyOOMKilled: the etcd container is over the restart threshold and + // sits in State.Terminated=OOMKilled right now (not yet backed off into + // Waiting/CrashLoopBackOff), so LastTerminationState is empty. + currentlyOOMKilled := &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", Ready: false, RestartCount: dataLossRestartThreshold + 4, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled", ExitCode: 137}}, + }}}} + // deletingPod: a stuck-looking container, but the Pod is terminating + // (DeletionTimestamp set) — a drain/eviction/restart, not an unrecoverable + // member. + now := metav1.Now() + deletingPod := mk("etcd", false, dataLossRestartThreshold+4, "Error") + deletingPod.DeletionTimestamp = &now + cases := []struct { name string pod *corev1.Pod @@ -712,7 +726,9 @@ func TestEtcdContainerStuck(t *testing.T) { {"stuck: no last-termination recorded yet", mk("etcd", false, dataLossRestartThreshold+1, ""), true}, {"ready", mk("etcd", true, dataLossRestartThreshold+4, "Error"), false}, {"below restart threshold", mk("etcd", false, dataLossRestartThreshold-1, "Error"), false}, - {"OOMKilled is excluded", mk("etcd", false, dataLossRestartThreshold+4, "OOMKilled"), false}, + {"OOMKilled (last termination) is excluded", mk("etcd", false, dataLossRestartThreshold+4, "OOMKilled"), false}, + {"OOMKilled (current state) is excluded", currentlyOOMKilled, false}, + {"pod being deleted is never stuck", deletingPod, false}, {"no etcd container", mk("other", false, dataLossRestartThreshold+4, "Error"), false}, } for _, tc := range cases { @@ -834,6 +850,40 @@ func TestUpdateStatus_KeepsStuckBootstrapMember(t *testing.T) { } } +// TestUpdateStatus_KeepsStuckMemberWhenStaleReadyCountIncludesIt: a member that +// just started crash-looping can still be counted in the cluster controller's +// lagging ReadyMembers and still record MemberReady=True in its own status. The +// quorum gate must subtract this member, so a stale-high count can't green-light +// a deletion that would actually drop the cluster below quorum. +func TestUpdateStatus_KeepsStuckMemberWhenStaleReadyCountIncludesIt(t *testing.T) { + ctx := context.Background() + cluster := &lll.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "ns"}, + Spec: lll.EtcdClusterSpec{Replicas: ptrInt32(3)}, + } + member := &lll.EtcdMember{ + ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: "ns", Labels: memberLabels("test", "test-1")}, + Spec: lll.EtcdMemberSpec{ClusterName: "test", Version: "3.5.17", Storage: lll.StorageSpec{Size: quickQty(t, "1Gi")}, InitialCluster: "x", ClusterToken: "test"}, + } + // Stale: this member is still listed ready in its own status, mirroring a + // ReadyMembers count that has not yet been decremented for it. + member.Status.Conditions = []metav1.Condition{{ + Type: lll.MemberReady, Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now(), + }} + c, _ := newTestClient(t, cluster, member, crashLoopPod("test-1", "ns")) + clusterWithReady(t, c, "test", "ns", 2) // stale-high: still counts test-1 + + r := &EtcdMemberReconciler{Client: c, Scheme: testScheme(t)} + if _, err := r.updateStatus(ctx, member); err != nil { + t.Fatalf("updateStatus: %v", err) + } + + // 2 ready minus this still-counted member = 1 < quorum(2) → must be kept. + if err := c.Get(ctx, types.NamespacedName{Name: "test-1", Namespace: "ns"}, &lll.EtcdMember{}); err != nil { + t.Fatalf("member must NOT be deleted while it is still double-counted in ReadyMembers; Get err = %v", err) + } +} + // TestRemoveMemberFromEtcd_LastMemberIsNoOp: if no other members exist (the // cluster is being torn down or this is genuinely the last member), the // finalizer can't reach a peer to call MemberRemove. Don't block — return diff --git a/docs/concepts.md b/docs/concepts.md index e6d1dc97..a0daf24c 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -127,7 +127,7 @@ Each member's data dir is configured via `spec.storage`, a struct with `size`, ` | `spec.storage.medium` | Backend | Lifetime | Pod loss → | |---|---|---|---| -| `""` (default) | PVC; `spec.storage.storageClassName` if set, else the namespace default; `ReadWriteOnce` | Survives Pod restart, eviction, node failure (re-attached to new Pod). | Same Pod / new Pod re-uses existing data dir; etcd rejoins with the same member ID and `ClusterID`. | +| `""` (default) | PVC; `spec.storage.storageClassName` if set, else the namespace default; `ReadWriteOnce` | Survives Pod restart, eviction, node failure (re-attached to new Pod). | Same Pod / new Pod re-uses existing data dir; etcd rejoins with the same member ID and `ClusterID`. **Exception:** if the data dir is lost or corrupt so etcd cannot boot and the member crash-loops past the threshold, the operator deletes and replaces it with a *fresh* member ID (quorum-gated) — see [Crash-loop self-heal](#crash-loop-self-heal-pvc-members) below. | | `"Memory"` | `emptyDir{medium: Memory}` with `sizeLimit: spec.storage.size` | Bound to the Pod. Container restart preserves tmpfs; Pod deletion / eviction / node failure destroys it. | Operator detects Pod loss via recorded `Status.PodUID`, self-deletes the `EtcdMember`, finalizer calls `MemberRemove`, scale-up gap-fill creates a replacement with a fresh member ID. | `spec.storage.storageClassName` mirrors the corev1 PVC field of the same name: **nil** uses the namespace's default `StorageClass`, the **empty string** explicitly disables dynamic provisioning (a pre-provisioned PV must already match), any other value names a specific `StorageClass`. It's immutable post-create — `PersistentVolumeClaim.spec.storageClassName` is itself immutable, so there is no in-place change a controller could honour without rolling every PVC. Ignored when `medium=Memory` (no PVC is created). @@ -153,7 +153,18 @@ The member controller self-deletes the `EtcdMember`. The existing finalizer runs If quorum is already lost across multiple simultaneous failures, `MemberRemove` will fail and the dying members stay in `Terminating` until quorum returns. That is the correct outcome: the cluster is dead and the user has to recreate it. The operator does not try to be clever about restoring a quorum from inconsistent half-states. -`Status.BrokenMembers` stays at 0 in normal operation, including across a memory pod-loss + auto-replacement cycle. The `isBroken` predicate is implemented for memory members (lost-Pod state), but the member controller intercepts the loss and self-deletes the member in the same reconcile pass — by the time the cluster controller computes the count, the lost member is already `Terminating` and excluded from the running set. The field exists as a future hook for broken-member detection policies that don't immediately tear the member down (e.g. PVC corruption with a grace period). For PVC-backed members today, `isBroken` stays a stub; richer detection is a future concern. +`Status.BrokenMembers` stays at 0 in normal operation, including across a memory pod-loss + auto-replacement cycle. The `isBroken` predicate is implemented for memory members (lost-Pod state), but the member controller intercepts the loss and self-deletes the member in the same reconcile pass — by the time the cluster controller computes the count, the lost member is already `Terminating` and excluded from the running set. The field exists as a future hook for broken-member detection policies that don't immediately tear the member down. PVC crash-loop self-heal (next section) does **not** flow through `isBroken`/`BrokenMembers` — it triggers off the Pod's container restart count directly, so `BrokenMembers` stays 0 across that cycle too. + +### Crash-loop self-heal (PVC members) + +A PVC-backed member normally survives Pod loss: the PVC re-attaches and etcd rejoins with the same member ID (see the [storage table](#storage)). But that assumes the data dir is intact. If the data dir is lost or corrupt — classically a volume lost on node failure while the cluster membership moved on, leaving the member's *frozen* `--initial-cluster` stale — etcd refuses to boot (`error validating peerURLs ... member count is unequal`) and the Pod crash-loops forever. "Pod lost ≠ data lost" for a PVC, so the memory-style `Status.PodUID` loss check never fires here. + +The member controller detects this and replaces the member: + +- **Trigger.** The etcd container is not ready and has restarted at least `dataLossRestartThreshold` (5) times. `OOMKilled` is excluded (whether it's the current or the last termination) — that's a resource problem re-creating the member would not fix — and a Pod that is itself being deleted (drain/eviction/manual restart) is never treated as stuck. +- **Quorum gate.** The operator deletes the member only when the *rest* of the cluster still has quorum, so a cluster-wide outage (many members crashing at once) never cascades into mass deletion. The count is read from `Status.ReadyMembers`, which the cluster controller maintains and which can lag; if the stuck member is still counted ready, the gate subtracts it. As a second line of defence the finalizer's `MemberRemove` is itself quorum-gated, so even a stale-high count cannot delete data below quorum. +- **Replacement.** Deleting the `EtcdMember` runs the finalizer's clean `MemberRemove`, the member-owned PVC is GC'd (discarding the corrupt data dir), and the cluster controller gap-fills a fresh `GenerateName` member with a current `--initial-cluster` and a **new** etcd member ID — not a same-ID rejoin. +- **Latency.** `CrashLoopBackOff` caps its backoff at 5 minutes, so reaching 5 restarts takes on the order of **tens of minutes**, not the ~5s of the memory Pod-loss path. A deliberately-deleted-and-replaced member during this window is expected operator behavior, not a fault. A slow restore or slow learner join on the *replacement* can itself trip the threshold and be replaced again; this is quorum-gated and self-limiting, but expect it on a struggling cluster. ### What is missing from memory clusters today diff --git a/docs/operations.md b/docs/operations.md index c5ce76f3..d616696f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -478,6 +478,18 @@ kubectl get etcdmember.etcd-operator.cozystack.io -n default -w # appear, and READY=3 restore within a minute or so. ``` +### PVC member crash-loop replacement + +The `Status.PodUID` mechanism above keys on the Pod disappearing — the right signal for a memory-backed member, where Pod loss *is* data loss. A PVC-backed member is different: the PVC survives Pod restarts, so a lost Pod re-attaches the same data dir and etcd rejoins with the same member ID. There is a separate, second trigger for the case where the data dir itself is gone or corrupt. + +If a non-bootstrap PVC member's etcd cannot start — classically because its data dir was lost (e.g. a volume lost on node failure) while the cluster membership moved on, leaving its frozen `--initial-cluster` stale (`error validating peerURLs ... member count is unequal`) — it crash-loops forever with no recovery path of its own. The operator detects this and replaces it: + +- **Detection**: the etcd container is not ready and has restarted at least 5 times (`dataLossRestartThreshold`), excluding `OOMKilled` (a resource problem, not a lost data dir — raising `spec.resources.limits.memory` is the fix there, not replacement). A Pod that is being deleted (drain, eviction, manual restart) is never treated as stuck. +- **Quorum gate**: the operator deletes the member only when the *rest* of the cluster still has quorum, so a cluster-wide outage never cascades into mass deletion. The gate reads `Status.ReadyMembers` (maintained by the cluster controller, and possibly lagging) and subtracts the stuck member if it is still counted; the finalizer's `MemberRemove` is independently quorum-gated as a backstop. +- **Replacement**: the `EtcdMember` CR is deleted → finalizer `MemberRemove` → the member-owned `data-` PVC is GC'd (discarding the corrupt data dir) → the cluster controller gap-fills a fresh `GenerateName` member with a current `--initial-cluster` and a **new** etcd member ID. + +**Detection latency is much longer than the Pod-loss path.** `CrashLoopBackOff` caps backoff at 5 minutes, so reaching 5 restarts takes **tens of minutes**, not ~5 s. Budget for that before concluding the operator is misbehaving — a member that vanishes and is replaced by a fresh-named one after a long crash-loop is the operator working as designed, not flapping. Note also that a replacement which is itself slow to come up (slow restore, slow learner join) can trip the same threshold and be replaced again; this is quorum-gated and harmless to the cluster, but expect repeated replacement on a genuinely unhealthy member. + ### Pause is not supported Setting `spec.replicas: 0` on a memory cluster is **rejected by the apiserver** (CEL validation rule on `EtcdClusterSpec`): diff --git a/test/e2e/kamaji_datastore_test.go b/test/e2e/kamaji_datastore_test.go index af51594e..a93bd9de 100644 --- a/test/e2e/kamaji_datastore_test.go +++ b/test/e2e/kamaji_datastore_test.go @@ -178,7 +178,11 @@ func TestKamajiDataStore(t *testing.T) { if !apierrors.IsNotFound(err) { return err } - if names := memberNames(ctx, t); len(names) != 3 { + names, err := listMemberNames(ctx) + if err != nil { + return err + } + if len(names) != 3 { return fmt.Errorf("have %d members, want 3: %v", len(names), names) } ec := &etcdv1alpha2.EtcdCluster{} @@ -400,17 +404,28 @@ func etcdKeys(ctx context.Context, t *testing.T) string { // by the cluster label (member pods/CRs carry GenerateName-hashed names). func memberNames(ctx context.Context, t *testing.T) []string { t.Helper() + names, err := listMemberNames(ctx) + if err != nil { + t.Fatalf("list etcd members: %v", err) + } + return names +} + +// listMemberNames is the error-returning form of memberNames, safe to call +// inside a waitFor retry callback (where t.Fatalf would abort the test on a +// transient API error instead of letting the poll retry). +func listMemberNames(ctx context.Context) ([]string, error) { list := &etcdv1alpha2.EtcdMemberList{} if err := kube.List(ctx, list, client.InNamespace(e2eNamespace), client.MatchingLabels{"etcd-operator.cozystack.io/cluster": clusterName}); err != nil { - t.Fatalf("list etcd members: %v", err) + return nil, err } names := make([]string, 0, len(list.Items)) for i := range list.Items { names = append(names, list.Items[i].Name) } sort.Strings(names) - return names + return names, nil } func podExec(ctx context.Context, namespace, pod, container string, command []string) (string, string, error) { diff --git a/test/e2e/member_selfheal_test.go b/test/e2e/member_selfheal_test.go new file mode 100644 index 00000000..e5e61b0d --- /dev/null +++ b/test/e2e/member_selfheal_test.go @@ -0,0 +1,289 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + etcdv1alpha2 "github.com/cozystack/etcd-operator/api/v1alpha2" +) + +const ( + selfHealNamespace = "selfheal-e2e" + selfHealCluster = "etcd" +) + +// TestPVCMemberCrashLoopSelfHeal proves the new self-heal path end to end on a +// real cluster: a PVC-backed member whose data dir is corrupted crash-loops, +// and once it passes the restart threshold (with the other two members holding +// quorum) the operator deletes it, GCs its PVC, and gap-fills a fresh member — +// the cluster returns to 3 ready members and still serves reads/writes. +// +// This exercises etcdContainerStuck + the quorum gate + finalizer MemberRemove +// + PVC owner-ref GC + cluster-controller gap-fill, none of which the unit +// tests (fake client) or the member-churn block in TestKamajiDataStore (normal +// MemberRemove path, never a crash-loop) reach. +// +// It is deliberately slow: CrashLoopBackOff caps backoff at 5m, so reaching the +// 5-restart threshold takes on the order of ten minutes. The waits below are +// sized for that. +func TestPVCMemberCrashLoopSelfHeal(t *testing.T) { + ctx := context.Background() + + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"}, + ObjectMeta: metav1.ObjectMeta{Name: selfHealNamespace}, + } + if err := kube.Patch(ctx, ns, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + t.Fatalf("create namespace %s: %v", selfHealNamespace, err) + } + t.Cleanup(func() { + _ = kube.Delete(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: selfHealNamespace}}) + }) + + // A minimal plaintext 3-member PVC cluster — TLS is orthogonal to the + // self-heal path under test, and plaintext lets etcdctl probe the cluster + // without certs. + three := int32(3) + ec := &etcdv1alpha2.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: selfHealCluster, Namespace: selfHealNamespace}, + Spec: etcdv1alpha2.EtcdClusterSpec{ + Replicas: &three, + Version: "3.6.11", + Storage: etcdv1alpha2.StorageSpec{Size: resource.MustParse("1Gi")}, + }, + } + if err := kube.Create(ctx, ec); err != nil { + t.Fatalf("create EtcdCluster: %v", err) + } + + waitFor(ctx, t, 5*time.Minute, "cluster Available", etcdClusterAvailable(selfHealNamespace, selfHealCluster)) + waitFor(ctx, t, 2*time.Minute, "3 members ready", readyMembersIs(selfHealCluster, 3)) + + original := selfHealMembers(ctx, t) + if len(original) != 3 { + t.Fatalf("expected 3 members, got %d: %v", len(original), original) + } + victim := original[0] + victimPVC := "data-" + victim + t.Logf("corrupting data dir of victim member %q (pvc %q)", victim, victimPVC) + + corruptMemberDataDir(ctx, t, victim) + + // Force the etcd container to restart onto the now-corrupt data dir. PVC + // members re-use their volume across Pod restarts, so the replacement Pod + // reads the corrupted data and etcd crash-loops (it will not fall back to + // --initial-cluster while a data dir is present). + if err := kube.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: victim, Namespace: selfHealNamespace}}); err != nil { + t.Fatalf("delete victim pod %q: %v", victim, err) + } + + // Self-heal: the stuck member is deleted once it crosses the restart + // threshold. Generous timeout for the CrashLoopBackOff ramp. + waitFor(ctx, t, 15*time.Minute, fmt.Sprintf("stuck member %q deleted for replacement", victim), + func(ctx context.Context) error { + err := kube.Get(ctx, client.ObjectKey{Namespace: selfHealNamespace, Name: victim}, &etcdv1alpha2.EtcdMember{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("victim %q still present (crash-loop not yet past threshold)", victim) + }) + + // The corrupt member's PVC must be GC'd (owner-ref), discarding the bad + // data dir so the replacement starts clean. + waitFor(ctx, t, 5*time.Minute, fmt.Sprintf("victim PVC %q GC'd", victimPVC), + func(ctx context.Context) error { + err := kube.Get(ctx, client.ObjectKey{Namespace: selfHealNamespace, Name: victimPVC}, &corev1.PersistentVolumeClaim{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("PVC %q still present", victimPVC) + }) + + // A fresh member (new GenerateName, not the victim) gap-fills and the + // cluster returns to 3 ready. + waitFor(ctx, t, 10*time.Minute, "cluster back to 3 ready members with a fresh replacement", + func(ctx context.Context) error { + if err := readyMembersIs(selfHealCluster, 3)(ctx); err != nil { + return err + } + names := selfHealMembersErr(ctx) + for _, n := range names { + if n == victim { + return fmt.Errorf("victim %q still in member set: %v", victim, names) + } + } + return nil + }) + + // The replaced cluster still serves reads and writes. + assertEtcdReadWrite(ctx, t) + t.Log("PVC member crash-loop was self-healed; cluster recovered to 3 members and still serves traffic") +} + +// readyMembersIs returns a waitFor condition that the cluster reports `want` +// ready members. +func readyMembersIs(name string, want int32) func(context.Context) error { + return func(ctx context.Context) error { + ec := &etcdv1alpha2.EtcdCluster{} + if err := kube.Get(ctx, client.ObjectKey{Namespace: selfHealNamespace, Name: name}, ec); err != nil { + return err + } + if ec.Status.ReadyMembers != want { + return fmt.Errorf("readyMembers=%d, want %d", ec.Status.ReadyMembers, want) + } + return nil + } +} + +// selfHealMembers returns the cluster's EtcdMember names, failing the test on +// a list error. +func selfHealMembers(ctx context.Context, t *testing.T) []string { + t.Helper() + list := &etcdv1alpha2.EtcdMemberList{} + if err := kube.List(ctx, list, client.InNamespace(selfHealNamespace), + client.MatchingLabels{"etcd-operator.cozystack.io/cluster": selfHealCluster}); err != nil { + t.Fatalf("list members: %v", err) + } + names := make([]string, 0, len(list.Items)) + for i := range list.Items { + names = append(names, list.Items[i].Name) + } + return names +} + +// selfHealMembersErr is the error-tolerant form for use inside waitFor (a list +// error returns an empty slice; the caller's own assertions then retry). +func selfHealMembersErr(ctx context.Context) []string { + list := &etcdv1alpha2.EtcdMemberList{} + if err := kube.List(ctx, list, client.InNamespace(selfHealNamespace), + client.MatchingLabels{"etcd-operator.cozystack.io/cluster": selfHealCluster}); err != nil { + return nil + } + names := make([]string, 0, len(list.Items)) + for i := range list.Items { + names = append(names, list.Items[i].Name) + } + return names +} + +// corruptMemberDataDir overwrites the head of every file in the member's data +// dir with random bytes, via an ephemeral container that mounts the same PVC +// volume. bbolt/WAL files with corrupt headers make etcd panic on startup — +// the data-loss signature the self-heal targets. +func corruptMemberDataDir(ctx context.Context, t *testing.T, member string) { + t.Helper() + pod, err := clientset.CoreV1().Pods(selfHealNamespace).Get(ctx, member, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get victim pod %q: %v", member, err) + } + pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: "corrupt-data", + Image: "busybox:1.36", + Command: []string{"sh", "-c", "find /var/lib/etcd -type f -exec dd if=/dev/urandom of={} bs=4096 count=1 conv=notrunc \\; ; sync; echo corrupted"}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data", MountPath: "/var/lib/etcd"}, + }, + }, + }) + if _, err := clientset.CoreV1().Pods(selfHealNamespace).UpdateEphemeralContainers(ctx, member, pod, metav1.UpdateOptions{}); err != nil { + t.Fatalf("add corrupt-data ephemeral container to %q: %v", member, err) + } + waitFor(ctx, t, 3*time.Minute, "data-dir corruption container finished", func(ctx context.Context) error { + p, err := clientset.CoreV1().Pods(selfHealNamespace).Get(ctx, member, metav1.GetOptions{}) + if err != nil { + return err + } + for _, cs := range p.Status.EphemeralContainerStatuses { + if cs.Name != "corrupt-data" { + continue + } + if cs.State.Terminated != nil { + if cs.State.Terminated.ExitCode != 0 { + t.Fatalf("corrupt-data exited %d: %s", cs.State.Terminated.ExitCode, cs.State.Terminated.Reason) + } + return nil + } + return fmt.Errorf("corrupt-data not finished: %+v", cs.State) + } + return fmt.Errorf("corrupt-data status not reported yet") + }) +} + +// assertEtcdReadWrite puts and reads back a key via etcdctl in a ready member, +// proving the recovered cluster serves traffic. Plaintext endpoint (the test +// cluster has no TLS). +func assertEtcdReadWrite(ctx context.Context, t *testing.T) { + t.Helper() + pods := &corev1.PodList{} + if err := kube.List(ctx, pods, client.InNamespace(selfHealNamespace), + client.MatchingLabels{"etcd-operator.cozystack.io/cluster": selfHealCluster}); err != nil { + t.Fatalf("list member pods: %v", err) + } + var podName string + for i := range pods.Items { + p := &pods.Items[i] + if p.Status.Phase != corev1.PodRunning { + continue + } + for _, cs := range p.Status.ContainerStatuses { + if cs.Name == "etcd" && cs.Ready { + podName = p.Name + break + } + } + if podName != "" { + break + } + } + if podName == "" { + t.Fatalf("no ready etcd member pod to probe") + } + + const key, val = "/e2e/selfheal-probe", "ok" + if _, stderr, err := podExec(ctx, selfHealNamespace, podName, "etcd", []string{ + "etcdctl", "--endpoints=http://localhost:2379", "put", key, val, + }); err != nil { + t.Fatalf("etcdctl put: %v (stderr: %s)", err, stderr) + } + stdout, stderr, err := podExec(ctx, selfHealNamespace, podName, "etcd", []string{ + "etcdctl", "--endpoints=http://localhost:2379", "get", key, "--print-value-only", + }) + if err != nil { + t.Fatalf("etcdctl get: %v (stderr: %s)", err, stderr) + } + if got := trimSpace(stdout); got != val { + t.Fatalf("etcdctl get %q = %q, want %q", key, got, val) + } +} + +// trimSpace strips trailing whitespace/newline from etcdctl output without +// pulling in strings just for this. +func trimSpace(s string) string { + for len(s) > 0 { + c := s[len(s)-1] + if c == '\n' || c == '\r' || c == ' ' || c == '\t' { + s = s[:len(s)-1] + continue + } + break + } + return s +}