backport of commit 0187338 (#31441)

Co-authored-by: Charles Nwokotubo <charles.nwokotubo@hashicorp.com>

Author: hc-github-team-secure-vault-core

enos/enos-descriptions.hcl

@@ -181,7 +181,7 @@ globals {
     EOF
 
     verify_secrets_engines_read = <<-EOF
-      Verify that data that we've created previously is still valid, consistent, and duarable.
+      Verify that data that we've created previously is still valid, consistent, and durable.
       This includes:
         - v1/auth/userpass/*
         - v1/identity/*
@@ -191,6 +191,12 @@ globals {
         - v1/ldap/*
     EOF
 
+    verify_secrets_engines_delete = <<-EOF
+      Verify that data that we've created previously can be deleted with no errors.
+      This includes:
+        - v1/ssh/*
+    EOF
+
     verify_ui = <<-EOF
       The Vault UI assets are embedded in the Vault binary and available when running.
     EOF

enos/enos-dev-scenario-pr-replication.hcl

@@ -679,6 +679,8 @@ scenario "dev_pr_replication" {
     }
 
     variables {
+      ports             = global.ports
+      ipv4_cidr         = step.create_vpc.ipv4_cidr
       hosts             = step.create_primary_cluster_targets.hosts
       leader_host       = step.get_primary_cluster_ips.leader_host
       vault_addr        = step.create_primary_cluster.api_addr_localhost

enos/enos-modules.hcl

@@ -334,6 +334,12 @@ module "vault_verify_secrets_engines_read" {
   vault_install_dir         = var.vault_install_dir
 }
 
+module "vault_verify_secrets_engines_delete" {
+  source = "./modules/verify_secrets_engines/modules/delete"
+
+  vault_install_dir = var.vault_install_dir
+}
+
 module "vault_verify_default_lcq" {
   source = "./modules/vault_verify_default_lcq"
 

enos/enos-qualities.hcl

@@ -147,6 +147,10 @@ quality "vault_api_sys_auth_userpass_user_write" {
   description = "The v1/sys/auth/userpass/users/<user> Vault API associates a superuser policy with a user"
 }
 
+quality "vault_api_ssh_role_delete" {
+  description = "The v1/ssh/role Vault API deletes an SSH role associated with a key and clients"
+}
+
 quality "vault_api_sys_config_read" {
   description = <<-EOF
     The v1/sys/config/sanitized Vault API returns sanitized configuration which matches our given

enos/enos-scenario-agent.hcl

@@ -549,7 +549,8 @@ scenario "agent" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_vault_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_vault_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = global.vault_install_dir[matrix.artifact_type]
@@ -660,6 +661,32 @@ scenario "agent" {
     }
   }
 
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_create,
+      step.verify_secrets_engines_read
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state      = step.verify_secrets_engines_create.state
+      hosts             = step.get_vault_cluster_ips.follower_hosts
+      leader_host       = step.get_vault_cluster_ips.leader_host
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-autopilot.hcl

@@ -417,7 +417,8 @@ scenario "autopilot" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_vault_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_vault_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = local.vault_install_dir
@@ -674,6 +675,32 @@ scenario "autopilot" {
     }
   }
 
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_create,
+      step.verify_secrets_engines_read
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state      = step.verify_secrets_engines_create.state
+      hosts             = step.get_vault_cluster_ips.follower_hosts
+      leader_host       = step.get_vault_cluster_ips.leader_host
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   step "raft_remove_peers" {
     description = <<-EOF
       Remove the nodes that were running the prior version of Vault from the raft cluster

enos/enos-scenario-dr-replication.hcl

@@ -748,7 +748,8 @@ scenario "dr_replication" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_primary_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_primary_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = global.vault_install_dir[matrix.artifact_type]
@@ -1159,6 +1160,7 @@ scenario "dr_replication" {
       vault_root_token        = step.create_secondary_cluster.root_token
       verify_pki_certs        = false
       verify_aws_engine_creds = false
+      verify_ssh_secrets      = false
     }
   }
 
@@ -1300,6 +1302,39 @@ scenario "dr_replication" {
       vault_root_token        = step.create_secondary_cluster.root_token
       verify_pki_certs        = false
       verify_aws_engine_creds = false
+      verify_ssh_secrets      = false
+    }
+  }
+
+  # Verification is intentionally disabled for DR clusters because they do not handle client requests.
+  # However, we still include this step to future-proof the module, making it easier to enable delete
+  # verification later for other secrets engines or use cases where DR verification becomes relevant.
+  # For now, the script will short-circuit if verification is disabled. Potential future work will include
+  # verifying against the primary cluster.
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_on_primary,
+      step.verify_failover_replicated_data
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state       = step.verify_secrets_engines_on_primary.state
+      hosts              = step.get_secondary_cluster_ips.follower_hosts
+      leader_host        = step.get_secondary_cluster_ips.leader_host
+      vault_addr         = step.create_secondary_cluster.api_addr_localhost
+      vault_install_dir  = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token   = step.create_secondary_cluster.root_token
+      verify_ssh_secrets = false
     }
   }
 

enos/enos-scenario-pr-replication.hcl

@@ -770,7 +770,8 @@ scenario "pr_replication" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_primary_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_primary_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = global.vault_install_dir[matrix.artifact_type]
@@ -991,6 +992,34 @@ scenario "pr_replication" {
       vault_root_token        = step.create_secondary_cluster.root_token
       verify_pki_certs        = false
       verify_aws_engine_creds = false
+      verify_ssh_secrets      = false
+    }
+  }
+
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_on_primary,
+      step.verify_replicated_data
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state       = step.verify_secrets_engines_on_primary.state
+      hosts              = step.get_secondary_cluster_ips.follower_hosts
+      leader_host        = step.get_secondary_cluster_ips.leader_host
+      vault_addr         = step.create_secondary_cluster.api_addr_localhost
+      vault_install_dir  = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token   = step.create_secondary_cluster.root_token
+      verify_ssh_secrets = false
     }
   }
 

enos/enos-scenario-proxy.hcl

@@ -525,7 +525,8 @@ scenario "proxy" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_vault_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_vault_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = global.vault_install_dir[matrix.artifact_type]
@@ -636,6 +637,32 @@ scenario "proxy" {
     }
   }
 
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_create,
+      step.verify_secrets_engines_read
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state      = step.verify_secrets_engines_create.state
+      hosts             = step.get_vault_cluster_ips.follower_hosts
+      leader_host       = step.get_vault_cluster_ips.leader_host
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-seal-ha.hcl

@@ -505,7 +505,8 @@ scenario "seal_ha" {
       ip_version             = matrix.ip_version
       integration_host_state = step.set_up_external_integration_target.state
       leader_host            = step.get_vault_cluster_ips.leader_host
-      ports                  = global.integration_host_ports
+      ports                  = global.ports
+      ipv4_cidr              = step.create_vpc.ipv4_cidr
       vault_addr             = step.create_vault_cluster.api_addr_localhost
       vault_edition          = matrix.edition
       vault_install_dir      = global.vault_install_dir[matrix.artifact_type]
@@ -1091,6 +1092,32 @@ scenario "seal_ha" {
     }
   }
 
+  step "verify_secrets_engines_delete" {
+    description = global.description.verify_secrets_engines_delete
+    module      = module.vault_verify_secrets_engines_delete
+    depends_on = [
+      step.verify_secrets_engines_create,
+      step.verify_secrets_engines_read_after_migration
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_ssh_role_delete
+    ]
+
+    variables {
+      create_state      = step.verify_secrets_engines_create.state
+      hosts             = step.get_vault_cluster_ips.follower_hosts
+      leader_host       = step.get_vault_cluster_ips.leader_host
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   // Make sure we have our secondary seal type after migration
   step "verify_seal_type_after_migration" {
     // Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches.