diff --git a/scripts/_cleanup_utils.py b/scripts/_cleanup_utils.py new file mode 100644 index 00000000..a03e7649 --- /dev/null +++ b/scripts/_cleanup_utils.py @@ -0,0 +1,86 @@ +"""Shared helpers for the orphaned integration-test resource cleanup scripts. + +Both cleanup scripts (memories, gateways) share the same shape: list everything +in a shared test account, keep what's protected/non-test/too-recent, and delete +the rest — dry-run by default. This module holds that common scaffolding so each +script only declares its resource-specific bits (prefixes, list/delete calls). +""" + +import argparse +import datetime + + +def build_parser(description, noun): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--region", default="us-west-2", help="AWS region (default: us-west-2)") + parser.add_argument( + "--min-age-days", + type=float, + default=1.0, + help=f"Only delete {noun} older than this many days (default: 1)", + ) + parser.add_argument("--apply", action="store_true", help="Actually delete (default is dry-run)") + return parser + + +def paginate(client, operation, items_key, **kwargs): + """Yield all items across pages of a list_* operation that uses nextToken.""" + token = None + while True: + page = getattr(client, operation)(**kwargs, **({"nextToken": token} if token else {})) + yield from page.get(items_key, []) + token = page.get("nextToken") + if not token: + return + + +def run_cleanup(noun, items, *, label_of, created_of, is_test, delete_one, args, is_protected=None): + """Bucket items into kept/deleted, print a report, and delete (unless dry-run).""" + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=args.min_age_days) + + to_delete, protected, recent, nonmatch = [], [], [], [] + for item in items: + if is_protected and is_protected(item): + protected.append(item) + elif not is_test(item): + nonmatch.append(item) + elif created_of(item) and created_of(item) > cutoff: + recent.append(item) + else: + to_delete.append(item) + + print(f"Region: {args.region}") + print(f"Total {noun}: {len(items)}") + if is_protected: + print(f" protected (kept): {len(protected)}") + print(f" non-test (kept): {len(nonmatch)}") + print(f" too recent (kept): {len(recent)} (younger than {args.min_age_days}d)") + print(f" orphaned test (delete): {len(to_delete)}") + print() + + if not to_delete: + print("Nothing to delete.") + return 0 + + print(f"=== {'DELETING' if args.apply else 'DRY RUN (would delete)'} {len(to_delete)} {noun} ===") + failures = 0 + for item in to_delete: + label = label_of(item) + if not args.apply: + print(f" would delete: {label}") + continue + try: + delete_one(item) + print(f" deleted: {label}") + except Exception as e: # noqa: BLE001 - best-effort cleanup + failures += 1 + print(f" FAILED: {label} ({e})") + + if not args.apply: + print("\nDry run only. Re-run with --apply to delete.") + elif failures: + print(f"\nCompleted with {failures} failures.") + return 1 + else: + print(f"\nDeleted {len(to_delete)} orphaned {noun}.") + return 0 diff --git a/scripts/cleanup_orphaned_test_gateways.py b/scripts/cleanup_orphaned_test_gateways.py new file mode 100755 index 00000000..5a7960b0 --- /dev/null +++ b/scripts/cleanup_orphaned_test_gateways.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Clean up orphaned AgentCore Gateway resources left behind by integration tests. + +The gateway integration tests run in a shared AWS account. Gateways and their +targets are normally torn down, but interrupted or failed runs leave orphans +behind (a target that fails to reach READY can leave the parent gateway +undeleted). These accumulate over time and clutter the account. + +This script lists every gateway in the account and deletes the ones that look +like orphaned test fixtures, deleting their targets first so the gateway delete +succeeds. + +Safety: + * Dry-run by default. Pass ``--apply`` to actually delete. + * Only deletes gateways whose name starts with a known test prefix AND that are + older than ``--min-age-days`` (default 1 day), so it never races a live run. + +Usage: + python scripts/cleanup_orphaned_test_gateways.py # dry run + python scripts/cleanup_orphaned_test_gateways.py --apply # delete + python scripts/cleanup_orphaned_test_gateways.py --region us-west-2 --min-age-days 2 --apply +""" + +import sys +import time + +import boto3 +from _cleanup_utils import build_parser, paginate, run_cleanup + +# Gateway name prefix created by the integration test suites. Only gateways whose +# name starts with this are eligible for deletion. +TEST_PREFIX = "sdk-integ-" + + +def delete_gateway(client, gateway_id, timeout_s=120, poll_s=5): + """Delete a gateway's targets, wait for the async deletes to settle, then delete it. + + The parent gateway cannot be deleted until its targets are fully removed. + """ + + def target_ids(): + return [ + t["targetId"] + for t in paginate(client, "list_gateway_targets", "items", gatewayIdentifier=gateway_id, maxResults=100) + ] + + for tid in target_ids(): + client.delete_gateway_target(gatewayIdentifier=gateway_id, targetId=tid) + + deadline = time.monotonic() + timeout_s + while target_ids(): + if time.monotonic() > deadline: + raise TimeoutError(f"targets for gateway {gateway_id} not deleted within {timeout_s}s") + time.sleep(poll_s) + + client.delete_gateway(gatewayIdentifier=gateway_id) + + +def main() -> int: + args = build_parser(__doc__, "gateways").parse_args() + client = boto3.client("bedrock-agentcore-control", region_name=args.region) + + return run_cleanup( + "gateways", + list(paginate(client, "list_gateways", "items", maxResults=100)), + label_of=lambda g: f"{g.get('name', '')} ({g.get('gatewayId')})", + created_of=lambda g: g.get("createdAt"), + is_test=lambda g: g.get("name", "").startswith(TEST_PREFIX), + delete_one=lambda g: delete_gateway(client, g.get("gatewayId")), + args=args, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/cleanup_orphaned_test_memories.py b/scripts/cleanup_orphaned_test_memories.py new file mode 100755 index 00000000..c9bbd47b --- /dev/null +++ b/scripts/cleanup_orphaned_test_memories.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Clean up orphaned AgentCore Memory resources left behind by integration tests. + +The integration tests run in a shared AWS account. Memories created by tests are +normally torn down, but interrupted or failed runs leave orphans behind. Once the +account accumulates more than ~100 memories, ``test_list_memories`` (and any other +test that lists with the default page cap) can flake. + +This script lists every memory in the account and deletes the ones that look like +orphaned test fixtures, while protecting long-lived memories that tests depend on +(e.g. the pre-populated memory referenced by the ``MEMORY_PREPOPULATED_ID`` secret). + +Safety: + * Dry-run by default. Pass ``--apply`` to actually delete. + * Only deletes memories whose id starts with a known test prefix AND that are + older than ``--min-age-days`` (default 1 day), so it never races a live run. + * Never deletes protected memories (see PROTECTED_SUBSTRINGS). + +Usage: + python scripts/cleanup_orphaned_test_memories.py # dry run + python scripts/cleanup_orphaned_test_memories.py --apply # delete + python scripts/cleanup_orphaned_test_memories.py --region us-west-2 --min-age-days 2 --apply +""" + +import sys + +import boto3 +from _cleanup_utils import build_parser, paginate, run_cleanup + +# Memory id prefixes created by the integration test suites. Only memories whose +# id starts with one of these are eligible for deletion. +TEST_PREFIXES = ("test_cp_", "mc_2026", "memory_") + +# Substrings of memory ids that must never be deleted (referenced by CI secrets). +PROTECTED_SUBSTRINGS = ("prepopulated", "observability") + + +def _id(memory): + return memory.get("id") or memory.get("memoryId") or "" + + +def main() -> int: + args = build_parser(__doc__, "memories").parse_args() + client = boto3.client("bedrock-agentcore-control", region_name=args.region) + + return run_cleanup( + "memories", + list(paginate(client, "list_memories", "memories", maxResults=100)), + label_of=_id, + created_of=lambda m: m.get("createdAt"), + is_test=lambda m: _id(m).startswith(TEST_PREFIXES), + is_protected=lambda m: any(s in _id(m) for s in PROTECTED_SUBSTRINGS), + delete_one=lambda m: client.delete_memory(memoryId=_id(m)), + args=args, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests_integ/gateway/test_gateway_kb_targets.py b/tests_integ/gateway/test_gateway_kb_targets.py index 000cf746..ecc44d65 100644 --- a/tests_integ/gateway/test_gateway_kb_targets.py +++ b/tests_integ/gateway/test_gateway_kb_targets.py @@ -1,16 +1,15 @@ """Integration tests for GatewayClient KB target helper methods. Requires environment variables: - BEDROCK_TEST_REGION: AWS region (default: us-east-1) + BEDROCK_TEST_REGION: AWS region (default: us-west-2) GATEWAY_ROLE_ARN: IAM role ARN with AgentCore gateway trust policy - KB_ROLE_ARN: IAM role ARN with bedrock:InvokeModel, s3:*, and s3vectors:* permissions + KB_ROLE_ARN: IAM role ARN with bedrock:InvokeModel permissions for the embedding model """ import os import time import uuid -import boto3 import pytest from bedrock_agentcore.gateway.client import GatewayClient @@ -37,36 +36,18 @@ def setup_class(cls): cls.kb_id = None cls.target_ids = [] - # Create S3 Vectors resources - cls.s3vectors_client = boto3.client("s3vectors", region_name=cls.region) - cls.vector_bucket_name = f"kb-gw-integ-vb-{cls.test_suffix}" - cls.vector_index_name = f"kb-gw-integ-idx-{cls.test_suffix}" - cls.s3vectors_client.create_vector_bucket(vectorBucketName=cls.vector_bucket_name) - index_resp = cls.s3vectors_client.create_index( - vectorBucketName=cls.vector_bucket_name, - indexName=cls.vector_index_name, - dataType="float32", - dimension=1024, - distanceMetric="cosine", - ) - cls.index_arn = index_resp["indexArn"] - - # Create Knowledge Base + # Create a MANAGED knowledge base. Gateway KB targets require the MANAGED + # type; Bedrock owns the vector store internally, so no storageConfiguration + # or self-provisioned S3 Vectors index is needed. cls.kb = cls.kb_client.create_knowledge_base_and_wait( name=f"{cls.test_prefix}-kb", roleArn=cls.kb_role_arn, knowledgeBaseConfiguration={ - "type": "VECTOR", - "vectorKnowledgeBaseConfiguration": { + "type": "MANAGED", + "managedKnowledgeBaseConfiguration": { "embeddingModelArn": f"arn:aws:bedrock:{cls.region}::foundation-model/amazon.titan-embed-text-v2:0", }, }, - storageConfiguration={ - "type": "S3_VECTORS", - "s3VectorsConfiguration": { - "indexArn": cls.index_arn, - }, - }, ) cls.kb_id = cls.kb["knowledgeBaseId"] @@ -105,16 +86,6 @@ def teardown_class(cls): except Exception as e: print(f"Failed to delete KB {cls.kb_id}: {e}") - # Delete S3 Vectors - try: - cls.s3vectors_client.delete_index( - vectorBucketName=cls.vector_bucket_name, - indexName=cls.vector_index_name, - ) - cls.s3vectors_client.delete_vector_bucket(vectorBucketName=cls.vector_bucket_name) - except Exception as e: - print(f"Failed to clean up vector bucket: {e}") - @pytest.mark.order(1) def test_create_knowledge_base_target_minimal(self): target = self.gateway_client.create_knowledge_base_target( diff --git a/tests_integ/memory/test_memory_client.py b/tests_integ/memory/test_memory_client.py index 9facf82e..8b980a8f 100644 --- a/tests_integ/memory/test_memory_client.py +++ b/tests_integ/memory/test_memory_client.py @@ -437,7 +437,10 @@ def test_create_memory_and_wait(self): def test_list_memories(self): if not getattr(self, "lifecycle_memory_id", None): pytest.skip("create test did not run") - memories = self.client.list_memories() + # Paginate through all memories. The shared integ-test account can hold + # more than the default 100-item cap, so a capped list may not include + # the just-created memory even though it exists. + memories = self.client.list_memories(max_results=10000) assert any( m.get("memoryId") == self.lifecycle_memory_id or m.get("id") == self.lifecycle_memory_id for m in memories )