From 616238d79da23f2dde231a961ba19bde291abfe2 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 5 Feb 2021 14:25:28 -0500 Subject: [PATCH 1/8] testing samples, ignore push mode samples --- scripts/devops_tasks/test_run_samples.py | 25 ++- .../client_identity_authentication_async.py | 1 + sdk/eventhub/test-resources.json | 194 ++++++++++++++++++ 3 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 sdk/eventhub/test-resources.json diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index c7d1bb7cdbc5..7b951573f5a5 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -25,7 +25,30 @@ "__init__.py", "consume_cloud_events_from_eventhub.py", "consume_cloud_events_from_service_bus_queue.py", - "consume_cloud_events_from_storage_queue.py"] + "consume_cloud_events_from_storage_queue.py"], + "azure-eventhub": [ + "authenticate_with_sas_token.py", + "proxy.py", + "receive_batch_with_checkpoint.py", + "recv.py", + "recv_track_last_enqueued_event_prop.py", + "recv_with_checkpoint_by_event_count.py", + "recv_with_checkpoint_by_time_interval.py", + "recv_with_checkpoint_store.py", + "recv_with_custom_starting_position.py", + "sample_code_eventhub.py", + "authenticate_with_sas_token_async.py", + "iot_hub_connection_string_receive_async.py", + "proxy_async.py", + "receive_batch_with_checkpoint_async.py", + "recv_async.py", + "recv_track_last_enqueued_event_prop_async.py", + "recv_with_checkpoint_by_event_count_async.py", + "recv_with_checkpoint_by_time_interval_async.py", + "recv_with_checkpoint_store_async.py", + "recv_with_custom_starting_position_async.py", + "sample_code_eventhub_async.py" + ] } diff --git a/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py b/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py index 250bdfa1bab1..be34f3461e01 100644 --- a/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py +++ b/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py @@ -60,6 +60,7 @@ async def run(): # New EventDataBatch object can be created here to send more data. break await producer.send_batch(event_data_batch) + await credential.close() loop = asyncio.get_event_loop() loop.run_until_complete(run()) diff --git a/sdk/eventhub/test-resources.json b/sdk/eventhub/test-resources.json new file mode 100644 index 000000000000..93d7a852a0d3 --- /dev/null +++ b/sdk/eventhub/test-resources.json @@ -0,0 +1,194 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "baseName": { + "type": "string", + "defaultValue": "[resourceGroup().name]", + "metadata": { + "description": "The base resource name." + } + }, + "subscriptionId": { + "type": "string", + "defaultValue": "[subscription().subscriptionId]", + "metadata": { + "description": "The subscription ID to which the application and resources belong." + } + }, + "tenantId": { + "type": "string", + "defaultValue": "[subscription().tenantId]", + "metadata": { + "description": "The tenant ID to which the application and resources belong." + } + }, + "testApplicationOid": { + "type": "string", + "metadata": { + "description": "The client OID to grant access to test resources." + } + }, + "testApplicationId": { + "type": "string", + "metadata": { + "description": "The application client ID used to run tests." + } + }, + "testApplicationSecret": { + "type": "string", + "metadata": { + "description": "The application client secret used to run tests." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The location of the resources. By default, this is the same as the resource group." + } + }, + "storageEndpointSuffix": { + "type": "string", + "defaultValue": "core.windows.net", + "metadata": { + "description": "The url suffix to use when creating storage connection strings." + } + }, + "perTestExecutionLimitMinutes": { + "type": "string", + "defaultValue": "10", + "metadata": { + "description": "The maximum duration, in minutes, that a single test is permitted to run before it is considered at-risk for being hung." + } + } + }, + "variables": { + "ehVersion": "2017-04-01", + "contributorRoleId": "b24988ac-6180-42a0-ab88-20f7382dd24c", + "eventHubsDataOwnerRoleId": "f526a384-b230-433a-b45c-95f59c4a2dec", + "eventHubsNamespace": "[concat('eh-', parameters('baseName'))]", + "eventHubName": "[concat('eh-', parameters('baseName'), '-hub')]", + "eventHubAuthRuleName": "[concat('eh-', parameters('baseName'), '-hub-auth-rule')]", + "storageAccount": "[concat('blb', parameters('baseName'))]", + "containerName": "your-blob-container-name", + "defaultSASKeyName": "RootManageSharedAccessKey", + "eventHubsAuthRuleResourceId": "[resourceId('Microsoft.EventHub/namespaces/authorizationRules', variables('eventHubsNamespace'), variables('defaultSASKeyName'))]", + "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccount'))]", + }, + "resources": [ + { + "apiVersion": "[variables('ehVersion')]", + "name": "[variables('eventHubsNamespace')]", + "type": "Microsoft.EventHub/Namespaces", + "location": "[parameters('location')]", + "sku": { + "name": "Standard", + "tier": "Standard" + }, + "resources": [ + { + "type": "Microsoft.EventHub/namespaces/eventhubs", + "apiVersion": "[variables('ehVersion')]", + "name": "[concat(variables('eventHubsNamespace'), '/', variables('eventHubName'))]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces', variables('eventHubsNamespace'))]" + ], + "properties": { + "messageRetentionInDays": 7, + "partitionCount": 1 + }, + "resources": [ + { + "apiVersion": "[variables('ehVersion')]", + "name": "[variables('eventHubAuthRuleName')]", + "type": "authorizationRules", + "dependsOn": [ + "[variables('eventHubName')]" + ], + "properties": { + "Rights": ["Manage", "Send", "Listen"] + } + } + ] + } + ], + "properties": {} + }, + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-06-01", + "name": "[variables('storageAccount')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "accessTier": "Hot" + }, + "resources": [ + { + "type": "blobServices/containers", + "apiVersion": "2019-06-01", + "name": "[concat('default/', variables('containerName'))]", + "dependsOn": [ + "[variables('storageAccount')]" + ] + } + ] + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2019-04-01-preview", + "name": "[guid(resourceGroup().id, deployment().name, parameters('baseName'), variables('eventHubsDataOwnerRoleId'), parameters('testApplicationOid'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('eventHubsDataOwnerRoleId'))]", + "principalId": "[parameters('testApplicationOid')]", + "scope": "[resourceGroup().id]" + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2019-04-01-preview", + "name": "[guid(resourceGroup().id, deployment().name, parameters('baseName'), variables('contributorRoleId'), parameters('testApplicationOid'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('contributorRoleId'))]", + "principalId": "[parameters('testApplicationOid')]", + "scope": "[resourceGroup().id]" + } + } + ], + "outputs": { + "EVENT_HUB_NAMESPACE_CONNECTION_STRING": { + "type": "string", + "value": "[listkeys(variables('eventHubsAuthRuleResourceId'), '2015-08-01').primaryConnectionString]" + }, + "EVENT_HUB_HOSTNAME": { + "type": "string", + "value": "[concat(variables('eventHubsNamespace'), '.servicebus.windows.net')]" + }, + "EVENT_HUB_CONN_STR": { + "type": "string", + "value": "[listkeys(variables('eventHubAuthRuleName'), variables('ehVersion')).primaryConnectionString]" + }, + "EVENT_HUB_NAME": { + "type": "string", + "value": "[variables('eventHubName')]" + }, + "EVENT_HUB_SAS_POLICY":{ + "type": "string", + "value": "[variables('eventHubAuthRuleName')]" + }, + "EVENT_HUB_SAS_KEY":{ + "type": "string", + "value": "[listkeys(variables('eventHubAuthRuleName'), variables('ehVersion')).primaryKey]" + }, + "AZURE_STORAGE_CONN_STR": { + "type": "string", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccount'), ';AccountKey=', listKeys(variables('storageAccountId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value, ';EndpointSuffix=', parameters('storageEndpointSuffix'))]" + } + } +} From b382fcbdeb166a8eddff25989601f2dd5cd70afa Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 5 Feb 2021 14:32:56 -0500 Subject: [PATCH 2/8] update tests.yml for sample testing --- sdk/eventhub/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index da472e8ec61c..d57fca091e95 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -5,6 +5,8 @@ stages: parameters: AllocateResourceGroup: false ServiceDirectory: eventhub + DeployArmTemplate: true + TestSamples: true Matrix: Linux_Python35: Pool: azsdk-pool-mms-ubuntu-1804-general From 5a0434465ac31c9384c1b60f52bf3c9c71f98e46 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Tue, 9 Feb 2021 11:32:25 -0500 Subject: [PATCH 3/8] ignore custom endpoint samples --- scripts/devops_tasks/test_run_samples.py | 2 ++ .../async_samples/client_identity_authentication_async.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 7b951573f5a5..12a893ec3858 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -28,6 +28,7 @@ "consume_cloud_events_from_storage_queue.py"], "azure-eventhub": [ "authenticate_with_sas_token.py", + "connection_to_custom_endpoint_address.py", "proxy.py", "receive_batch_with_checkpoint.py", "recv.py", @@ -38,6 +39,7 @@ "recv_with_custom_starting_position.py", "sample_code_eventhub.py", "authenticate_with_sas_token_async.py", + "connection_to_custom_endpoint_address_async.py", "iot_hub_connection_string_receive_async.py", "proxy_async.py", "receive_batch_with_checkpoint_async.py", diff --git a/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py b/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py index be34f3461e01..250bdfa1bab1 100644 --- a/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py +++ b/sdk/eventhub/azure-eventhub/samples/async_samples/client_identity_authentication_async.py @@ -60,7 +60,6 @@ async def run(): # New EventDataBatch object can be created here to send more data. break await producer.send_batch(event_data_batch) - await credential.close() loop = asyncio.get_event_loop() loop.run_until_complete(run()) From edd6d8b31f821aefc91ed83646aa5795e3ea14ed Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Wed, 10 Feb 2021 15:24:21 -0500 Subject: [PATCH 4/8] use env vars defined, not arm template --- sdk/eventhub/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index d57fca091e95..646bb98d1611 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -5,7 +5,7 @@ stages: parameters: AllocateResourceGroup: false ServiceDirectory: eventhub - DeployArmTemplate: true + BuildTargetingString: azure-eventhub* TestSamples: true Matrix: Linux_Python35: From fb6c69d26df70c2f030a4630a183d9859324c348 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Wed, 10 Feb 2021 16:02:08 -0500 Subject: [PATCH 5/8] added more resources to arm template --- sdk/eventhub/test-resources.json | 12 ++++++++++-- sdk/eventhub/tests.yml | 10 +--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk/eventhub/test-resources.json b/sdk/eventhub/test-resources.json index 93d7a852a0d3..3566eb9745f9 100644 --- a/sdk/eventhub/test-resources.json +++ b/sdk/eventhub/test-resources.json @@ -162,9 +162,9 @@ } ], "outputs": { - "EVENT_HUB_NAMESPACE_CONNECTION_STRING": { + "EVENT_HUB_NAMESPACE": { "type": "string", - "value": "[listkeys(variables('eventHubsAuthRuleResourceId'), '2015-08-01').primaryConnectionString]" + "value": "[variables('eventHubsNamespace')]" }, "EVENT_HUB_HOSTNAME": { "type": "string", @@ -189,6 +189,14 @@ "AZURE_STORAGE_CONN_STR": { "type": "string", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccount'), ';AccountKey=', listKeys(variables('storageAccountId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value, ';EndpointSuffix=', parameters('storageEndpointSuffix'))]" + }, + "AZURE_STORAGE_ACCOUNT": { + "type": "string", + "value": "[variables('storageAccount')]" + }, + "AZURE_STORAGE_ACCESS_KEY":{ + "type": "string", + "value": "[listKeys(variables('storageAccountId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value]" } } } diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index 646bb98d1611..4ed17d3117aa 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -6,6 +6,7 @@ stages: AllocateResourceGroup: false ServiceDirectory: eventhub BuildTargetingString: azure-eventhub* + DeployArmTemplate: true TestSamples: true Matrix: Linux_Python35: @@ -29,16 +30,7 @@ stages: PythonVersion: '3.7' CoverageArg: '--disablecov' EnvVars: - AZURE_STORAGE_ACCOUNT: $(python-eh-livetest-event-hub-storage-account) - AZURE_STORAGE_ACCESS_KEY: $(python-eh-livetest-event-hub-storage-access-key) - AZURE_STORAGE_CONN_STR: $(python-eh-livetest-event-hub-storage-conn-str) AZURE_STORAGE_DATA_LAKE_ENABLED_CONN_STR: $(python-eh-livetest-event-hub-storage-data-lake-enabled-conn-str) - EVENT_HUB_CONN_STR: $(python-eh-livetest-event-hub-conn-str) - EVENT_HUB_HOSTNAME: $(python-eh-livetest-event-hub-hostname) - EVENT_HUB_NAME: $(python-eh-livetest-event-hub-name) - EVENT_HUB_SAS_POLICY: $(python-eh-livetest-event-hub-sas-policy) - EVENT_HUB_SAS_KEY: $(python-eh-livetest-event-hub-sas-key) - EVENT_HUB_NAMESPACE: $(python-eh-livetest-event-hub-namespace) IOTHUB_CONNECTION_STR: $(python-eh-livetest-event-hub-iothub-connection-str) IOTHUB_DEVICE: $(python-eh-livetest-event-hub-iothub-device) AZURE_CLIENT_ID: $(python-eh-livetest-event-hub-aad-client-id) From 0f525a8f6e2a6d4f791bba71c2a61659da2db2de Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Thu, 11 Feb 2021 10:58:52 -0500 Subject: [PATCH 6/8] ignore chkpt samples --- scripts/devops_tasks/test_run_samples.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 12a893ec3858..b8a776986710 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -49,7 +49,11 @@ "recv_with_checkpoint_by_time_interval_async.py", "recv_with_checkpoint_store_async.py", "recv_with_custom_starting_position_async.py", - "sample_code_eventhub_async.py" + "sample_code_eventhub_async.py", + "receive_events_using_checkpoint_store.py", + "receive_events_using_checkpoint_store_storage_api_version.py", + "receive_events_using_checkpoint_store_async.py", + "receive_events_using_checkpoint_store_storage_api_version_async.py" ] } From aec8796ef37b2af666ac9c043c9ecfa8b3401f09 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Thu, 11 Feb 2021 13:34:05 -0500 Subject: [PATCH 7/8] fix ignore chkpt samples --- scripts/devops_tasks/test_run_samples.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index b8a776986710..b3505862705e 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -49,9 +49,13 @@ "recv_with_checkpoint_by_time_interval_async.py", "recv_with_checkpoint_store_async.py", "recv_with_custom_starting_position_async.py", - "sample_code_eventhub_async.py", + "sample_code_eventhub_async.py" + ], + "azure-eventhub-checkpointstoreblob": [ "receive_events_using_checkpoint_store.py", - "receive_events_using_checkpoint_store_storage_api_version.py", + "receive_events_using_checkpoint_store_storage_api_version.py" + ], + "azure-eventhub-checkpointstoreblob-aio": [ "receive_events_using_checkpoint_store_async.py", "receive_events_using_checkpoint_store_storage_api_version_async.py" ] From c71e0a96d5c8609c9e1454e685c4493a0191d75b Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 12 Feb 2021 13:56:57 -0500 Subject: [PATCH 8/8] fix merge conflict for sb ignore samples --- scripts/devops_tasks/test_run_samples.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index b3505862705e..72bab696b8f9 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -58,6 +58,25 @@ "azure-eventhub-checkpointstoreblob-aio": [ "receive_events_using_checkpoint_store_async.py", "receive_events_using_checkpoint_store_storage_api_version_async.py" + ], + "azure-servicebus": [ + "failure_and_recovery.py", + "mgmt_queue.py", + "mgmt_rule.py", + "mgmt_subscription.py", + "mgmt_topic.py", + "proxy.py", + "receive_deferred_message_queue.py", + "receive_iterator_queue.py", + "session_pool_receive.py", + "mgmt_queue_async.py", + "mgmt_rule_async.py", + "mgmt_subscription_async.py", + "mgmt_topic_async.py", + "proxy_async.py", + "receive_deferred_message_queue_async.py", + "receive_iterator_queue_async.py", + "session_pool_receive_async.py" ] }