From abac980f8951d60f05ee8df328f60484e028a961 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Fri, 5 Mar 2021 14:18:29 -0800 Subject: [PATCH 1/4] add optional sample timeout to tox sample runner --- scripts/devops_tasks/test_run_samples.py | 128 +++++++++++++++++++---- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 95ddfec79cf9..3bdbc9bf1679 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -11,6 +11,7 @@ import os import logging from fnmatch import fnmatch +from subprocess import check_call, CalledProcessError, TimeoutExpired from common_tasks import ( run_check_call, process_glob_string, @@ -20,6 +21,22 @@ root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +""" +Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format: +TIMEOUT_SAMPLES = { + "": { + "": (, ) + } +} +""" +TIMEOUT_SAMPLES = { + "azure-eventhub": { + "receive_batch_with_checkpoint.py": (3, True) + } +} + + +# Add your library + sample file if you do not want a particular sample to be run IGNORED_SAMPLES = { "azure-eventgrid": [ "__init__.py", @@ -29,10 +46,20 @@ "sample_publish_events_to_a_topic_using_sas_credential.py", "sample_publish_events_to_a_topic_using_sas_credential_async.py"], "azure-eventhub": [ + "send.py", + "send_async.py", + "send_stream.py", + "send_stream_async.py", + "recv_for_period_async.py", + "client_creation_async.py", + "connection_string_authentication.py", + "connection_string_authentication_async.py", + "client_identity_authentication_async.py", + "client_creation.py", + "client_identity_authentication.py", "authenticate_with_sas_token.py", "connection_to_custom_endpoint_address.py", "proxy.py", - "receive_batch_with_checkpoint.py", "recv.py", "recv_track_last_enqueued_event_prop.py", "recv_with_checkpoint_by_event_count.py", @@ -87,45 +114,104 @@ } +def run_check_call_with_timeout( + command_array, + working_directory, + acceptable_return_codes=[], + always_exit=False, + timeout=None, + pass_if_timeout=False +): + """This is copied from common_tasks.py with some additions. + Don't want to break anyone that's using the original code. + """ + try: + logging.info( + "Command Array: {0}, Target Working Directory: {1}".format( + command_array, working_directory + ) + ) + check_call(command_array, cwd=working_directory, timeout=timeout) + except CalledProcessError as err: + if err.returncode not in acceptable_return_codes: + logging.error(err) # , file = sys.stderr + if always_exit: + exit(1) + else: + return err + except TimeoutExpired as err: + if pass_if_timeout: + logging.info( + "Sample timed out successfully" + ) + else: + logging.info( + "Fail: Sample timed out" + ) + return err + + +def execute_sample(sample, samples_errors, timed): + if isinstance(sample, tuple): + sample, timeout, pass_if_timeout = sample + + if sys.version_info < (3, 5) and sample.endswith("_async.py"): + return + + logging.info( + "Testing {}".format(sample) + ) + command_array = [sys.executable, sample] + + if not timed: + errors = run_check_call(command_array, root_dir) + else: + errors = run_check_call_with_timeout( + command_array, root_dir, timeout=timeout, pass_if_timeout=pass_if_timeout + ) + + sample_name = os.path.basename(sample) + if errors: + samples_errors.append(sample_name) + logging.info( + "ERROR: {}".format(sample_name) + ) + else: + logging.info( + "SUCCESS: {}.".format(sample_name) + ) + + def run_samples(targeted_package): logging.info("running samples for {}".format(targeted_package)) samples_errors = [] sample_paths = [] + timed_sample_paths = [] + samples_dir_path = os.path.abspath(os.path.join(targeted_package, "samples")) package_name = os.path.basename(targeted_package) + samples_need_timeout = TIMEOUT_SAMPLES.get(package_name, {}) for path, subdirs, files in os.walk(samples_dir_path): for name in files: - if fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []): + if fnmatch(name, "*.py") and name in samples_need_timeout: + timeout, pass_if_timeout = samples_need_timeout[name] + timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout)) + elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []): sample_paths.append(os.path.abspath(os.path.join(path, name))) - if not sample_paths: + if not sample_paths and not timed_sample_paths: logging.info( "No samples found in {}".format(targeted_package) ) exit(0) for sample in sample_paths: - if sys.version_info < (3, 5) and sample.endswith("_async.py"): - continue + execute_sample(sample, samples_errors, timed=False) - logging.info( - "Testing {}".format(sample) - ) - command_array = [sys.executable, sample] - errors = run_check_call(command_array, root_dir, always_exit=False) - - sample_name = os.path.basename(sample) - if errors: - samples_errors.append(sample_name) - logging.info( - "ERROR: {}".format(sample_name) - ) - else: - logging.info( - "SUCCESS: {}.".format(sample_name) - ) + for sample in timed_sample_paths: + execute_sample(sample, samples_errors, timed=True) if samples_errors: logging.error("Sample(s) that ran with errors: {}".format(samples_errors)) From d44087a70f0665c7377c1d2b1958c19d14e8d104 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Tue, 9 Mar 2021 10:36:33 -0500 Subject: [PATCH 2/4] updated sample timeout to install sample specific dependencies reqs --- scripts/devops_tasks/test_run_samples.py | 91 ++++++++++--------- .../samples/sample_dev_requirements.txt | 2 + 2 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 3bdbc9bf1679..78051e25f436 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -31,7 +31,43 @@ """ TIMEOUT_SAMPLES = { "azure-eventhub": { - "receive_batch_with_checkpoint.py": (3, True) + "authenticate_with_sas_token.py": (5, True), + "receive_batch_with_checkpoint.py": (5, True), + "recv.py": (5, True), + "recv_track_last_enqueued_event_prop.py": (5, True), + "recv_with_checkpoint_by_event_count.py": (5, True), + "recv_with_checkpoint_by_time_interval.py": (5, True), + "recv_with_checkpoint_store.py": (5, True), + "recv_with_custom_starting_position.py": (5, True), + "sample_code_eventhub.py": (10, True), + "authenticate_with_sas_token_async.py": (5, True), + "receive_batch_with_checkpoint_async.py": (5, True), + "recv_async.py": (5, True), + "recv_track_last_enqueued_event_prop_async.py": (5, True), + "recv_with_checkpoint_by_event_count_async.py": (5, True), + "recv_with_checkpoint_by_time_interval_async.py": (5, True), + "recv_with_checkpoint_store_async.py": (5, True), + "recv_with_custom_starting_position_async.py": (5, True), + "sample_code_eventhub_async.py": (10, True) + }, + "azure-eventhub-checkpointstoreblob": { + "receive_events_using_checkpoint_store.py": (5, True), + "receive_events_using_checkpoint_store_storage_api_version.py": (5, True) + }, + "azure-eventhub-checkpointstoreblob-aio": { + "receive_events_using_checkpoint_store_async.py": (5, True), + "receive_events_using_checkpoint_store_storage_api_version_async.py": (5, True) + }, + "azure-servicebus": { + "auto_lock_renew.py": (120, True), + "failure_and_recovery.py": (5, True), + "receive_iterator_queue.py": (5, True), + "sample_code_servicebus.py": (30, True), + "session_pool_receive.py": (20, True), + "auto_lock_renew_async.py": (120, True), + "receive_iterator_queue_async.py": (5, True), + "sample_code_servicebus_async.py": (30, True), + "session_pool_receive_async.py": (20, True) } } @@ -46,66 +82,25 @@ "sample_publish_events_to_a_topic_using_sas_credential.py", "sample_publish_events_to_a_topic_using_sas_credential_async.py"], "azure-eventhub": [ - "send.py", - "send_async.py", - "send_stream.py", - "send_stream_async.py", - "recv_for_period_async.py", - "client_creation_async.py", - "connection_string_authentication.py", - "connection_string_authentication_async.py", - "client_identity_authentication_async.py", - "client_creation.py", - "client_identity_authentication.py", - "authenticate_with_sas_token.py", "connection_to_custom_endpoint_address.py", "proxy.py", - "recv.py", - "recv_track_last_enqueued_event_prop.py", - "recv_with_checkpoint_by_event_count.py", - "recv_with_checkpoint_by_time_interval.py", - "recv_with_checkpoint_store.py", - "recv_with_custom_starting_position.py", - "sample_code_eventhub.py", - "authenticate_with_sas_token_async.py", "connection_to_custom_endpoint_address_async.py", "iot_hub_connection_string_receive_async.py", - "proxy_async.py", - "receive_batch_with_checkpoint_async.py", - "recv_async.py", - "recv_track_last_enqueued_event_prop_async.py", - "recv_with_checkpoint_by_event_count_async.py", - "recv_with_checkpoint_by_time_interval_async.py", - "recv_with_checkpoint_store_async.py", - "recv_with_custom_starting_position_async.py", - "sample_code_eventhub_async.py" - ], - "azure-eventhub-checkpointstoreblob": [ - "receive_events_using_checkpoint_store.py", - "receive_events_using_checkpoint_store_storage_api_version.py" - ], - "azure-eventhub-checkpointstoreblob-aio": [ - "receive_events_using_checkpoint_store_async.py", - "receive_events_using_checkpoint_store_storage_api_version_async.py" + "proxy_async.py" ], "azure-servicebus": [ - "failure_and_recovery.py", "mgmt_queue.py", "mgmt_rule.py", "mgmt_subscription.py", "mgmt_topic.py", "proxy.py", "receive_deferred_message_queue.py", - "receive_iterator_queue.py", - "session_pool_receive.py", "mgmt_queue_async.py", "mgmt_rule_async.py", "mgmt_subscription_async.py", "mgmt_topic_async.py", "proxy_async.py", - "receive_deferred_message_queue_async.py", - "receive_iterator_queue_async.py", - "session_pool_receive_async.py" + "receive_deferred_message_queue_async.py" ], "azure-ai-formrecognizer": [ "sample_recognize_receipts_from_url.py", @@ -193,6 +188,14 @@ def run_samples(targeted_package): package_name = os.path.basename(targeted_package) samples_need_timeout = TIMEOUT_SAMPLES.get(package_name, {}) + # install extra dependencies for samples if needed + try: + with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs: + for dep in sample_dev_reqs.readlines(): + check_call([sys.executable, '-m', 'pip', 'install', dep]) + except: + pass + for path, subdirs, files in os.walk(samples_dir_path): for name in files: if fnmatch(name, "*.py") and name in samples_need_timeout: diff --git a/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt b/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt new file mode 100644 index 000000000000..1fe63a3bd091 --- /dev/null +++ b/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt @@ -0,0 +1,2 @@ +azure-eventhub-checkpointstoreblob +azure-eventhub-checkpointstoreblob-aio \ No newline at end of file From 482e8002563e8c2a49dbf119aef0e73c0174b06e Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Tue, 9 Mar 2021 13:48:28 -0500 Subject: [PATCH 3/4] except with FileNotFoundError --- scripts/devops_tasks/test_run_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 78051e25f436..4c5c8a665232 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -193,7 +193,7 @@ def run_samples(targeted_package): with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs: for dep in sample_dev_reqs.readlines(): check_call([sys.executable, '-m', 'pip', 'install', dep]) - except: + except FileNotFoundError: pass for path, subdirs, files in os.walk(samples_dir_path): From 73f53b624c408080feaa563614f992fc4c5e247c Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Thu, 11 Mar 2021 13:46:04 -0500 Subject: [PATCH 4/4] made timeout a required param and pass_if_timeout default to True --- scripts/devops_tasks/test_run_samples.py | 77 ++++++++++++------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 4c5c8a665232..a42ff7b456e9 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -25,49 +25,47 @@ Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format: TIMEOUT_SAMPLES = { "": { - "": (, ) + "": (, ) } } """ TIMEOUT_SAMPLES = { "azure-eventhub": { - "authenticate_with_sas_token.py": (5, True), - "receive_batch_with_checkpoint.py": (5, True), - "recv.py": (5, True), - "recv_track_last_enqueued_event_prop.py": (5, True), - "recv_with_checkpoint_by_event_count.py": (5, True), - "recv_with_checkpoint_by_time_interval.py": (5, True), - "recv_with_checkpoint_store.py": (5, True), - "recv_with_custom_starting_position.py": (5, True), - "sample_code_eventhub.py": (10, True), - "authenticate_with_sas_token_async.py": (5, True), - "receive_batch_with_checkpoint_async.py": (5, True), - "recv_async.py": (5, True), - "recv_track_last_enqueued_event_prop_async.py": (5, True), - "recv_with_checkpoint_by_event_count_async.py": (5, True), - "recv_with_checkpoint_by_time_interval_async.py": (5, True), - "recv_with_checkpoint_store_async.py": (5, True), - "recv_with_custom_starting_position_async.py": (5, True), - "sample_code_eventhub_async.py": (10, True) + "authenticate_with_sas_token.py": (5), + "receive_batch_with_checkpoint.py": (5), + "recv.py": (5), + "recv_track_last_enqueued_event_prop.py": (5), + "recv_with_checkpoint_by_event_count.py": (5), + "recv_with_checkpoint_by_time_interval.py": (5), + "recv_with_checkpoint_store.py": (5), + "recv_with_custom_starting_position.py": (5), + "sample_code_eventhub.py": (10), + "authenticate_with_sas_token_async.py": (5), + "receive_batch_with_checkpoint_async.py": (5), + "recv_async.py": (5), + "recv_track_last_enqueued_event_prop_async.py": (5), + "recv_with_checkpoint_by_event_count_async.py": (5), + "recv_with_checkpoint_by_time_interval_async.py": (5), + "recv_with_checkpoint_store_async.py": (5), + "recv_with_custom_starting_position_async.py": (5), + "sample_code_eventhub_async.py": (10) }, "azure-eventhub-checkpointstoreblob": { - "receive_events_using_checkpoint_store.py": (5, True), - "receive_events_using_checkpoint_store_storage_api_version.py": (5, True) + "receive_events_using_checkpoint_store.py": (5), + "receive_events_using_checkpoint_store_storage_api_version.py": (5) }, "azure-eventhub-checkpointstoreblob-aio": { - "receive_events_using_checkpoint_store_async.py": (5, True), - "receive_events_using_checkpoint_store_storage_api_version_async.py": (5, True) + "receive_events_using_checkpoint_store_async.py": (5), + "receive_events_using_checkpoint_store_storage_api_version_async.py": (5) }, "azure-servicebus": { - "auto_lock_renew.py": (120, True), - "failure_and_recovery.py": (5, True), - "receive_iterator_queue.py": (5, True), - "sample_code_servicebus.py": (30, True), - "session_pool_receive.py": (20, True), - "auto_lock_renew_async.py": (120, True), - "receive_iterator_queue_async.py": (5, True), - "sample_code_servicebus_async.py": (30, True), - "session_pool_receive_async.py": (20, True) + "failure_and_recovery.py": (5), + "receive_iterator_queue.py": (5), + "sample_code_servicebus.py": (30), + "session_pool_receive.py": (20), + "receive_iterator_queue_async.py": (5), + "sample_code_servicebus_async.py": (30), + "session_pool_receive_async.py": (20) } } @@ -112,10 +110,10 @@ def run_check_call_with_timeout( command_array, working_directory, + timeout, + pass_if_timeout, acceptable_return_codes=[], - always_exit=False, - timeout=None, - pass_if_timeout=False + always_exit=False ): """This is copied from common_tasks.py with some additions. Don't want to break anyone that's using the original code. @@ -162,7 +160,7 @@ def execute_sample(sample, samples_errors, timed): errors = run_check_call(command_array, root_dir) else: errors = run_check_call_with_timeout( - command_array, root_dir, timeout=timeout, pass_if_timeout=pass_if_timeout + command_array, root_dir, timeout, pass_if_timeout ) sample_name = os.path.basename(sample) @@ -199,7 +197,12 @@ def run_samples(targeted_package): for path, subdirs, files in os.walk(samples_dir_path): for name in files: if fnmatch(name, "*.py") and name in samples_need_timeout: - timeout, pass_if_timeout = samples_need_timeout[name] + timeout = samples_need_timeout[name] + # timeout, pass_if_timeout is True by default if nothing passed in + if isinstance(timeout, tuple): + timeout, pass_if_timeout = timeout + else: + pass_if_timeout = True timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout)) elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []): sample_paths.append(os.path.abspath(os.path.join(path, name)))