From 41cc01f348bc92ef1a898a90c5aeec6136cbe77c Mon Sep 17 00:00:00 2001 From: bambriz Date: Wed, 23 Apr 2025 17:50:46 -0700 Subject: [PATCH 1/3] Update workload_utils.py Added filters for the worklaod test logger handler. It will only log when database account read happens, when latency is above 1000ms, and if there is an error. It won't log certain errors such as 404/0, 409/0, and 412/0. Given the test uses randint for upserting and reading documents a 404/0 and 409/0 is bound to occur and would just be considered noise for the workload tests. --- .../tests/workloads/workload_utils.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py index cd20fdb3684a..fb6a4901f258 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py @@ -8,6 +8,8 @@ from logging.handlers import RotatingFileHandler from workload_configs import NUMBER_OF_LOGICAL_PARTITIONS, PARTITION_KEY +_NOISY_ERRORS = set([404, 409, 412]) +_NOISY_SUB_STATUS_CODES = set([0, None]) def get_random_item(): random_int = random.randint(0, NUMBER_OF_LOGICAL_PARTITIONS) @@ -49,9 +51,33 @@ def create_logger(file_name): # Create a rotating file handler handler = RotatingFileHandler( "log-" + prefix + "-" + datetime.now().strftime("%Y%m%d-%H%M%S") + '.log', - maxBytes=1024 * 1024 * 10, # 10 mb + maxBytes=1024 * 1024 * 10, # 10 mb backupCount=3 ) logger.setLevel(logging.DEBUG) + # create filters for the logger handler to reduce the noise + workload_logger_filter = WorkloadLoggerFilter() + handler.addFilter(workload_logger_filter) logger.addHandler(handler) - return prefix, logger \ No newline at end of file + return prefix, logger + + +class WorkloadLoggerFilter(logging.Filter): + def filter(self, record): + # Check if the required attributes exists in the log record + required_attributes = ['duration', 'operation_type', 'status_code', 'resource_type', 'verb', 'sub_status_code'] + + if all(hasattr(record, attr) for attr in required_attributes): + # Check the conditions + # Check database account reads + if record.resource_type == "databaseaccount" and record.verb == "GET" and record.operation_type == "Read": + return True + # Check if there is an error and omit noisy errors + if record.status_code >= 400 and not ( + record.status_code in _NOISY_ERRORS and record.sub_status_code in _NOISY_SUB_STATUS_CODES): + return True + # Check if the latency (duration) was above 1000 ms + if record.duration > 1000: + return True + + return False From 80278324353db931d27b9c3e02b9673234d339b9 Mon Sep 17 00:00:00 2001 From: bambriz Date: Wed, 23 Apr 2025 17:56:38 -0700 Subject: [PATCH 2/3] Update workload_utils.py --- sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py index fb6a4901f258..91e4949d32d8 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py @@ -10,6 +10,7 @@ _NOISY_ERRORS = set([404, 409, 412]) _NOISY_SUB_STATUS_CODES = set([0, None]) +_REQUIRED_ATTRIBUTES = ["resource_type", "verb", "operation_type", "status_code", "sub_status_code", "duration"] def get_random_item(): random_int = random.randint(0, NUMBER_OF_LOGICAL_PARTITIONS) @@ -64,10 +65,8 @@ def create_logger(file_name): class WorkloadLoggerFilter(logging.Filter): def filter(self, record): - # Check if the required attributes exists in the log record - required_attributes = ['duration', 'operation_type', 'status_code', 'resource_type', 'verb', 'sub_status_code'] - - if all(hasattr(record, attr) for attr in required_attributes): + # Check if the required attributes exist in the log record + if all(hasattr(record, attr) for attr in _REQUIRED_ATTRIBUTES): # Check the conditions # Check database account reads if record.resource_type == "databaseaccount" and record.verb == "GET" and record.operation_type == "Read": @@ -79,5 +78,4 @@ def filter(self, record): # Check if the latency (duration) was above 1000 ms if record.duration > 1000: return True - return False From 9a5bb6ee44308b0d196ce40edf653c13c9c8a0cb Mon Sep 17 00:00:00 2001 From: bambriz Date: Wed, 23 Apr 2025 17:58:53 -0700 Subject: [PATCH 3/3] Update workload_utils.py --- sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py index 91e4949d32d8..20ef37059ae3 100644 --- a/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py +++ b/sdk/cosmos/azure-cosmos/tests/workloads/workload_utils.py @@ -76,6 +76,6 @@ def filter(self, record): record.status_code in _NOISY_ERRORS and record.sub_status_code in _NOISY_SUB_STATUS_CODES): return True # Check if the latency (duration) was above 1000 ms - if record.duration > 1000: + if record.duration >= 1000: return True return False