From 787429a53988f7d9e44da9df1e3ba04a8a854c09 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 26 Jul 2021 15:28:48 -0700 Subject: [PATCH 1/2] Perftests for monitor query --- .../tests/perfstress_tests/README.md | 48 +++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/batch_query.py | 81 +++++++++++++++++++ .../tests/perfstress_tests/metric_query.py | 70 ++++++++++++++++ .../tests/perfstress_tests/single_query.py | 68 ++++++++++++++++ 5 files changed, 267 insertions(+) create mode 100644 sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md create mode 100644 sdk/monitor/azure-monitor-query/tests/perfstress_tests/__init__.py create mode 100644 sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py create mode 100644 sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py create mode 100644 sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..8cb05ecf32d0 --- /dev/null +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md @@ -0,0 +1,48 @@ +# Monitor Query Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start by creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. + +### Setup for test resources + +These tests will run against a pre-configured Log Workspace. The following environment variable will need to be set for the tests to access the live resources: +``` +LOG_WORKSPACE_ID= +METRICS_RESOURCE_URI= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-monitor-query> pip install -r dev_requirements.txt +(env) ~/azure-monitor-query> pip install -e . +``` + +## Test commands + +```cmd +(env) ~/azure-monitor-query> cd tests +(env) ~/azure-monitor-query/tests> perfstress +``` + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +### T2 Tests +The tests currently written for the T2 SDK: +- `LogsPerfTest` queries a single query. +- `LogsBatchPerfTest` queries multiple queries using the bath operation. +- `MetricsPerfTest` to test a metrics query on eventgrid resource + +## Example command +```cmd +(env) ~/azure-monitor-query/tests> perfstress LogsPerfTest +(env) ~/azure-monitor-query/tests> perfstress LogsBatchPerfTest +(env) ~/azure-monitor-query/tests> perfstress MetricsPerfTest +``` diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/__init__.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py new file mode 100644 index 000000000000..cebaa2e3714f --- /dev/null +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py @@ -0,0 +1,81 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import asyncio +from datetime import date, datetime, timezone +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.monitor.query import LogsQueryClient as SyncLogsQueryClient, LogsQueryRequest +from azure.monitor.query.aio import LogsQueryClient as AsyncLogsQueryClient + +from azure.identity import DefaultAzureCredential as SyncDefaultAzureCredential +from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential + +class LogsBatchPerfTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + + # auth configuration + self.workspace_id = self.get_from_env('LOG_WORKSPACE_ID') + + # Create clients + self.logs_client = SyncLogsQueryClient( + credential=SyncDefaultAzureCredential() + ) + self.async_logs_client = AsyncLogsQueryClient( + credential=AsyncDefaultAzureCredential() + ) + + self.requests = [ + LogsQueryRequest( + query="AzureActivity | summarize count()", + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now(), + workspace_id= self.workspace_id + ), + LogsQueryRequest( + query= """AppRequests | take 10 | + summarize avgRequestDuration=avg(DurationMs) by bin(TimeGenerated, 10m), _ResourceId""", + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now(), + workspace_id= self.workspace_id + ), + LogsQueryRequest( + query= "AppRequests | take 20", + workspace_id= self.workspace_id, + include_statistics=True + ), + ] + + async def close(self): + """This is run after cleanup. + + Use this to close any open handles or clients. + """ + await self.async_logs_client.close() + await super().close() + + def run_sync(self): + """The synchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + self.logs_client.batch_query( + self.requests + ) + + async def run_async(self): + """The asynchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + await self.async_logs_client.batch_query( + self.requests + ) diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py new file mode 100644 index 000000000000..6edd6ec693a3 --- /dev/null +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py @@ -0,0 +1,70 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import asyncio +from datetime import datetime, timezone +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.monitor.query import MetricsQueryClient as SyncMetricsQueryClient, AggregationType +from azure.monitor.query.aio import MetricsQueryClient as AsyncMetricsQueryClient + +from azure.identity import DefaultAzureCredential as SyncDefaultAzureCredential +from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential + +class MetricsPerfTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + + # auth configuration + self.metrics_uri = self.get_from_env('METRICS_RESOURCE_URI') + self.names = ["MatchedEventCount"] + self.aggregations = [AggregationType.COUNT] + + # Create clients + self.metrics_client = SyncMetricsQueryClient( + credential=SyncDefaultAzureCredential() + ) + self.async_metrics_client = AsyncMetricsQueryClient( + credential=AsyncDefaultAzureCredential() + ) + + async def close(self): + """This is run after cleanup. + + Use this to close any open handles or clients. + """ + await self.async_metrics_client.close() + await super().close() + + def run_sync(self): + """The synchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + self.metrics_client.query( + self.metrics_uri, + self.names, + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now(tz=timezone.utc), + aggregations=self.aggregations + ) + + async def run_async(self): + """The asynchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + await self.async_metrics_client.query( + self.metrics_uri, + self.names, + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now(tz=timezone.utc), + aggregations=self.aggregations + ) diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py new file mode 100644 index 000000000000..81c4cc762a07 --- /dev/null +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py @@ -0,0 +1,68 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import asyncio +from datetime import date, datetime, timezone +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.monitor.query import LogsQueryClient as SyncLogsQueryClient +from azure.monitor.query.aio import LogsQueryClient as AsyncLogsQueryClient + +from azure.identity import DefaultAzureCredential as SyncDefaultAzureCredential +from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential + +class LogsPerfTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + + # auth configuration + self.workspace_id = self.get_from_env('LOG_WORKSPACE_ID') + + self.query = "AppRequests | summarize avgRequestDuration=avg(DurationMs) by bin(TimeGenerated, 10m), _ResourceId" + + # Create clients + self.logs_client = SyncLogsQueryClient( + credential=SyncDefaultAzureCredential() + ) + self.async_logs_client = AsyncLogsQueryClient( + credential=AsyncDefaultAzureCredential() + ) + + async def close(self): + """This is run after cleanup. + + Use this to close any open handles or clients. + """ + await self.async_logs_client.close() + await super().close() + + def run_sync(self): + """The synchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + self.logs_client.query( + self.workspace_id, + self.query, + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now() + ) + + async def run_async(self): + """The asynchronous perf test. + + Try to keep this minimal and focused. Using only a single client API. + Avoid putting any ancilliary logic (e.g. generating UUIDs), and put this in the setup/init instead + so that we're only measuring the client API call. + """ + await self.async_logs_client.query( + self.workspace_id, + self.query, + start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), + end_time=datetime.now(tz=timezone.utc) + ) From ebb30aec5ac0d953132ab7c747609d9537082e1e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 27 Jul 2021 11:09:45 -0700 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> --- .../azure-monitor-query/tests/perfstress_tests/README.md | 4 ++-- .../azure-monitor-query/tests/perfstress_tests/batch_query.py | 4 ++-- .../tests/perfstress_tests/metric_query.py | 4 ++-- .../tests/perfstress_tests/single_query.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md index 8cb05ecf32d0..a197406f8af0 100644 --- a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/README.md @@ -5,7 +5,7 @@ Start by creating a new virtual environment for your perf tests. This will need ### Setup for test resources -These tests will run against a pre-configured Log Workspace. The following environment variable will need to be set for the tests to access the live resources: +These tests will run against a pre-configured Log Analytics workspace. The following environment variable will need to be set for the tests to access the live resources: ``` LOG_WORKSPACE_ID= METRICS_RESOURCE_URI= @@ -37,7 +37,7 @@ These options are available for all perf tests: ### T2 Tests The tests currently written for the T2 SDK: - `LogsPerfTest` queries a single query. -- `LogsBatchPerfTest` queries multiple queries using the bath operation. +- `LogsBatchPerfTest` queries multiple queries using the batch operation. - `MetricsPerfTest` to test a metrics query on eventgrid resource ## Example command diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py index cebaa2e3714f..caa250dfdc6b 100644 --- a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/batch_query.py @@ -33,14 +33,14 @@ def __init__(self, arguments): LogsQueryRequest( query="AzureActivity | summarize count()", start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now(), + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), workspace_id= self.workspace_id ), LogsQueryRequest( query= """AppRequests | take 10 | summarize avgRequestDuration=avg(DurationMs) by bin(TimeGenerated, 10m), _ResourceId""", start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now(), + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), workspace_id= self.workspace_id ), LogsQueryRequest( diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py index 6edd6ec693a3..39512ae97e29 100644 --- a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/metric_query.py @@ -50,7 +50,7 @@ def run_sync(self): self.metrics_uri, self.names, start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now(tz=timezone.utc), + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), aggregations=self.aggregations ) @@ -65,6 +65,6 @@ async def run_async(self): self.metrics_uri, self.names, start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now(tz=timezone.utc), + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), aggregations=self.aggregations ) diff --git a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py index 81c4cc762a07..a8530961df54 100644 --- a/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py +++ b/sdk/monitor/azure-monitor-query/tests/perfstress_tests/single_query.py @@ -50,7 +50,7 @@ def run_sync(self): self.workspace_id, self.query, start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now() + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), ) async def run_async(self): @@ -64,5 +64,5 @@ async def run_async(self): self.workspace_id, self.query, start_time=datetime(2021, 7, 25, 0, 0, 0, tzinfo=timezone.utc), - end_time=datetime.now(tz=timezone.utc) + end_time=datetime(2021, 7, 26, 0, 0, 0, tzinfo=timezone.utc), )