From 943756c788a464d6556a94829e85bf183f4c4840 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 29 Jan 2024 15:35:20 -0800 Subject: [PATCH 01/21] ping --- .../exporter/_quickpulse/__init__.py | 4 +- .../exporter/_quickpulse/_exporter.py | 111 ++++++++++++++++-- .../exporter/_quickpulse/_live_metrics.py | 33 +++++- .../monitor/opentelemetry/exporter/_utils.py | 25 +++- 4 files changed, 149 insertions(+), 24 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py index 0cee259a4da9..19b6021a00b8 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py @@ -4,8 +4,8 @@ # license information. # ------------------------------------------------------------------------- -from azure.monitor.opentelemetry.exporter._quickpulse._exporter import QuickpulseExporter +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import _QuickpulseExporter __all__ = [ - "QuickpulseExporter", + "_QuickpulseExporter", ] diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index 1ebe4b481e38..7dc4de97b079 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -2,7 +2,17 @@ # Licensed under the MIT License. import logging -from typing import Any +from enum import Enum +from typing import Any, Optional + +from azure.core.exceptions import HttpResponseError +from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import ( + CollectionConfigurationInfo, + MonitoringDataPoint, +) +from azure.monitor.opentelemetry.exporter._connection_string_parser import ConnectionStringParser +from azure.monitor.opentelemetry.exporter._utils import _ticks_since_dot_net_epoch from opentelemetry.sdk.metrics import ( Counter, @@ -17,16 +27,13 @@ MetricExporter, MetricExportResult, MetricsData as OTMetricsData, + PeriodicExportingMetricReader, ) -from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient -from azure.monitor.opentelemetry.exporter._connection_string_parser import ConnectionStringParser _logger = logging.getLogger(__name__) -__all__ = ["QuickpulseExporter"] - -APPLICATION_INSIGHTS_METRIC_TEMPORALITIES = { +_APPLICATION_INSIGHTS_METRIC_TEMPORALITIES = { Counter: AggregationTemporality.DELTA, Histogram: AggregationTemporality.DELTA, ObservableCounter: AggregationTemporality.DELTA, @@ -35,27 +42,32 @@ UpDownCounter: AggregationTemporality.CUMULATIVE, } +_SHORT_PING_INTERVAL_SECONDS = 5 +_SHORT_POST_INTERVAL_SECONDS = 1 +_LONG_PING_INTERVAL_SECONDS = 60 +_LONG_POST_INTERVAL_SECONDS = 20 + -class QuickpulseExporter(MetricExporter): +class _QuickpulseExporter(MetricExporter): - def __init__(self, **kwargs: Any) -> None: + def __init__(self, connection_string: str) -> None: """Metric exporter for Quickpulse. - :keyword str connection_string: The connection string used for your Application Insights resource. + :param str connection_string: The connection string used for your Application Insights resource. :rtype: None """ - parsed_connection_string = ConnectionStringParser(kwargs.get('connection_string')) + parsed_connection_string = ConnectionStringParser(connection_string) self._endpoint = parsed_connection_string.endpoint + self._instrumentation_key = parsed_connection_string.instrumentation_key # TODO: Support AADaudience (scope)/credentials - self.client = QuickpulseClient(host=self._endpoint, **kwargs) + self._client = QuickpulseClient(host=self._endpoint) # TODO: Support redirect MetricExporter.__init__( self, - preferred_temporality=APPLICATION_INSIGHTS_METRIC_TEMPORALITIES, # type: ignore - preferred_aggregation=kwargs.get("preferred_aggregation"), # type: ignore + preferred_temporality=_APPLICATION_INSIGHTS_METRIC_TEMPORALITIES, # type: ignore ) def export( @@ -104,3 +116,76 @@ def shutdown( :param timeout_millis: The maximum amount of time to wait for shutdown. Not currently used. :type timeout_millis: float """ + + + def _ping(self, monitoring_data_point) -> Optional[CollectionConfigurationInfo]: + try: + ping_response = self._client.ping( + monitoring_data_point=monitoring_data_point, + ikey=self._instrumentation_key, + x_ms_qps_transmission_time=_ticks_since_dot_net_epoch() + ) + if isinstance(ping_response, CollectionConfigurationInfo): + pass + else: + # Responses that are not 200s are ignored + return None + except HttpResponseError as response_error: + # Errors are not reported + return None + + +class QuickpulseState(Enum): + """Current state of quickpulse service. + The numerical value represents the ping/post interval in ms for those states. + """ + + PING_SHORT = _SHORT_PING_INTERVAL_SECONDS + PING_LONG = _LONG_PING_INTERVAL_SECONDS + POST_SHORT = _SHORT_POST_INTERVAL_SECONDS + POST_LONG = _LONG_POST_INTERVAL_SECONDS + + +class _QuickpulseMetricReader(PeriodicExportingMetricReader): + + def __init__( + self, + exporter: _QuickpulseExporter, + base_monitoring_data_point: MonitoringDataPoint, + ) -> None: + self._exporter = exporter + self._quick_pulse_state = QuickpulseState.PING_SHORT + self._base_monitoring_data_point = base_monitoring_data_point + self._elapsed_num_seconds = 0 + super().__init__( + exporter=exporter, + export_interval_millis=_SHORT_POST_INTERVAL_SECONDS * 1000, + ) + + def _ticker(self) -> None: + if self._is_ping_state(): + # Send a ping if elapsed number of request meets the threshold + if self._elapsed_num_seconds % int(self._quick_pulse_state.value) == 0: + print("pinging...") + ping_response = self._exporter._ping( + self._base_monitoring_data_point, + ) + if ping_response and ping_response.response_headers: + if ping_response.response_headers.get("x-ms-qps-subscribed"): + # Switch state to post if subscribed + self._quick_pulse_state = QuickpulseState.POST_SHORT + # TODO: Implement redirect + # TODO: Implement interval hint + else: + # Erroroneous responses instigate backoff logic + # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests + if self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + self._quick_pulse_state = QuickpulseState.PING_LONG + pass + else: + print("posting") + pass + + def _is_ping_state(self): + return self._quick_pulse_state in (QuickpulseState.PING_SHORT, QuickpulseState.PING_LONG) + \ No newline at end of file diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py index 24a9e0d80a24..5ea5e8baa44c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py @@ -1,18 +1,41 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import platform + +from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _QuickpulseExporter, + _QuickpulseMetricReader, +) +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +from azure.monitor.opentelemetry.exporter._utils import _get_sdk_version, _populate_part_a_fields +from opentelemetry.sdk.trace.id_generator import RandomIdGenerator +from opentelemetry.sdk.resources import Resource def enable_live_metrics(connection_string: str) -> None: - QuickpulseStateManager(connection_string) + QuickpulseManager(connection_string) -class QuickpulseStateManager: +class QuickpulseManager: def __new__(cls, *args, **kwargs): if not hasattr(cls, 'instance'): - cls._instance = super(QuickpulseStateManager, cls).__new__(cls, *args, **kwargs) + cls._instance = super(QuickpulseManager, cls).__new__(cls, *args, **kwargs) return cls._instance - def __init__(self, connection_string): + def __init__(self, connection_string: str, resource: Resource) -> None: self._connection_string = connection_string - # TODO + self._exporter = _QuickpulseExporter(self._connection_string) + part_a_fields = _populate_part_a_fields(resource) + id_generator = RandomIdGenerator() + self._base_monitoring_data_point = MonitoringDataPoint( + version=_get_sdk_version(), + invariant_version=1, + instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), + role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE), + machine_name=platform.node(), + stream_id=id_generator.generate_trace_id() + ) + self._reader = _QuickpulseMetricReader(self._exporter, self._base_monitoring_data_point) + \ No newline at end of file diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index dc7c0b240f86..aa9d0f66bfd3 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import datetime import locale from os import environ from os.path import isdir @@ -56,6 +56,12 @@ def _get_sdk_version_prefix(): return sdk_version_prefix +def _get_sdk_version(): + return "{}py{}:otel{}:ext{}".format( + _get_sdk_version_prefix(), platform.python_version(), opentelemetry_version, ext_version + ) + + def _getlocale(): try: with warnings.catch_warnings(): @@ -75,9 +81,7 @@ def _getlocale(): ContextTagKeys.AI_DEVICE_LOCALE: _getlocale(), ContextTagKeys.AI_DEVICE_OS_VERSION: platform.version(), ContextTagKeys.AI_DEVICE_TYPE: "Other", - ContextTagKeys.AI_INTERNAL_SDK_VERSION: "{}py{}:otel{}:ext{}".format( - _get_sdk_version_prefix(), platform.python_version(), opentelemetry_version, ext_version - ), + ContextTagKeys.AI_INTERNAL_SDK_VERSION: _get_sdk_version(), } @@ -91,6 +95,19 @@ def ns_to_duration(nanoseconds: int): days, hours, minutes, seconds, microseconds ) + +# Replicate .netDateTime.Ticks(), which is the UTC time, expressed as the number +# of 100-nanosecond intervals that have elapsed since 12:00:00 midnight on +# January 1, 0001. +def _ticks_since_dot_net_epoch(): + # Since time.time() is the elapsed time since UTC January 1, 1970, we have + # to shift this start time, and then multiply by 10^7 to get the number of + # 100-nanosecond intervals + shift_time = int((datetime.datetime(1970, 1, 1, 0, 0, 0) - datetime.datetime(1, 1, 1, 0, 0, 0)).total_seconds()) * (10 ** 7) + # Add shift time to 100-ns intervals since time.time() + return int(time.time() * (10**7)) + shift_time + + _INSTRUMENTATIONS_BIT_MASK = 0 _INSTRUMENTATIONS_BIT_MASK_LOCK = threading.Lock() From 0842b4ab66189aa3e066e84a10070ad0fa267486 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Fri, 2 Feb 2024 18:49:12 -0800 Subject: [PATCH 02/21] qp --- .../CHANGELOG.md | 2 + .../exporter/_quickpulse/_constants.py | 36 ++++ .../exporter/_quickpulse/_exporter.py | 193 +++++++++++++++--- .../exporter/_quickpulse/_live_metrics.py | 36 ++-- .../monitor/opentelemetry/exporter/_utils.py | 8 + 5 files changed, 227 insertions(+), 48 deletions(-) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index b4636820b152..74e59ef2ba65 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -6,6 +6,8 @@ - Add live metrics skeleton + swagger definitions ([#33983](https://github.com/Azure/azure-sdk-for-python/pull/33983)) +- Add live metrics exporting functionality + ([#33983](https://github.com/Azure/azure-sdk-for-python/pull/33983)) ### Breaking Changes diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py new file mode 100644 index 000000000000..d932ea90ee1d --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py @@ -0,0 +1,36 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# cSpell:disable + +# (OpenTelemetry metric name, Quickpulse metric name) +# Memory +_COMMITTED_BYTES_NAME = ("azuremonitor.memorycommittedbytes", "\\Memory\\Committed Bytes") +# CPU +_PROCESSOR_TIME_NAME = ("azuremonitor.processortotalprocessortime", "\\Processor(_Total)\\% Processor Time") +# Request +_REQUEST_RATE_NAME = ("azuremonitor.requestssec", "\\ApplicationInsights\\Requests/Sec") +_REQUEST_FAILURE_RATE_NAME = ("azuremonitor.requestsfailedsec", "\\ApplicationInsights\\Requests Failed/Sec") +_REQUEST_DURATION_NAME = ("azuremonitor.requestduration", "\\ApplicationInsights\\Request Duration") +# Dependency +_DEPENDENCY_RATE_NAME = ("azuremonitor.dependencycallssec", "\\ApplicationInsights\\Dependency Calls/Sec") +_DEPENDENCY_FAILURE_RATE_NAME = ("azuremonitor.dependencycallsfailedsec", "\\ApplicationInsights\\Dependency Calls Failed/Sec") +_DEPENDENCY_DURATION_NAME = ("azuremonitor.dependencycallduration", "\\ApplicationInsights\\Dependency Call Duration") +# Exception +_EXCEPTION_RATE_NAME = ("azuremonitor.exceptionssec", "\\ApplicationInsights\\Exceptions/Sec") + +_QUICKPULSE_METRIC_NAME_MAPPINGS = dict( + [ + _COMMITTED_BYTES_NAME, + _PROCESSOR_TIME_NAME, + _PROCESSOR_TIME_NAME, + _REQUEST_RATE_NAME, + _REQUEST_FAILURE_RATE_NAME, + _REQUEST_DURATION_NAME, + _DEPENDENCY_RATE_NAME, + _DEPENDENCY_FAILURE_RATE_NAME, + _DEPENDENCY_DURATION_NAME, + _EXCEPTION_RATE_NAME, + ] +) + +# cSpell:disable \ No newline at end of file diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index 7dc4de97b079..b33322b054bc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -1,19 +1,26 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import logging - +from datetime import datetime, timezone from enum import Enum -from typing import Any, Optional +from typing import Any, Optional, Sequence from azure.core.exceptions import HttpResponseError +from azure.monitor.opentelemetry.exporter._quickpulse._constants import _QUICKPULSE_METRIC_NAME_MAPPINGS from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import ( - CollectionConfigurationInfo, + DocumentIngress, + MetricPoint, MonitoringDataPoint, ) from azure.monitor.opentelemetry.exporter._connection_string_parser import ConnectionStringParser -from azure.monitor.opentelemetry.exporter._utils import _ticks_since_dot_net_epoch +from azure.monitor.opentelemetry.exporter._utils import _ticks_since_dot_net_epoch, PeriodicTask +from opentelemetry.context import ( + _SUPPRESS_INSTRUMENTATION_KEY, + attach, + detach, + set_value, +) from opentelemetry.sdk.metrics import ( Counter, Histogram, @@ -22,16 +29,19 @@ ObservableUpDownCounter, UpDownCounter, ) +from opentelemetry.sdk.metrics._internal.point import ( + NumberDataPoint, + HistogramDataPoint, + MetricsData, +) from opentelemetry.sdk.metrics.export import ( AggregationTemporality, MetricExporter, MetricExportResult, MetricsData as OTMetricsData, - PeriodicExportingMetricReader, + MetricReader, ) -_logger = logging.getLogger(__name__) - _APPLICATION_INSIGHTS_METRIC_TEMPORALITIES = { Counter: AggregationTemporality.DELTA, @@ -48,9 +58,17 @@ _LONG_POST_INTERVAL_SECONDS = 20 +class Response: + + def __init__(self, pipeline_response, deserialized, response_headers): + self._pipeline_response = pipeline_response + self._deserialized = deserialized + self._response_headers = response_headers + + class _QuickpulseExporter(MetricExporter): - def __init__(self, connection_string: str) -> None: + def __init__(self, connection_string: Optional[str]) -> None: """Metric exporter for Quickpulse. :param str connection_string: The connection string used for your Application Insights resource. @@ -58,11 +76,11 @@ def __init__(self, connection_string: str) -> None: """ parsed_connection_string = ConnectionStringParser(connection_string) - self._endpoint = parsed_connection_string.endpoint + self._live_endpoint = parsed_connection_string.live_endpoint self._instrumentation_key = parsed_connection_string.instrumentation_key # TODO: Support AADaudience (scope)/credentials - self._client = QuickpulseClient(host=self._endpoint) + self._client = QuickpulseClient(host=self._live_endpoint) # TODO: Support redirect MetricExporter.__init__( @@ -85,8 +103,35 @@ def export( :return: The result of the export. :rtype: ~opentelemetry.sdk.metrics.export.MetricExportResult """ - # TODO - return MetricExportResult.SUCCESS + result = MetricExportResult.SUCCESS + if metrics_data is None: + return result + data_points = _metric_to_quick_pulse_data_points( + metrics_data, + base_monitoring_data_point=kwargs.get("base_monitoring_data_point"), + documents=kwargs.get("documents"), + ) + + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + post_response = self._client.post( + monitoring_data_points=data_points, + ikey=self._instrumentation_key, + x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), + cls=Response, + ) + if not post_response: + result = MetricExportResult.FAILURE + header = post_response._response_headers.get("x-ms-qps-subscribed") + if header != "true": + # We raise an exception to indicate that quickpulse is not activated anymore + raise Exception() + except HttpResponseError: + # Errors are not reported + result = MetricExportResult.FAILURE + finally: + detach(token) + return result def force_flush( self, @@ -118,21 +163,22 @@ def shutdown( """ - def _ping(self, monitoring_data_point) -> Optional[CollectionConfigurationInfo]: + def _ping(self, monitoring_data_point) -> Optional[Response]: + ping_response = None + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: ping_response = self._client.ping( monitoring_data_point=monitoring_data_point, ikey=self._instrumentation_key, - x_ms_qps_transmission_time=_ticks_since_dot_net_epoch() + x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), + cls=Response, ) - if isinstance(ping_response, CollectionConfigurationInfo): - pass - else: - # Responses that are not 200s are ignored - return None - except HttpResponseError as response_error: + return ping_response + except HttpResponseError: # Errors are not reported - return None + pass + detach(token) + return ping_response class QuickpulseState(Enum): @@ -143,10 +189,9 @@ class QuickpulseState(Enum): PING_SHORT = _SHORT_PING_INTERVAL_SECONDS PING_LONG = _LONG_PING_INTERVAL_SECONDS POST_SHORT = _SHORT_POST_INTERVAL_SECONDS - POST_LONG = _LONG_POST_INTERVAL_SECONDS -class _QuickpulseMetricReader(PeriodicExportingMetricReader): +class _QuickpulseMetricReader(MetricReader): def __init__( self, @@ -157,10 +202,17 @@ def __init__( self._quick_pulse_state = QuickpulseState.PING_SHORT self._base_monitoring_data_point = base_monitoring_data_point self._elapsed_num_seconds = 0 + self._worker = PeriodicTask( + interval=_SHORT_POST_INTERVAL_SECONDS, + function=self._ticker, + name="QuickpulseMetricReader", + ) + self._worker.daemon = True super().__init__( - exporter=exporter, - export_interval_millis=_SHORT_POST_INTERVAL_SECONDS * 1000, + preferred_temporality=self._exporter._preferred_temporality, + preferred_aggregation=self._exporter._preferred_aggregation, ) + self._worker.start() def _ticker(self) -> None: if self._is_ping_state(): @@ -170,22 +222,95 @@ def _ticker(self) -> None: ping_response = self._exporter._ping( self._base_monitoring_data_point, ) - if ping_response and ping_response.response_headers: - if ping_response.response_headers.get("x-ms-qps-subscribed"): + if ping_response: + header = ping_response._response_headers.get("x-ms-qps-subscribed") + if header and header == "true": + print("ping succeeded: switching to post") # Switch state to post if subscribed self._quick_pulse_state = QuickpulseState.POST_SHORT + self._elapsed_num_seconds = 0 + else: + # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests + if self._quick_pulse_state is QuickpulseState.PING_SHORT and self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + print("ping failed for 60s, switching to pinging every 60s") + self._quick_pulse_state = QuickpulseState.PING_LONG # TODO: Implement redirect - # TODO: Implement interval hint else: # Erroroneous responses instigate backoff logic # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests - if self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + if self._quick_pulse_state is QuickpulseState.PING_SHORT and self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + print("ping failed for 60s, switching to pinging every 60s") self._quick_pulse_state = QuickpulseState.PING_LONG - pass else: - print("posting") - pass + print("posting...") + try: + self.collect() + except Exception as ex: # pylint: disable=broad-except,invalid-name + # Unsuccessful exports instigate backoff logic + # Backoff after _LONG_POST_INTERVAL_SECONDS (20s) of no successful requests + # And resume pinging + if self._elapsed_num_seconds >= _LONG_POST_INTERVAL_SECONDS: + print("post failed for 20s, switching to pinging") + self._quick_pulse_state = QuickpulseState.PING_SHORT + self._elapsed_num_seconds = 0 + + self._elapsed_num_seconds += 1 + + def _receive_metrics( + self, + metrics_data: MetricsData, + timeout_millis: float = 10_000, + **kwargs, + ) -> None: + result = self._exporter.export( + metrics_data, + timeout_millis=timeout_millis, + base_monitoring_data_point=self._base_monitoring_data_point, + documents=[], + ) + if result is MetricExportResult.FAILURE: + # There is currently no way to propagate unsuccessful metric post so we raise an exception + # MUST handle this exception whenever `collect()` is called + raise Exception() + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + self._worker.cancel() + self._worker.join() def _is_ping_state(self): return self._quick_pulse_state in (QuickpulseState.PING_SHORT, QuickpulseState.PING_LONG) - \ No newline at end of file + +def _metric_to_quick_pulse_data_points( + metrics_data: OTMetricsData, + base_monitoring_data_point: MonitoringDataPoint, + documents: Sequence[DocumentIngress], +) -> Sequence[MonitoringDataPoint]: + metric_points = [] + for resource_metric in metrics_data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + for point in metric.data.data_points: + if point is not None: + metric_point = MetricPoint( + name=_QUICKPULSE_METRIC_NAME_MAPPINGS[metric.name.lower()], + weight=1, + ) + if isinstance(point, HistogramDataPoint): + metric_point.value = point.sum + elif isinstance(point, NumberDataPoint): + metric_point.value = point.value + else: + metric_point.value = 0 + metric_points.append(metric_point) + return [ + MonitoringDataPoint( + version=base_monitoring_data_point.version, + instance=base_monitoring_data_point.instance, + role_name=base_monitoring_data_point.role_name, + machine_name=base_monitoring_data_point.machine_name, + stream_id=base_monitoring_data_point.stream_id, + timestamp=datetime.now(tz=timezone.utc), + metrics=metric_points, + documents=documents, + ) + ] diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py index 5ea5e8baa44c..3d8d010ec12f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import platform +from typing import Any, Optional from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( @@ -8,34 +9,41 @@ _QuickpulseMetricReader, ) from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint -from azure.monitor.opentelemetry.exporter._utils import _get_sdk_version, _populate_part_a_fields +from azure.monitor.opentelemetry.exporter._utils import ( + _get_sdk_version, + _populate_part_a_fields, + Singleton, +) +from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace.id_generator import RandomIdGenerator from opentelemetry.sdk.resources import Resource -def enable_live_metrics(connection_string: str) -> None: - QuickpulseManager(connection_string) +def enable_live_metrics(**kwargs: Any) -> None: + """Azure Monitor base exporter for OpenTelemetry. + :keyword str connection_string: The connection string used for your Application Insights resource. + :keyword Resource resource: The OpenTelemetry Resource used for this Python application. + :rtype: None + """ + return _QuickpulseManager(kwargs.get('connection_string'), kwargs.get('resource')) -class QuickpulseManager: - def __new__(cls, *args, **kwargs): - if not hasattr(cls, 'instance'): - cls._instance = super(QuickpulseManager, cls).__new__(cls, *args, **kwargs) - return cls._instance +class _QuickpulseManager(metaclass=Singleton): - def __init__(self, connection_string: str, resource: Resource) -> None: - self._connection_string = connection_string - self._exporter = _QuickpulseExporter(self._connection_string) - part_a_fields = _populate_part_a_fields(resource) + def __init__(self, connection_string: Optional[str], resource: Optional[Resource]) -> None: + self._exporter = _QuickpulseExporter(connection_string) + part_a_fields = {} + if resource: + part_a_fields = _populate_part_a_fields(resource) id_generator = RandomIdGenerator() self._base_monitoring_data_point = MonitoringDataPoint( version=_get_sdk_version(), invariant_version=1, instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), - role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE), + role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), machine_name=platform.node(), stream_id=id_generator.generate_trace_id() ) self._reader = _QuickpulseMetricReader(self._exporter, self._base_monitoring_data_point) - \ No newline at end of file + self._meter_provider = MeterProvider([self._reader]) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index aa9d0f66bfd3..9421e3c64692 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -207,3 +207,11 @@ def _filter_custom_properties(properties: Attributes, filter=None) -> Dict[str, continue truncated_properties[key] = str(val)[:8192] return truncated_properties + + +class Singleton(type): + _instance = None + def __call__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instance From cfdeab79fa913ef7952a9588ec0b428b9252f56a Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 5 Feb 2024 09:51:22 -0800 Subject: [PATCH 03/21] spelling --- .../CHANGELOG.md | 2 +- .../exporter/_quickpulse/_constants.py | 4 +- .../exporter/_quickpulse/_exporter.py | 63 ++++++++++--------- .../exporter/_quickpulse/_live_metrics.py | 18 +++--- .../monitor/opentelemetry/exporter/_utils.py | 6 +- 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 28fbd8c2b572..ca04cd1bc9e7 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -7,7 +7,7 @@ - Add live metrics skeleton + swagger definitions ([#33983](https://github.com/Azure/azure-sdk-for-python/pull/33983)) - Add live metrics exporting functionality - ([#33983](https://github.com/Azure/azure-sdk-for-python/pull/33983)) + ([#34141](https://github.com/Azure/azure-sdk-for-python/pull/34141)) - Only create temporary folder if local storage is enabled without storage directory. ([#34061](https://github.com/Azure/azure-sdk-for-python/pull/34061)) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py index d932ea90ee1d..b591258f2ee0 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_constants.py @@ -13,7 +13,7 @@ _REQUEST_DURATION_NAME = ("azuremonitor.requestduration", "\\ApplicationInsights\\Request Duration") # Dependency _DEPENDENCY_RATE_NAME = ("azuremonitor.dependencycallssec", "\\ApplicationInsights\\Dependency Calls/Sec") -_DEPENDENCY_FAILURE_RATE_NAME = ("azuremonitor.dependencycallsfailedsec", "\\ApplicationInsights\\Dependency Calls Failed/Sec") +_DEPENDENCY_FAILURE_RATE_NAME = ("azuremonitor.dependencycallsfailedsec", "\\ApplicationInsights\\Dependency Calls Failed/Sec") # pylint: disable=line-too-long _DEPENDENCY_DURATION_NAME = ("azuremonitor.dependencycallduration", "\\ApplicationInsights\\Dependency Call Duration") # Exception _EXCEPTION_RATE_NAME = ("azuremonitor.exceptionssec", "\\ApplicationInsights\\Exceptions/Sec") @@ -33,4 +33,4 @@ ] ) -# cSpell:disable \ No newline at end of file +# cSpell:disable diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index b33322b054bc..6a08496d1d7a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -2,18 +2,7 @@ # Licensed under the MIT License. from datetime import datetime, timezone from enum import Enum -from typing import Any, Optional, Sequence - -from azure.core.exceptions import HttpResponseError -from azure.monitor.opentelemetry.exporter._quickpulse._constants import _QUICKPULSE_METRIC_NAME_MAPPINGS -from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient -from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import ( - DocumentIngress, - MetricPoint, - MonitoringDataPoint, -) -from azure.monitor.opentelemetry.exporter._connection_string_parser import ConnectionStringParser -from azure.monitor.opentelemetry.exporter._utils import _ticks_since_dot_net_epoch, PeriodicTask +from typing import Any, List, Optional from opentelemetry.context import ( _SUPPRESS_INSTRUMENTATION_KEY, @@ -42,6 +31,17 @@ MetricReader, ) +from azure.core.exceptions import HttpResponseError +from azure.monitor.opentelemetry.exporter._quickpulse._constants import _QUICKPULSE_METRIC_NAME_MAPPINGS +from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import ( + DocumentIngress, + MetricPoint, + MonitoringDataPoint, +) +from azure.monitor.opentelemetry.exporter._connection_string_parser import ConnectionStringParser +from azure.monitor.opentelemetry.exporter._utils import _ticks_since_dot_net_epoch, PeriodicTask + _APPLICATION_INSIGHTS_METRIC_TEMPORALITIES = { Counter: AggregationTemporality.DELTA, @@ -104,25 +104,26 @@ def export( :rtype: ~opentelemetry.sdk.metrics.export.MetricExportResult """ result = MetricExportResult.SUCCESS - if metrics_data is None: + base_monitoring_data_point = kwargs.get("base_monitoring_data_point") + if metrics_data is None or base_monitoring_data_point is None: return result data_points = _metric_to_quick_pulse_data_points( metrics_data, - base_monitoring_data_point=kwargs.get("base_monitoring_data_point"), + base_monitoring_data_point=base_monitoring_data_point, documents=kwargs.get("documents"), ) token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: - post_response = self._client.post( + post_response = self._client.post( # type: ignore monitoring_data_points=data_points, ikey=self._instrumentation_key, x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), - cls=Response, + cls=Response, ) if not post_response: result = MetricExportResult.FAILURE - header = post_response._response_headers.get("x-ms-qps-subscribed") + header = post_response._response_headers.get("x-ms-qps-subscribed") # pylint: disable=protected-access if header != "true": # We raise an exception to indicate that quickpulse is not activated anymore raise Exception() @@ -167,13 +168,13 @@ def _ping(self, monitoring_data_point) -> Optional[Response]: ping_response = None token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: - ping_response = self._client.ping( + ping_response = self._client.ping( # type: ignore monitoring_data_point=monitoring_data_point, ikey=self._instrumentation_key, x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), - cls=Response, + cls=Response, ) - return ping_response + return ping_response # type: ignore except HttpResponseError: # Errors are not reported pass @@ -219,11 +220,11 @@ def _ticker(self) -> None: # Send a ping if elapsed number of request meets the threshold if self._elapsed_num_seconds % int(self._quick_pulse_state.value) == 0: print("pinging...") - ping_response = self._exporter._ping( + ping_response = self._exporter._ping( # pylint: disable=protected-access self._base_monitoring_data_point, ) if ping_response: - header = ping_response._response_headers.get("x-ms-qps-subscribed") + header = ping_response._response_headers.get("x-ms-qps-subscribed") # pylint: disable=protected-access if header and header == "true": print("ping succeeded: switching to post") # Switch state to post if subscribed @@ -231,21 +232,23 @@ def _ticker(self) -> None: self._elapsed_num_seconds = 0 else: # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests - if self._quick_pulse_state is QuickpulseState.PING_SHORT and self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + if self._quick_pulse_state is QuickpulseState.PING_SHORT and \ + self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: print("ping failed for 60s, switching to pinging every 60s") self._quick_pulse_state = QuickpulseState.PING_LONG # TODO: Implement redirect else: - # Erroroneous responses instigate backoff logic + # Erroneous responses instigate backoff logic # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests - if self._quick_pulse_state is QuickpulseState.PING_SHORT and self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: + if self._quick_pulse_state is QuickpulseState.PING_SHORT and \ + self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: print("ping failed for 60s, switching to pinging every 60s") self._quick_pulse_state = QuickpulseState.PING_LONG else: print("posting...") try: self.collect() - except Exception as ex: # pylint: disable=broad-except,invalid-name + except Exception: # pylint: disable=broad-except,invalid-name # Unsuccessful exports instigate backoff logic # Backoff after _LONG_POST_INTERVAL_SECONDS (20s) of no successful requests # And resume pinging @@ -279,12 +282,12 @@ def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: def _is_ping_state(self): return self._quick_pulse_state in (QuickpulseState.PING_SHORT, QuickpulseState.PING_LONG) - -def _metric_to_quick_pulse_data_points( + +def _metric_to_quick_pulse_data_points( # pylint: disable=too-many-nested-blocks metrics_data: OTMetricsData, base_monitoring_data_point: MonitoringDataPoint, - documents: Sequence[DocumentIngress], -) -> Sequence[MonitoringDataPoint]: + documents: Optional[List[DocumentIngress]], +) -> List[MonitoringDataPoint]: metric_points = [] for resource_metric in metrics_data.resource_metrics: for scope_metric in resource_metric.scope_metrics: diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py index 3d8d010ec12f..83bb3e073e3d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_live_metrics.py @@ -3,6 +3,9 @@ import platform from typing import Any, Optional +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace.id_generator import RandomIdGenerator +from opentelemetry.sdk.resources import Resource from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( _QuickpulseExporter, @@ -14,19 +17,16 @@ _populate_part_a_fields, Singleton, ) -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.trace.id_generator import RandomIdGenerator -from opentelemetry.sdk.resources import Resource def enable_live_metrics(**kwargs: Any) -> None: """Azure Monitor base exporter for OpenTelemetry. - :keyword str connection_string: The connection string used for your Application Insights resource. - :keyword Resource resource: The OpenTelemetry Resource used for this Python application. - :rtype: None - """ - return _QuickpulseManager(kwargs.get('connection_string'), kwargs.get('resource')) + :keyword str connection_string: The connection string used for your Application Insights resource. + :keyword Resource resource: The OpenTelemetry Resource used for this Python application. + :rtype: None + """ + _QuickpulseManager(kwargs.get('connection_string'), kwargs.get('resource')) class _QuickpulseManager(metaclass=Singleton): @@ -43,7 +43,7 @@ def __init__(self, connection_string: Optional[str], resource: Optional[Resource instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), machine_name=platform.node(), - stream_id=id_generator.generate_trace_id() + stream_id=str(id_generator.generate_trace_id()), ) self._reader = _QuickpulseMetricReader(self._exporter, self._base_monitoring_data_point) self._meter_provider = MeterProvider([self._reader]) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index 9421e3c64692..e19d5f7872f3 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -103,7 +103,11 @@ def _ticks_since_dot_net_epoch(): # Since time.time() is the elapsed time since UTC January 1, 1970, we have # to shift this start time, and then multiply by 10^7 to get the number of # 100-nanosecond intervals - shift_time = int((datetime.datetime(1970, 1, 1, 0, 0, 0) - datetime.datetime(1, 1, 1, 0, 0, 0)).total_seconds()) * (10 ** 7) + shift_time = int( + ( + datetime.datetime(1970, 1, 1, 0, 0, 0) - + datetime.datetime(1, 1, 1, 0, 0, 0)).total_seconds() + ) * (10 ** 7) # Add shift time to 100-ns intervals since time.time() return int(time.time() * (10**7)) + shift_time From 358d8aeef3dc1d7e9a7da2ce7034511e82606a15 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 5 Feb 2024 10:35:53 -0800 Subject: [PATCH 04/21] lint --- .../azure/monitor/opentelemetry/exporter/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index e19d5f7872f3..90a9887f7207 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -105,7 +105,7 @@ def _ticks_since_dot_net_epoch(): # 100-nanosecond intervals shift_time = int( ( - datetime.datetime(1970, 1, 1, 0, 0, 0) - + datetime.datetime(1970, 1, 1, 0, 0, 0) - datetime.datetime(1, 1, 1, 0, 0, 0)).total_seconds() ) * (10 ** 7) # Add shift time to 100-ns intervals since time.time() From 2ddb0f2fe7c0c7f19ea2e66d1ffc6d6ef5db25cf Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Thu, 15 Feb 2024 10:21:06 -0800 Subject: [PATCH 05/21] comments --- .../exporter/_quickpulse/_exporter.py | 34 ++++++++++++------- .../monitor/opentelemetry/exporter/_utils.py | 1 + 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index 6a08496d1d7a..7862ffb0727a 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -59,13 +59,19 @@ class Response: - + """Response that encapsulates pipeline response and response headers from + QuickPulse client. + """ def __init__(self, pipeline_response, deserialized, response_headers): self._pipeline_response = pipeline_response self._deserialized = deserialized self._response_headers = response_headers +class UnsuccessfulQuickPulsePostError(Exception): + """Exception raised to indicate unsuccessful QuickPulse post for backoff logic.""" + + class _QuickpulseExporter(MetricExporter): def __init__(self, connection_string: Optional[str]) -> None: @@ -97,7 +103,7 @@ def export( """Exports a batch of metric data :param metrics_data: OpenTelemetry Metric(s) to export. - :type metrics_data: Sequence[~opentelemetry.sdk.metrics._internal.point.MetricsData] + :type metrics_data: ~opentelemetry.sdk.metrics._internal.point.MetricsData :param timeout_millis: The maximum amount of time to wait for each export. Not currently used. :type timeout_millis: float :return: The result of the export. @@ -105,7 +111,7 @@ def export( """ result = MetricExportResult.SUCCESS base_monitoring_data_point = kwargs.get("base_monitoring_data_point") - if metrics_data is None or base_monitoring_data_point is None: + if base_monitoring_data_point is None: return result data_points = _metric_to_quick_pulse_data_points( metrics_data, @@ -122,13 +128,14 @@ def export( cls=Response, ) if not post_response: + # If no response, assume unsuccessful result = MetricExportResult.FAILURE header = post_response._response_headers.get("x-ms-qps-subscribed") # pylint: disable=protected-access if header != "true": - # We raise an exception to indicate that quickpulse is not activated anymore - raise Exception() - except HttpResponseError: - # Errors are not reported + # User leaving the live metrics page will be treated as an unsuccessful + result = MetricExportResult.FAILURE + except Exception: # pylint: disable=broad-except,invalid-name + # Errors are not reported and assumed as unsuccessful result = MetricExportResult.FAILURE finally: detach(token) @@ -238,7 +245,7 @@ def _ticker(self) -> None: self._quick_pulse_state = QuickpulseState.PING_LONG # TODO: Implement redirect else: - # Erroneous responses instigate backoff logic + # Erroneous ping responses instigate backoff logic # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests if self._quick_pulse_state is QuickpulseState.PING_SHORT and \ self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: @@ -248,8 +255,8 @@ def _ticker(self) -> None: print("posting...") try: self.collect() - except Exception: # pylint: disable=broad-except,invalid-name - # Unsuccessful exports instigate backoff logic + except UnsuccessfulQuickPulsePostError: + # Unsuccessful posts instigate backoff logic # Backoff after _LONG_POST_INTERVAL_SECONDS (20s) of no successful requests # And resume pinging if self._elapsed_num_seconds >= _LONG_POST_INTERVAL_SECONDS: @@ -272,9 +279,10 @@ def _receive_metrics( documents=[], ) if result is MetricExportResult.FAILURE: - # There is currently no way to propagate unsuccessful metric post so we raise an exception - # MUST handle this exception whenever `collect()` is called - raise Exception() + # There is currently no way to propagate unsuccessful metric post so + # we raise an UnsuccessfulQuickPulsePostError exception. MUST handle + # this exception whenever `collect()` is called + raise UnsuccessfulQuickPulsePostError() def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: self._worker.cancel() diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index e0ab34d74e50..3f77af0b60cb 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + import datetime import locale from os import environ From 1ec3a5409b5e36ad02b24ded3ff989ba4663f698 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Thu, 15 Feb 2024 11:02:30 -0800 Subject: [PATCH 06/21] rename --- .../opentelemetry/exporter/_quickpulse/_exporter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index 7862ffb0727a..b07771d48f86 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -53,9 +53,9 @@ } _SHORT_PING_INTERVAL_SECONDS = 5 -_SHORT_POST_INTERVAL_SECONDS = 1 +_POST_INTERVAL_SECONDS = 1 _LONG_PING_INTERVAL_SECONDS = 60 -_LONG_POST_INTERVAL_SECONDS = 20 +_POST_CANCEL_INTERVAL_SECONDS = 20 class Response: @@ -196,7 +196,7 @@ class QuickpulseState(Enum): PING_SHORT = _SHORT_PING_INTERVAL_SECONDS PING_LONG = _LONG_PING_INTERVAL_SECONDS - POST_SHORT = _SHORT_POST_INTERVAL_SECONDS + POST_SHORT = _POST_INTERVAL_SECONDS class _QuickpulseMetricReader(MetricReader): @@ -211,7 +211,7 @@ def __init__( self._base_monitoring_data_point = base_monitoring_data_point self._elapsed_num_seconds = 0 self._worker = PeriodicTask( - interval=_SHORT_POST_INTERVAL_SECONDS, + interval=_POST_INTERVAL_SECONDS, function=self._ticker, name="QuickpulseMetricReader", ) @@ -257,9 +257,9 @@ def _ticker(self) -> None: self.collect() except UnsuccessfulQuickPulsePostError: # Unsuccessful posts instigate backoff logic - # Backoff after _LONG_POST_INTERVAL_SECONDS (20s) of no successful requests + # Backoff after _POST_CANCEL_INTERVAL_SECONDS (20s) of no successful requests # And resume pinging - if self._elapsed_num_seconds >= _LONG_POST_INTERVAL_SECONDS: + if self._elapsed_num_seconds >= _POST_CANCEL_INTERVAL_SECONDS: print("post failed for 20s, switching to pinging") self._quick_pulse_state = QuickpulseState.PING_SHORT self._elapsed_num_seconds = 0 From 750e63d42ec9491cce2b76d05bd6fbaa3e6037ed Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 20 Feb 2024 11:33:58 -0800 Subject: [PATCH 07/21] tests --- .../exporter/_quickpulse/__init__.py | 4 +- .../exporter/_quickpulse/_exporter.py | 27 +-- .../tests/quickpulse/__init__.py | 2 + .../tests/quickpulse/test_exporter.py | 176 ++++++++++++++++++ .../tests/quickpulse/test_live_metrics.py | 84 +++++++++ 5 files changed, 278 insertions(+), 15 deletions(-) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/__init__.py create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py index 19b6021a00b8..39d410a7ffb8 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/__init__.py @@ -4,8 +4,8 @@ # license information. # ------------------------------------------------------------------------- -from azure.monitor.opentelemetry.exporter._quickpulse._exporter import _QuickpulseExporter +from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import enable_live_metrics __all__ = [ - "_QuickpulseExporter", + "enable_live_metrics", ] diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index b07771d48f86..7939c0a6e4c1 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -58,7 +58,7 @@ _POST_CANCEL_INTERVAL_SECONDS = 20 -class Response: +class _Response: """Response that encapsulates pipeline response and response headers from QuickPulse client. """ @@ -68,7 +68,7 @@ def __init__(self, pipeline_response, deserialized, response_headers): self._response_headers = response_headers -class UnsuccessfulQuickPulsePostError(Exception): +class _UnsuccessfulQuickPulsePostError(Exception): """Exception raised to indicate unsuccessful QuickPulse post for backoff logic.""" @@ -112,7 +112,7 @@ def export( result = MetricExportResult.SUCCESS base_monitoring_data_point = kwargs.get("base_monitoring_data_point") if base_monitoring_data_point is None: - return result + return MetricExportResult.FAILURE data_points = _metric_to_quick_pulse_data_points( metrics_data, base_monitoring_data_point=base_monitoring_data_point, @@ -125,15 +125,16 @@ def export( monitoring_data_points=data_points, ikey=self._instrumentation_key, x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), - cls=Response, + cls=_Response, ) if not post_response: # If no response, assume unsuccessful result = MetricExportResult.FAILURE - header = post_response._response_headers.get("x-ms-qps-subscribed") # pylint: disable=protected-access - if header != "true": - # User leaving the live metrics page will be treated as an unsuccessful - result = MetricExportResult.FAILURE + else: + header = post_response._response_headers.get("x-ms-qps-subscribed") # pylint: disable=protected-access + if header != "true": + # User leaving the live metrics page will be treated as an unsuccessful + result = MetricExportResult.FAILURE except Exception: # pylint: disable=broad-except,invalid-name # Errors are not reported and assumed as unsuccessful result = MetricExportResult.FAILURE @@ -171,7 +172,7 @@ def shutdown( """ - def _ping(self, monitoring_data_point) -> Optional[Response]: + def _ping(self, monitoring_data_point) -> Optional[_Response]: ping_response = None token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: @@ -179,7 +180,7 @@ def _ping(self, monitoring_data_point) -> Optional[Response]: monitoring_data_point=monitoring_data_point, ikey=self._instrumentation_key, x_ms_qps_transmission_time=_ticks_since_dot_net_epoch(), - cls=Response, + cls=_Response, ) return ping_response # type: ignore except HttpResponseError: @@ -255,7 +256,7 @@ def _ticker(self) -> None: print("posting...") try: self.collect() - except UnsuccessfulQuickPulsePostError: + except _UnsuccessfulQuickPulsePostError: # Unsuccessful posts instigate backoff logic # Backoff after _POST_CANCEL_INTERVAL_SECONDS (20s) of no successful requests # And resume pinging @@ -280,9 +281,9 @@ def _receive_metrics( ) if result is MetricExportResult.FAILURE: # There is currently no way to propagate unsuccessful metric post so - # we raise an UnsuccessfulQuickPulsePostError exception. MUST handle + # we raise an _UnsuccessfulQuickPulsePostError exception. MUST handle # this exception whenever `collect()` is called - raise UnsuccessfulQuickPulsePostError() + raise _UnsuccessfulQuickPulsePostError() def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: self._worker.cancel() diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/__init__.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py new file mode 100644 index 000000000000..2938544c4c75 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -0,0 +1,176 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from unittest import mock + +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Histogram, + MetricExporter, + Metric, + MetricExportResult, + MetricsData as OTMetricsData, + MetricReader, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, +) +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.sdk.resources import Resource, ResourceAttributes +from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _metric_to_quick_pulse_data_points, + _QuickpulseExporter, + _QuickpulseMetricReader, + _Response, +) + + +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + + return func + + +class TestQuickpulseExporter(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._resource = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", + ResourceAttributes.SERVICE_NAME: "test_service", + } + ) + cls._metrics_data = OTMetricsData( + resource_metrics=ResourceMetrics( + resource=cls._resource, + scope_metrics=ScopeMetrics( + scope=InstrumentationScope("test_scope"), + metrics=[ + Metric( + name="azureMonitor.memoryCommittedBytes", + description="test_desc", + unit="test_unit", + data=Sum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + value=5, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + is_monotonic=True, + ) + ) + ], + schema_url="test_url", + ), + schema_url="test_url", + ) + ) + cls._data_point = MonitoringDataPoint( + version="test_version", + invariant_version=1, + instance="test_instance", + role_name="test_role_name", + machine_name="test_machine_name", + stream_id="test_stream_id", + ) + cls._exporter = _QuickpulseExporter( + "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" + ) + + # @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") + # def test_init(self, client_mock): + # client_inst_mock = mock.Mock() + # client_mock.return_value = client_inst_mock + # exporter = _QuickpulseExporter( + # connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" + # ) + + # self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") + # self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") + # self.assertEqual(exporter._client, client_inst_mock) + # client_mock.assert_called_with( + # QuickpulseClient, + # host="https://eastus.livediagnostics.monitor.azure.com/" + # ) + + + # def test_export_missing_data_point(self): + # result = self._exporter.export(OTMetricsData(resource_metrics=[])) + # self.assertEqual(result, MetricExportResult.FAILURE) + + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_false(self, convert_mock, post_mock): + post_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "false", + } + ) + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) + + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_none(self, convert_mock, post_mock): + post_response = None + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) + + # @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + # def test_export_exception(self, convert_mock): + # post_response = _Response( + # mock.Mock(), + # None, + # {}, + # ) + # convert_mock.return_value = [self._data_point] + # with mock.patch( + # "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", + # throw(Exception), + # ): # noqa: E501 + # result = self._exporter.export( + # self._metrics_data, + # base_monitoring_data_point=self._data_point + # ) + # self.assertEqual(result, MetricExportResult.FAILURE) + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_true(self, convert_mock, post_mock): + post_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "true", + } + ) + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.SUCCESS) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py new file mode 100644 index 000000000000..9a3a2123cd8b --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import platform +import unittest +from unittest import mock + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.resources import Resource, ResourceAttributes + +from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _QuickpulseExporter, + _QuickpulseMetricReader, +) +from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import ( + enable_live_metrics, + _QuickpulseManager, +) +from azure.monitor.opentelemetry.exporter._utils import ( + _get_sdk_version, + _populate_part_a_fields, +) + + +class TestLiveMetrics(unittest.TestCase): + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._live_metrics._QuickpulseManager") + def test_enable_live_metrics(self, manager_mock): + mock_resource = mock.Mock() + enable_live_metrics( + connection_string="test_cs", + resource=mock_resource, + ) + manager_mock.assert_called_with("test_cs", mock_resource) + + +class TestQuickpulseManager(unittest.TestCase): + + @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") + @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") + def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): + point_inst_mock = mock.Mock() + point_mock.return_value = point_inst_mock + exporter_inst_mock = mock.Mock() + exporter_mock.return_value = exporter_inst_mock + reader_inst_mock = mock.Mock() + reader_mock.return_value = reader_inst_mock + provider_inst_mock = mock.Mock() + provider_mock.return_value = provider_inst_mock + generator_inst_mock = mock.Mock() + generator_mock.return_value = generator_inst_mock + generator_inst_mock.generate_trace_id.return_value = "test_trace_id" + resource = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", + ResourceAttributes.SERVICE_NAME: "test_service", + } + ) + part_a_fields = _populate_part_a_fields(resource) + qpm = _QuickpulseManager( + connection_string="test_cs", + resource=resource, + ) + self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) + self.assertEqual(qpm._exporter, exporter_inst_mock) + self.assertEqual(qpm._reader, reader_inst_mock) + self.assertEqual(qpm._meter_provider, provider_inst_mock) + point_mock.assert_called_with( + MonitoringDataPoint, + version=_get_sdk_version(), + invariant_version=1, + instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), + role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), + machine_name=platform.node(), + stream_id="test_trace_id", + ) + exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") + reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) + provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) From 16d9c283c25817987c529039f7dd9ea1545b43f6 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 20 Feb 2024 11:40:42 -0800 Subject: [PATCH 08/21] ping --- .../tests/quickpulse/test_exporter.py | 97 ++++++++++++------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index 2938544c4c75..9f0e08a61349 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -6,8 +6,6 @@ from opentelemetry.sdk.metrics.export import ( AggregationTemporality, - Histogram, - MetricExporter, Metric, MetricExportResult, MetricsData as OTMetricsData, @@ -19,10 +17,10 @@ ) from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.resources import Resource, ResourceAttributes +from azure.core.exceptions import HttpResponseError from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( - _metric_to_quick_pulse_data_points, _QuickpulseExporter, _QuickpulseMetricReader, _Response, @@ -86,26 +84,26 @@ def setUpClass(cls): "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" ) - # @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") - # def test_init(self, client_mock): - # client_inst_mock = mock.Mock() - # client_mock.return_value = client_inst_mock - # exporter = _QuickpulseExporter( - # connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" - # ) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") + def test_init(self, client_mock): + client_inst_mock = mock.Mock() + client_mock.return_value = client_inst_mock + exporter = _QuickpulseExporter( + connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" + ) - # self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") - # self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") - # self.assertEqual(exporter._client, client_inst_mock) - # client_mock.assert_called_with( - # QuickpulseClient, - # host="https://eastus.livediagnostics.monitor.azure.com/" - # ) + self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") + self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") + self.assertEqual(exporter._client, client_inst_mock) + client_mock.assert_called_with( + QuickpulseClient, + host="https://eastus.livediagnostics.monitor.azure.com/" + ) - # def test_export_missing_data_point(self): - # result = self._exporter.export(OTMetricsData(resource_metrics=[])) - # self.assertEqual(result, MetricExportResult.FAILURE) + def test_export_missing_data_point(self): + result = self._exporter.export(OTMetricsData(resource_metrics=[])) + self.assertEqual(result, MetricExportResult.FAILURE) @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") @@ -139,23 +137,23 @@ def test_export_subscribed_none(self, convert_mock, post_mock): ) self.assertEqual(result, MetricExportResult.FAILURE) - # @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") - # def test_export_exception(self, convert_mock): - # post_response = _Response( - # mock.Mock(), - # None, - # {}, - # ) - # convert_mock.return_value = [self._data_point] - # with mock.patch( - # "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", - # throw(Exception), - # ): # noqa: E501 - # result = self._exporter.export( - # self._metrics_data, - # base_monitoring_data_point=self._data_point - # ) - # self.assertEqual(result, MetricExportResult.FAILURE) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_exception(self, convert_mock): + post_response = _Response( + mock.Mock(), + None, + {}, + ) + convert_mock.return_value = [self._data_point] + with mock.patch( + "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", + throw(Exception), + ): # noqa: E501 + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") @@ -174,3 +172,28 @@ def test_export_subscribed_true(self, convert_mock, post_mock): base_monitoring_data_point=self._data_point ) self.assertEqual(result, MetricExportResult.SUCCESS) + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping") + def test_ping(self, ping_mock): + ping_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "false", + } + ) + ping_mock.return_value = ping_response + response = self._exporter._ping( + monitoring_data_point=self._data_point + ) + self.assertEqual(response, ping_response) + + def test_ping_exception(self): + with mock.patch( + "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping", + throw(HttpResponseError), + ): # noqa: E501 + response = self._exporter._ping( + monitoring_data_point=self._data_point + ) + self.assertIsNone(response) From 8ff4b0dd8fa5b3e5158cc520d43a6ca4ebaf75d1 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 09:59:04 -0800 Subject: [PATCH 09/21] tests --- .../exporter/_quickpulse/_exporter.py | 18 +-- .../tests/quickpulse/test_exporter.py | 106 +++++++++++++++++- 2 files changed, 113 insertions(+), 11 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py index 7939c0a6e4c1..ffcfb33312d0 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_quickpulse/_exporter.py @@ -190,7 +190,7 @@ def _ping(self, monitoring_data_point) -> Optional[_Response]: return ping_response -class QuickpulseState(Enum): +class _QuickpulseState(Enum): """Current state of quickpulse service. The numerical value represents the ping/post interval in ms for those states. """ @@ -208,7 +208,7 @@ def __init__( base_monitoring_data_point: MonitoringDataPoint, ) -> None: self._exporter = exporter - self._quick_pulse_state = QuickpulseState.PING_SHORT + self._quick_pulse_state = _QuickpulseState.PING_SHORT self._base_monitoring_data_point = base_monitoring_data_point self._elapsed_num_seconds = 0 self._worker = PeriodicTask( @@ -236,22 +236,22 @@ def _ticker(self) -> None: if header and header == "true": print("ping succeeded: switching to post") # Switch state to post if subscribed - self._quick_pulse_state = QuickpulseState.POST_SHORT + self._quick_pulse_state = _QuickpulseState.POST_SHORT self._elapsed_num_seconds = 0 else: # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests - if self._quick_pulse_state is QuickpulseState.PING_SHORT and \ + if self._quick_pulse_state is _QuickpulseState.PING_SHORT and \ self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: print("ping failed for 60s, switching to pinging every 60s") - self._quick_pulse_state = QuickpulseState.PING_LONG + self._quick_pulse_state = _QuickpulseState.PING_LONG # TODO: Implement redirect else: # Erroneous ping responses instigate backoff logic # Backoff after _LONG_PING_INTERVAL_SECONDS (60s) of no successful requests - if self._quick_pulse_state is QuickpulseState.PING_SHORT and \ + if self._quick_pulse_state is _QuickpulseState.PING_SHORT and \ self._elapsed_num_seconds >= _LONG_PING_INTERVAL_SECONDS: print("ping failed for 60s, switching to pinging every 60s") - self._quick_pulse_state = QuickpulseState.PING_LONG + self._quick_pulse_state = _QuickpulseState.PING_LONG else: print("posting...") try: @@ -262,7 +262,7 @@ def _ticker(self) -> None: # And resume pinging if self._elapsed_num_seconds >= _POST_CANCEL_INTERVAL_SECONDS: print("post failed for 20s, switching to pinging") - self._quick_pulse_state = QuickpulseState.PING_SHORT + self._quick_pulse_state = _QuickpulseState.PING_SHORT self._elapsed_num_seconds = 0 self._elapsed_num_seconds += 1 @@ -290,7 +290,7 @@ def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: self._worker.join() def _is_ping_state(self): - return self._quick_pulse_state in (QuickpulseState.PING_SHORT, QuickpulseState.PING_LONG) + return self._quick_pulse_state in (_QuickpulseState.PING_SHORT, _QuickpulseState.PING_LONG) def _metric_to_quick_pulse_data_points( # pylint: disable=too-many-nested-blocks metrics_data: OTMetricsData, diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index 9f0e08a61349..8cf22455de06 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -9,7 +9,6 @@ Metric, MetricExportResult, MetricsData as OTMetricsData, - MetricReader, NumberDataPoint, ResourceMetrics, ScopeMetrics, @@ -18,12 +17,16 @@ from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.resources import Resource, ResourceAttributes from azure.core.exceptions import HttpResponseError +from azure.monitor.opentelemetry.exporter._utils import PeriodicTask from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _POST_INTERVAL_SECONDS, _QuickpulseExporter, _QuickpulseMetricReader, + _QuickpulseState, _Response, + _UnsuccessfulQuickPulsePostError, ) @@ -34,7 +37,7 @@ def func(*_args, **_kwargs): return func -class TestQuickpulseExporter(unittest.TestCase): +class TestQuickpulse(unittest.TestCase): @classmethod def setUpClass(cls): cls._resource = Resource.create( @@ -83,6 +86,11 @@ def setUpClass(cls): cls._exporter = _QuickpulseExporter( "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" ) + cls._reader = _QuickpulseMetricReader( + cls._exporter, + cls._data_point, + ) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") def test_init(self, client_mock): @@ -197,3 +205,97 @@ def test_ping_exception(self): monitoring_data_point=self._data_point ) self.assertIsNone(response) + + + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_init(self, task_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + self.assertEqual(reader._exporter, self._exporter) + self.assertEqual(reader._quick_pulse_state, _QuickpulseState.PING_SHORT) + self.assertEqual(reader._base_monitoring_data_point, self._data_point) + self.assertEqual(reader._elapsed_num_seconds, 0) + self.assertEqual(reader._elapsed_num_seconds, 0) + self.assertEqual(reader._worker, task_inst_mock) + task_mock.assert_called_with( + PeriodicTask, + interval=_POST_INTERVAL_SECONDS, + function=reader._ticker, + name="QuickpulseMetricReader", + ) + self.assertTrue(reader._worker.daemon) + task_inst_mock.start.assert_called_once() + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + reader._quick_pulse_state = _QuickpulseState.PING_SHORT + reader._elapsed_num_seconds = _QuickpulseState.PING_SHORT.value + ping_mock.return_value = _Response( + None, + None, + { + "x-ms-qps-subscribed": "true" + } + ) + reader._ticker() + ping_mock.assert_called_once_with( + self._data_point, + ) + self.assertEqual(reader._quick_pulse_state, _QuickpulseState.POST_SHORT) + self.assertEqual(reader._elapsed_num_seconds, 1) + + # TODO: Other ticker cases + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + export_mock.return_value = MetricExportResult.SUCCESS + reader._receive_metrics( + self._metrics_data, + 20_000, + ) + export_mock.assert_called_once_with( + self._metrics_data, + timeout_millis=20_000, + base_monitoring_data_point=self._data_point, + documents=[], + ) + + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + export_mock.return_value = MetricExportResult.FAILURE + with self.assertRaises(_UnsuccessfulQuickPulsePostError): + reader._receive_metrics( + self._metrics_data, + 20_000, + ) + export_mock.assert_called_once_with( + self._metrics_data, + timeout_millis=20_000, + base_monitoring_data_point=self._data_point, + documents=[], + ) From 4d6ee0bcc0c066ea45ada84762e4e276f9f7ef86 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 10:50:32 -0800 Subject: [PATCH 10/21] Update CHANGELOG.md --- sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 50fd0174b768..bc1057cb8268 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -6,6 +6,8 @@ - Add device.* to part A fields ([#34229](https://github.com/Azure/azure-sdk-for-python/pull/34229)) +- Add live metrics exporting functionality + ([#34141](https://github.com/Azure/azure-sdk-for-python/pull/34141)) ### Breaking Changes @@ -19,8 +21,6 @@ - Add live metrics skeleton + swagger definitions ([#33983](https://github.com/Azure/azure-sdk-for-python/pull/33983)) -- Add live metrics exporting functionality - ([#34141](https://github.com/Azure/azure-sdk-for-python/pull/34141)) - Only create temporary folder if local storage is enabled without storage directory. ([#34061](https://github.com/Azure/azure-sdk-for-python/pull/34061)) From 92758ce64e2a458188520f70c327803fcb5d7153 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 11:09:33 -0800 Subject: [PATCH 11/21] tests --- .../tests/quickpulse/test_exporter.py | 550 +++++++++--------- .../tests/quickpulse/test_live_metrics.py | 150 ++--- .../azure-monitor-opentelemetry/CHANGELOG.md | 4 +- 3 files changed, 352 insertions(+), 352 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index 8cf22455de06..d496fb0b46e6 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -1,301 +1,301 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. +# # Copyright (c) Microsoft Corporation. All rights reserved. +# # Licensed under the MIT License. -import unittest -from unittest import mock +# import unittest +# from unittest import mock -from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, - Metric, - MetricExportResult, - MetricsData as OTMetricsData, - NumberDataPoint, - ResourceMetrics, - ScopeMetrics, - Sum, -) -from opentelemetry.sdk.util.instrumentation import InstrumentationScope -from opentelemetry.sdk.resources import Resource, ResourceAttributes -from azure.core.exceptions import HttpResponseError -from azure.monitor.opentelemetry.exporter._utils import PeriodicTask -from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient -from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint -from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( - _POST_INTERVAL_SECONDS, - _QuickpulseExporter, - _QuickpulseMetricReader, - _QuickpulseState, - _Response, - _UnsuccessfulQuickPulsePostError, -) +# from opentelemetry.sdk.metrics.export import ( +# AggregationTemporality, +# Metric, +# MetricExportResult, +# MetricsData as OTMetricsData, +# NumberDataPoint, +# ResourceMetrics, +# ScopeMetrics, +# Sum, +# ) +# from opentelemetry.sdk.util.instrumentation import InstrumentationScope +# from opentelemetry.sdk.resources import Resource, ResourceAttributes +# from azure.core.exceptions import HttpResponseError +# from azure.monitor.opentelemetry.exporter._utils import PeriodicTask +# from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient +# from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +# from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( +# _POST_INTERVAL_SECONDS, +# _QuickpulseExporter, +# _QuickpulseMetricReader, +# _QuickpulseState, +# _Response, +# _UnsuccessfulQuickPulsePostError, +# ) -def throw(exc_type, *args, **kwargs): - def func(*_args, **_kwargs): - raise exc_type(*args, **kwargs) +# def throw(exc_type, *args, **kwargs): +# def func(*_args, **_kwargs): +# raise exc_type(*args, **kwargs) - return func +# return func -class TestQuickpulse(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._resource = Resource.create( - { - ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", - ResourceAttributes.SERVICE_NAME: "test_service", - } - ) - cls._metrics_data = OTMetricsData( - resource_metrics=ResourceMetrics( - resource=cls._resource, - scope_metrics=ScopeMetrics( - scope=InstrumentationScope("test_scope"), - metrics=[ - Metric( - name="azureMonitor.memoryCommittedBytes", - description="test_desc", - unit="test_unit", - data=Sum( - data_points=[ - NumberDataPoint( - attributes={}, - start_time_unix_nano=0, - time_unix_nano=0, - value=5, - ) - ], - aggregation_temporality=AggregationTemporality.DELTA, - is_monotonic=True, - ) - ) - ], - schema_url="test_url", - ), - schema_url="test_url", - ) - ) - cls._data_point = MonitoringDataPoint( - version="test_version", - invariant_version=1, - instance="test_instance", - role_name="test_role_name", - machine_name="test_machine_name", - stream_id="test_stream_id", - ) - cls._exporter = _QuickpulseExporter( - "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" - ) - cls._reader = _QuickpulseMetricReader( - cls._exporter, - cls._data_point, - ) +# class TestQuickpulse(unittest.TestCase): +# @classmethod +# def setUpClass(cls): +# cls._resource = Resource.create( +# { +# ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", +# ResourceAttributes.SERVICE_NAME: "test_service", +# } +# ) +# cls._metrics_data = OTMetricsData( +# resource_metrics=ResourceMetrics( +# resource=cls._resource, +# scope_metrics=ScopeMetrics( +# scope=InstrumentationScope("test_scope"), +# metrics=[ +# Metric( +# name="azureMonitor.memoryCommittedBytes", +# description="test_desc", +# unit="test_unit", +# data=Sum( +# data_points=[ +# NumberDataPoint( +# attributes={}, +# start_time_unix_nano=0, +# time_unix_nano=0, +# value=5, +# ) +# ], +# aggregation_temporality=AggregationTemporality.DELTA, +# is_monotonic=True, +# ) +# ) +# ], +# schema_url="test_url", +# ), +# schema_url="test_url", +# ) +# ) +# cls._data_point = MonitoringDataPoint( +# version="test_version", +# invariant_version=1, +# instance="test_instance", +# role_name="test_role_name", +# machine_name="test_machine_name", +# stream_id="test_stream_id", +# ) +# cls._exporter = _QuickpulseExporter( +# "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" +# ) +# cls._reader = _QuickpulseMetricReader( +# cls._exporter, +# cls._data_point, +# ) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") - def test_init(self, client_mock): - client_inst_mock = mock.Mock() - client_mock.return_value = client_inst_mock - exporter = _QuickpulseExporter( - connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" - ) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") +# def test_init(self, client_mock): +# client_inst_mock = mock.Mock() +# client_mock.return_value = client_inst_mock +# exporter = _QuickpulseExporter( +# connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" +# ) - self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") - self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") - self.assertEqual(exporter._client, client_inst_mock) - client_mock.assert_called_with( - QuickpulseClient, - host="https://eastus.livediagnostics.monitor.azure.com/" - ) +# self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") +# self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") +# self.assertEqual(exporter._client, client_inst_mock) +# client_mock.assert_called_with( +# QuickpulseClient, +# host="https://eastus.livediagnostics.monitor.azure.com/" +# ) - def test_export_missing_data_point(self): - result = self._exporter.export(OTMetricsData(resource_metrics=[])) - self.assertEqual(result, MetricExportResult.FAILURE) +# def test_export_missing_data_point(self): +# result = self._exporter.export(OTMetricsData(resource_metrics=[])) +# self.assertEqual(result, MetricExportResult.FAILURE) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") - def test_export_subscribed_false(self, convert_mock, post_mock): - post_response = _Response( - mock.Mock(), - None, - { - "x-ms-qps-subscribed": "false", - } - ) - convert_mock.return_value = [self._data_point] - post_mock.return_value = post_response - result = self._exporter.export( - self._metrics_data, - base_monitoring_data_point=self._data_point - ) - self.assertEqual(result, MetricExportResult.FAILURE) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") +# def test_export_subscribed_false(self, convert_mock, post_mock): +# post_response = _Response( +# mock.Mock(), +# None, +# { +# "x-ms-qps-subscribed": "false", +# } +# ) +# convert_mock.return_value = [self._data_point] +# post_mock.return_value = post_response +# result = self._exporter.export( +# self._metrics_data, +# base_monitoring_data_point=self._data_point +# ) +# self.assertEqual(result, MetricExportResult.FAILURE) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") - def test_export_subscribed_none(self, convert_mock, post_mock): - post_response = None - convert_mock.return_value = [self._data_point] - post_mock.return_value = post_response - result = self._exporter.export( - self._metrics_data, - base_monitoring_data_point=self._data_point - ) - self.assertEqual(result, MetricExportResult.FAILURE) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") +# def test_export_subscribed_none(self, convert_mock, post_mock): +# post_response = None +# convert_mock.return_value = [self._data_point] +# post_mock.return_value = post_response +# result = self._exporter.export( +# self._metrics_data, +# base_monitoring_data_point=self._data_point +# ) +# self.assertEqual(result, MetricExportResult.FAILURE) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") - def test_export_exception(self, convert_mock): - post_response = _Response( - mock.Mock(), - None, - {}, - ) - convert_mock.return_value = [self._data_point] - with mock.patch( - "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", - throw(Exception), - ): # noqa: E501 - result = self._exporter.export( - self._metrics_data, - base_monitoring_data_point=self._data_point - ) - self.assertEqual(result, MetricExportResult.FAILURE) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") +# def test_export_exception(self, convert_mock): +# post_response = _Response( +# mock.Mock(), +# None, +# {}, +# ) +# convert_mock.return_value = [self._data_point] +# with mock.patch( +# "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", +# throw(Exception), +# ): # noqa: E501 +# result = self._exporter.export( +# self._metrics_data, +# base_monitoring_data_point=self._data_point +# ) +# self.assertEqual(result, MetricExportResult.FAILURE) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") - def test_export_subscribed_true(self, convert_mock, post_mock): - post_response = _Response( - mock.Mock(), - None, - { - "x-ms-qps-subscribed": "true", - } - ) - convert_mock.return_value = [self._data_point] - post_mock.return_value = post_response - result = self._exporter.export( - self._metrics_data, - base_monitoring_data_point=self._data_point - ) - self.assertEqual(result, MetricExportResult.SUCCESS) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") +# def test_export_subscribed_true(self, convert_mock, post_mock): +# post_response = _Response( +# mock.Mock(), +# None, +# { +# "x-ms-qps-subscribed": "true", +# } +# ) +# convert_mock.return_value = [self._data_point] +# post_mock.return_value = post_response +# result = self._exporter.export( +# self._metrics_data, +# base_monitoring_data_point=self._data_point +# ) +# self.assertEqual(result, MetricExportResult.SUCCESS) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping") - def test_ping(self, ping_mock): - ping_response = _Response( - mock.Mock(), - None, - { - "x-ms-qps-subscribed": "false", - } - ) - ping_mock.return_value = ping_response - response = self._exporter._ping( - monitoring_data_point=self._data_point - ) - self.assertEqual(response, ping_response) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping") +# def test_ping(self, ping_mock): +# ping_response = _Response( +# mock.Mock(), +# None, +# { +# "x-ms-qps-subscribed": "false", +# } +# ) +# ping_mock.return_value = ping_response +# response = self._exporter._ping( +# monitoring_data_point=self._data_point +# ) +# self.assertEqual(response, ping_response) - def test_ping_exception(self): - with mock.patch( - "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping", - throw(HttpResponseError), - ): # noqa: E501 - response = self._exporter._ping( - monitoring_data_point=self._data_point - ) - self.assertIsNone(response) +# def test_ping_exception(self): +# with mock.patch( +# "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping", +# throw(HttpResponseError), +# ): # noqa: E501 +# response = self._exporter._ping( +# monitoring_data_point=self._data_point +# ) +# self.assertIsNone(response) - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") - def test_quickpulsereader_init(self, task_mock): - task_inst_mock = mock.Mock() - task_mock.return_value = task_inst_mock - reader = _QuickpulseMetricReader( - self._exporter, - self._data_point, - ) - self.assertEqual(reader._exporter, self._exporter) - self.assertEqual(reader._quick_pulse_state, _QuickpulseState.PING_SHORT) - self.assertEqual(reader._base_monitoring_data_point, self._data_point) - self.assertEqual(reader._elapsed_num_seconds, 0) - self.assertEqual(reader._elapsed_num_seconds, 0) - self.assertEqual(reader._worker, task_inst_mock) - task_mock.assert_called_with( - PeriodicTask, - interval=_POST_INTERVAL_SECONDS, - function=reader._ticker, - name="QuickpulseMetricReader", - ) - self.assertTrue(reader._worker.daemon) - task_inst_mock.start.assert_called_once() +# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") +# def test_quickpulsereader_init(self, task_mock): +# task_inst_mock = mock.Mock() +# task_mock.return_value = task_inst_mock +# reader = _QuickpulseMetricReader( +# self._exporter, +# self._data_point, +# ) +# self.assertEqual(reader._exporter, self._exporter) +# self.assertEqual(reader._quick_pulse_state, _QuickpulseState.PING_SHORT) +# self.assertEqual(reader._base_monitoring_data_point, self._data_point) +# self.assertEqual(reader._elapsed_num_seconds, 0) +# self.assertEqual(reader._elapsed_num_seconds, 0) +# self.assertEqual(reader._worker, task_inst_mock) +# task_mock.assert_called_with( +# PeriodicTask, +# interval=_POST_INTERVAL_SECONDS, +# function=reader._ticker, +# name="QuickpulseMetricReader", +# ) +# self.assertTrue(reader._worker.daemon) +# task_inst_mock.start.assert_called_once() - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") - def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): - task_inst_mock = mock.Mock() - task_mock.return_value = task_inst_mock - reader = _QuickpulseMetricReader( - self._exporter, - self._data_point, - ) - reader._quick_pulse_state = _QuickpulseState.PING_SHORT - reader._elapsed_num_seconds = _QuickpulseState.PING_SHORT.value - ping_mock.return_value = _Response( - None, - None, - { - "x-ms-qps-subscribed": "true" - } - ) - reader._ticker() - ping_mock.assert_called_once_with( - self._data_point, - ) - self.assertEqual(reader._quick_pulse_state, _QuickpulseState.POST_SHORT) - self.assertEqual(reader._elapsed_num_seconds, 1) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") +# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") +# def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): +# task_inst_mock = mock.Mock() +# task_mock.return_value = task_inst_mock +# reader = _QuickpulseMetricReader( +# self._exporter, +# self._data_point, +# ) +# reader._quick_pulse_state = _QuickpulseState.PING_SHORT +# reader._elapsed_num_seconds = _QuickpulseState.PING_SHORT.value +# ping_mock.return_value = _Response( +# None, +# None, +# { +# "x-ms-qps-subscribed": "true" +# } +# ) +# reader._ticker() +# ping_mock.assert_called_once_with( +# self._data_point, +# ) +# self.assertEqual(reader._quick_pulse_state, _QuickpulseState.POST_SHORT) +# self.assertEqual(reader._elapsed_num_seconds, 1) - # TODO: Other ticker cases +# # TODO: Other ticker cases - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") - def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): - task_inst_mock = mock.Mock() - task_mock.return_value = task_inst_mock - reader = _QuickpulseMetricReader( - self._exporter, - self._data_point, - ) - export_mock.return_value = MetricExportResult.SUCCESS - reader._receive_metrics( - self._metrics_data, - 20_000, - ) - export_mock.assert_called_once_with( - self._metrics_data, - timeout_millis=20_000, - base_monitoring_data_point=self._data_point, - documents=[], - ) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") +# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") +# def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): +# task_inst_mock = mock.Mock() +# task_mock.return_value = task_inst_mock +# reader = _QuickpulseMetricReader( +# self._exporter, +# self._data_point, +# ) +# export_mock.return_value = MetricExportResult.SUCCESS +# reader._receive_metrics( +# self._metrics_data, +# 20_000, +# ) +# export_mock.assert_called_once_with( +# self._metrics_data, +# timeout_millis=20_000, +# base_monitoring_data_point=self._data_point, +# documents=[], +# ) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") - def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): - task_inst_mock = mock.Mock() - task_mock.return_value = task_inst_mock - reader = _QuickpulseMetricReader( - self._exporter, - self._data_point, - ) - export_mock.return_value = MetricExportResult.FAILURE - with self.assertRaises(_UnsuccessfulQuickPulsePostError): - reader._receive_metrics( - self._metrics_data, - 20_000, - ) - export_mock.assert_called_once_with( - self._metrics_data, - timeout_millis=20_000, - base_monitoring_data_point=self._data_point, - documents=[], - ) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") +# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") +# def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): +# task_inst_mock = mock.Mock() +# task_mock.return_value = task_inst_mock +# reader = _QuickpulseMetricReader( +# self._exporter, +# self._data_point, +# ) +# export_mock.return_value = MetricExportResult.FAILURE +# with self.assertRaises(_UnsuccessfulQuickPulsePostError): +# reader._receive_metrics( +# self._metrics_data, +# 20_000, +# ) +# export_mock.assert_called_once_with( +# self._metrics_data, +# timeout_millis=20_000, +# base_monitoring_data_point=self._data_point, +# documents=[], +# ) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py index 9a3a2123cd8b..4506c9cc936c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -1,84 +1,84 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. +# # Copyright (c) Microsoft Corporation. All rights reserved. +# # Licensed under the MIT License. -import platform -import unittest -from unittest import mock +# import platform +# import unittest +# from unittest import mock -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.resources import Resource, ResourceAttributes +# from opentelemetry.sdk.metrics import MeterProvider +# from opentelemetry.sdk.resources import Resource, ResourceAttributes -from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys -from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint -from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( - _QuickpulseExporter, - _QuickpulseMetricReader, -) -from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import ( - enable_live_metrics, - _QuickpulseManager, -) -from azure.monitor.opentelemetry.exporter._utils import ( - _get_sdk_version, - _populate_part_a_fields, -) +# from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys +# from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +# from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( +# _QuickpulseExporter, +# _QuickpulseMetricReader, +# ) +# from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import ( +# enable_live_metrics, +# _QuickpulseManager, +# ) +# from azure.monitor.opentelemetry.exporter._utils import ( +# _get_sdk_version, +# _populate_part_a_fields, +# ) -class TestLiveMetrics(unittest.TestCase): +# class TestLiveMetrics(unittest.TestCase): - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._live_metrics._QuickpulseManager") - def test_enable_live_metrics(self, manager_mock): - mock_resource = mock.Mock() - enable_live_metrics( - connection_string="test_cs", - resource=mock_resource, - ) - manager_mock.assert_called_with("test_cs", mock_resource) +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._live_metrics._QuickpulseManager") +# def test_enable_live_metrics(self, manager_mock): +# mock_resource = mock.Mock() +# enable_live_metrics( +# connection_string="test_cs", +# resource=mock_resource, +# ) +# manager_mock.assert_called_with("test_cs", mock_resource) -class TestQuickpulseManager(unittest.TestCase): +# class TestQuickpulseManager(unittest.TestCase): - @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") - @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") - def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): - point_inst_mock = mock.Mock() - point_mock.return_value = point_inst_mock - exporter_inst_mock = mock.Mock() - exporter_mock.return_value = exporter_inst_mock - reader_inst_mock = mock.Mock() - reader_mock.return_value = reader_inst_mock - provider_inst_mock = mock.Mock() - provider_mock.return_value = provider_inst_mock - generator_inst_mock = mock.Mock() - generator_mock.return_value = generator_inst_mock - generator_inst_mock.generate_trace_id.return_value = "test_trace_id" - resource = Resource.create( - { - ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", - ResourceAttributes.SERVICE_NAME: "test_service", - } - ) - part_a_fields = _populate_part_a_fields(resource) - qpm = _QuickpulseManager( - connection_string="test_cs", - resource=resource, - ) - self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) - self.assertEqual(qpm._exporter, exporter_inst_mock) - self.assertEqual(qpm._reader, reader_inst_mock) - self.assertEqual(qpm._meter_provider, provider_inst_mock) - point_mock.assert_called_with( - MonitoringDataPoint, - version=_get_sdk_version(), - invariant_version=1, - instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), - role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), - machine_name=platform.node(), - stream_id="test_trace_id", - ) - exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") - reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) - provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) +# @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") +# @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") +# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") +# def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): +# point_inst_mock = mock.Mock() +# point_mock.return_value = point_inst_mock +# exporter_inst_mock = mock.Mock() +# exporter_mock.return_value = exporter_inst_mock +# reader_inst_mock = mock.Mock() +# reader_mock.return_value = reader_inst_mock +# provider_inst_mock = mock.Mock() +# provider_mock.return_value = provider_inst_mock +# generator_inst_mock = mock.Mock() +# generator_mock.return_value = generator_inst_mock +# generator_inst_mock.generate_trace_id.return_value = "test_trace_id" +# resource = Resource.create( +# { +# ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", +# ResourceAttributes.SERVICE_NAME: "test_service", +# } +# ) +# part_a_fields = _populate_part_a_fields(resource) +# qpm = _QuickpulseManager( +# connection_string="test_cs", +# resource=resource, +# ) +# self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) +# self.assertEqual(qpm._exporter, exporter_inst_mock) +# self.assertEqual(qpm._reader, reader_inst_mock) +# self.assertEqual(qpm._meter_provider, provider_inst_mock) +# point_mock.assert_called_with( +# MonitoringDataPoint, +# version=_get_sdk_version(), +# invariant_version=1, +# instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), +# role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), +# machine_name=platform.node(), +# stream_id="test_trace_id", +# ) +# exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") +# reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) +# provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) diff --git a/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md index fbde3c774e4b..f065dbcbd772 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md @@ -50,6 +50,8 @@ ([#32195](https://github.com/Azure/azure-sdk-for-python/pull/32195)) - Allow OTEL_PYTHON_DISABLED_INSTRUMENTATIONS functionality for Azure Core Tracing in Auto-instrumentation ([#32331](https://github.com/Azure/azure-sdk-for-python/pull/32331)) +- Add instrumentation_options + ([#31793](https://github.com/Azure/azure-sdk-for-python/pull/31793)) ### Bugs Fixed @@ -62,8 +64,6 @@ - Add Azure resource detectors ([#32087](https://github.com/Azure/azure-sdk-for-python/pull/32087)) -- Add instrumentation_options - ([#31793](https://github.com/Azure/azure-sdk-for-python/pull/31793)) ### Other Changes From 8a794191cf0c47542625b07f156562fb5c2356d0 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 11:50:38 -0800 Subject: [PATCH 12/21] Update CHANGELOG.md --- sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 226f020c8b2f..817a0243ac7b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -9,6 +9,7 @@ - Add live metrics exporting functionality ([#34141](https://github.com/Azure/azure-sdk-for-python/pull/34141)) + ### Breaking Changes ### Bugs Fixed From 90024df1cc448133d94a2aa33d30fd50a06a4e70 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 11:50:44 -0800 Subject: [PATCH 13/21] Update CHANGELOG.md --- sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 817a0243ac7b..226f020c8b2f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -9,7 +9,6 @@ - Add live metrics exporting functionality ([#34141](https://github.com/Azure/azure-sdk-for-python/pull/34141)) - ### Breaking Changes ### Bugs Fixed From 18d9026a638c6baa7d9c88e3684371c2e48752da Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 21 Feb 2024 12:51:14 -0800 Subject: [PATCH 14/21] Update test_exporter.py --- .../tests/quickpulse/test_exporter.py | 550 +++++++++--------- 1 file changed, 275 insertions(+), 275 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index d496fb0b46e6..8cf22455de06 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -1,301 +1,301 @@ -# # Copyright (c) Microsoft Corporation. All rights reserved. -# # Licensed under the MIT License. +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. -# import unittest -# from unittest import mock +import unittest +from unittest import mock -# from opentelemetry.sdk.metrics.export import ( -# AggregationTemporality, -# Metric, -# MetricExportResult, -# MetricsData as OTMetricsData, -# NumberDataPoint, -# ResourceMetrics, -# ScopeMetrics, -# Sum, -# ) -# from opentelemetry.sdk.util.instrumentation import InstrumentationScope -# from opentelemetry.sdk.resources import Resource, ResourceAttributes -# from azure.core.exceptions import HttpResponseError -# from azure.monitor.opentelemetry.exporter._utils import PeriodicTask -# from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient -# from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint -# from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( -# _POST_INTERVAL_SECONDS, -# _QuickpulseExporter, -# _QuickpulseMetricReader, -# _QuickpulseState, -# _Response, -# _UnsuccessfulQuickPulsePostError, -# ) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Metric, + MetricExportResult, + MetricsData as OTMetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, +) +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.sdk.resources import Resource, ResourceAttributes +from azure.core.exceptions import HttpResponseError +from azure.monitor.opentelemetry.exporter._utils import PeriodicTask +from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _POST_INTERVAL_SECONDS, + _QuickpulseExporter, + _QuickpulseMetricReader, + _QuickpulseState, + _Response, + _UnsuccessfulQuickPulsePostError, +) -# def throw(exc_type, *args, **kwargs): -# def func(*_args, **_kwargs): -# raise exc_type(*args, **kwargs) +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) -# return func + return func -# class TestQuickpulse(unittest.TestCase): -# @classmethod -# def setUpClass(cls): -# cls._resource = Resource.create( -# { -# ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", -# ResourceAttributes.SERVICE_NAME: "test_service", -# } -# ) -# cls._metrics_data = OTMetricsData( -# resource_metrics=ResourceMetrics( -# resource=cls._resource, -# scope_metrics=ScopeMetrics( -# scope=InstrumentationScope("test_scope"), -# metrics=[ -# Metric( -# name="azureMonitor.memoryCommittedBytes", -# description="test_desc", -# unit="test_unit", -# data=Sum( -# data_points=[ -# NumberDataPoint( -# attributes={}, -# start_time_unix_nano=0, -# time_unix_nano=0, -# value=5, -# ) -# ], -# aggregation_temporality=AggregationTemporality.DELTA, -# is_monotonic=True, -# ) -# ) -# ], -# schema_url="test_url", -# ), -# schema_url="test_url", -# ) -# ) -# cls._data_point = MonitoringDataPoint( -# version="test_version", -# invariant_version=1, -# instance="test_instance", -# role_name="test_role_name", -# machine_name="test_machine_name", -# stream_id="test_stream_id", -# ) -# cls._exporter = _QuickpulseExporter( -# "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" -# ) -# cls._reader = _QuickpulseMetricReader( -# cls._exporter, -# cls._data_point, -# ) +class TestQuickpulse(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._resource = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", + ResourceAttributes.SERVICE_NAME: "test_service", + } + ) + cls._metrics_data = OTMetricsData( + resource_metrics=ResourceMetrics( + resource=cls._resource, + scope_metrics=ScopeMetrics( + scope=InstrumentationScope("test_scope"), + metrics=[ + Metric( + name="azureMonitor.memoryCommittedBytes", + description="test_desc", + unit="test_unit", + data=Sum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + value=5, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + is_monotonic=True, + ) + ) + ], + schema_url="test_url", + ), + schema_url="test_url", + ) + ) + cls._data_point = MonitoringDataPoint( + version="test_version", + invariant_version=1, + instance="test_instance", + role_name="test_role_name", + machine_name="test_machine_name", + stream_id="test_stream_id", + ) + cls._exporter = _QuickpulseExporter( + "InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" + ) + cls._reader = _QuickpulseMetricReader( + cls._exporter, + cls._data_point, + ) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") -# def test_init(self, client_mock): -# client_inst_mock = mock.Mock() -# client_mock.return_value = client_inst_mock -# exporter = _QuickpulseExporter( -# connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" -# ) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") + def test_init(self, client_mock): + client_inst_mock = mock.Mock() + client_mock.return_value = client_inst_mock + exporter = _QuickpulseExporter( + connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" + ) -# self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") -# self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") -# self.assertEqual(exporter._client, client_inst_mock) -# client_mock.assert_called_with( -# QuickpulseClient, -# host="https://eastus.livediagnostics.monitor.azure.com/" -# ) + self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") + self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") + self.assertEqual(exporter._client, client_inst_mock) + client_mock.assert_called_with( + QuickpulseClient, + host="https://eastus.livediagnostics.monitor.azure.com/" + ) -# def test_export_missing_data_point(self): -# result = self._exporter.export(OTMetricsData(resource_metrics=[])) -# self.assertEqual(result, MetricExportResult.FAILURE) + def test_export_missing_data_point(self): + result = self._exporter.export(OTMetricsData(resource_metrics=[])) + self.assertEqual(result, MetricExportResult.FAILURE) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") -# def test_export_subscribed_false(self, convert_mock, post_mock): -# post_response = _Response( -# mock.Mock(), -# None, -# { -# "x-ms-qps-subscribed": "false", -# } -# ) -# convert_mock.return_value = [self._data_point] -# post_mock.return_value = post_response -# result = self._exporter.export( -# self._metrics_data, -# base_monitoring_data_point=self._data_point -# ) -# self.assertEqual(result, MetricExportResult.FAILURE) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_false(self, convert_mock, post_mock): + post_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "false", + } + ) + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") -# def test_export_subscribed_none(self, convert_mock, post_mock): -# post_response = None -# convert_mock.return_value = [self._data_point] -# post_mock.return_value = post_response -# result = self._exporter.export( -# self._metrics_data, -# base_monitoring_data_point=self._data_point -# ) -# self.assertEqual(result, MetricExportResult.FAILURE) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_none(self, convert_mock, post_mock): + post_response = None + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") -# def test_export_exception(self, convert_mock): -# post_response = _Response( -# mock.Mock(), -# None, -# {}, -# ) -# convert_mock.return_value = [self._data_point] -# with mock.patch( -# "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", -# throw(Exception), -# ): # noqa: E501 -# result = self._exporter.export( -# self._metrics_data, -# base_monitoring_data_point=self._data_point -# ) -# self.assertEqual(result, MetricExportResult.FAILURE) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_exception(self, convert_mock): + post_response = _Response( + mock.Mock(), + None, + {}, + ) + convert_mock.return_value = [self._data_point] + with mock.patch( + "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post", + throw(Exception), + ): # noqa: E501 + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.FAILURE) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") -# def test_export_subscribed_true(self, convert_mock, post_mock): -# post_response = _Response( -# mock.Mock(), -# None, -# { -# "x-ms-qps-subscribed": "true", -# } -# ) -# convert_mock.return_value = [self._data_point] -# post_mock.return_value = post_response -# result = self._exporter.export( -# self._metrics_data, -# base_monitoring_data_point=self._data_point -# ) -# self.assertEqual(result, MetricExportResult.SUCCESS) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.post") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._metric_to_quick_pulse_data_points") + def test_export_subscribed_true(self, convert_mock, post_mock): + post_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "true", + } + ) + convert_mock.return_value = [self._data_point] + post_mock.return_value = post_response + result = self._exporter.export( + self._metrics_data, + base_monitoring_data_point=self._data_point + ) + self.assertEqual(result, MetricExportResult.SUCCESS) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping") -# def test_ping(self, ping_mock): -# ping_response = _Response( -# mock.Mock(), -# None, -# { -# "x-ms-qps-subscribed": "false", -# } -# ) -# ping_mock.return_value = ping_response -# response = self._exporter._ping( -# monitoring_data_point=self._data_point -# ) -# self.assertEqual(response, ping_response) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping") + def test_ping(self, ping_mock): + ping_response = _Response( + mock.Mock(), + None, + { + "x-ms-qps-subscribed": "false", + } + ) + ping_mock.return_value = ping_response + response = self._exporter._ping( + monitoring_data_point=self._data_point + ) + self.assertEqual(response, ping_response) -# def test_ping_exception(self): -# with mock.patch( -# "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping", -# throw(HttpResponseError), -# ): # noqa: E501 -# response = self._exporter._ping( -# monitoring_data_point=self._data_point -# ) -# self.assertIsNone(response) + def test_ping_exception(self): + with mock.patch( + "azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.ping", + throw(HttpResponseError), + ): # noqa: E501 + response = self._exporter._ping( + monitoring_data_point=self._data_point + ) + self.assertIsNone(response) -# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") -# def test_quickpulsereader_init(self, task_mock): -# task_inst_mock = mock.Mock() -# task_mock.return_value = task_inst_mock -# reader = _QuickpulseMetricReader( -# self._exporter, -# self._data_point, -# ) -# self.assertEqual(reader._exporter, self._exporter) -# self.assertEqual(reader._quick_pulse_state, _QuickpulseState.PING_SHORT) -# self.assertEqual(reader._base_monitoring_data_point, self._data_point) -# self.assertEqual(reader._elapsed_num_seconds, 0) -# self.assertEqual(reader._elapsed_num_seconds, 0) -# self.assertEqual(reader._worker, task_inst_mock) -# task_mock.assert_called_with( -# PeriodicTask, -# interval=_POST_INTERVAL_SECONDS, -# function=reader._ticker, -# name="QuickpulseMetricReader", -# ) -# self.assertTrue(reader._worker.daemon) -# task_inst_mock.start.assert_called_once() + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_init(self, task_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + self.assertEqual(reader._exporter, self._exporter) + self.assertEqual(reader._quick_pulse_state, _QuickpulseState.PING_SHORT) + self.assertEqual(reader._base_monitoring_data_point, self._data_point) + self.assertEqual(reader._elapsed_num_seconds, 0) + self.assertEqual(reader._elapsed_num_seconds, 0) + self.assertEqual(reader._worker, task_inst_mock) + task_mock.assert_called_with( + PeriodicTask, + interval=_POST_INTERVAL_SECONDS, + function=reader._ticker, + name="QuickpulseMetricReader", + ) + self.assertTrue(reader._worker.daemon) + task_inst_mock.start.assert_called_once() -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") -# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") -# def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): -# task_inst_mock = mock.Mock() -# task_mock.return_value = task_inst_mock -# reader = _QuickpulseMetricReader( -# self._exporter, -# self._data_point, -# ) -# reader._quick_pulse_state = _QuickpulseState.PING_SHORT -# reader._elapsed_num_seconds = _QuickpulseState.PING_SHORT.value -# ping_mock.return_value = _Response( -# None, -# None, -# { -# "x-ms-qps-subscribed": "true" -# } -# ) -# reader._ticker() -# ping_mock.assert_called_once_with( -# self._data_point, -# ) -# self.assertEqual(reader._quick_pulse_state, _QuickpulseState.POST_SHORT) -# self.assertEqual(reader._elapsed_num_seconds, 1) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + reader._quick_pulse_state = _QuickpulseState.PING_SHORT + reader._elapsed_num_seconds = _QuickpulseState.PING_SHORT.value + ping_mock.return_value = _Response( + None, + None, + { + "x-ms-qps-subscribed": "true" + } + ) + reader._ticker() + ping_mock.assert_called_once_with( + self._data_point, + ) + self.assertEqual(reader._quick_pulse_state, _QuickpulseState.POST_SHORT) + self.assertEqual(reader._elapsed_num_seconds, 1) -# # TODO: Other ticker cases + # TODO: Other ticker cases -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") -# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") -# def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): -# task_inst_mock = mock.Mock() -# task_mock.return_value = task_inst_mock -# reader = _QuickpulseMetricReader( -# self._exporter, -# self._data_point, -# ) -# export_mock.return_value = MetricExportResult.SUCCESS -# reader._receive_metrics( -# self._metrics_data, -# 20_000, -# ) -# export_mock.assert_called_once_with( -# self._metrics_data, -# timeout_millis=20_000, -# base_monitoring_data_point=self._data_point, -# documents=[], -# ) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + export_mock.return_value = MetricExportResult.SUCCESS + reader._receive_metrics( + self._metrics_data, + 20_000, + ) + export_mock.assert_called_once_with( + self._metrics_data, + timeout_millis=20_000, + base_monitoring_data_point=self._data_point, + documents=[], + ) -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") -# @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") -# def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): -# task_inst_mock = mock.Mock() -# task_mock.return_value = task_inst_mock -# reader = _QuickpulseMetricReader( -# self._exporter, -# self._data_point, -# ) -# export_mock.return_value = MetricExportResult.FAILURE -# with self.assertRaises(_UnsuccessfulQuickPulsePostError): -# reader._receive_metrics( -# self._metrics_data, -# 20_000, -# ) -# export_mock.assert_called_once_with( -# self._metrics_data, -# timeout_millis=20_000, -# base_monitoring_data_point=self._data_point, -# documents=[], -# ) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") + @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): + task_inst_mock = mock.Mock() + task_mock.return_value = task_inst_mock + reader = _QuickpulseMetricReader( + self._exporter, + self._data_point, + ) + export_mock.return_value = MetricExportResult.FAILURE + with self.assertRaises(_UnsuccessfulQuickPulsePostError): + reader._receive_metrics( + self._metrics_data, + 20_000, + ) + export_mock.assert_called_once_with( + self._metrics_data, + timeout_millis=20_000, + base_monitoring_data_point=self._data_point, + documents=[], + ) From 31b658ce8578c03c4efaa57b59864f521715da28 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Thu, 22 Feb 2024 14:46:22 -0800 Subject: [PATCH 15/21] minversion --- .../setup.py | 2 +- .../tests/quickpulse/test_live_metrics.py | 150 +++++++++--------- 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py index 833da2774115..56f8392cef1c 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py @@ -81,7 +81,7 @@ }, python_requires=">=3.8", install_requires=[ - "azure-core<2.0.0,>=1.23.0", + "azure-core<2.0.0,>=1.28.0", "fixedint==0.1.6", "msrest>=0.6.10", "opentelemetry-api~=1.21", diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py index 4506c9cc936c..9a3a2123cd8b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -1,84 +1,84 @@ -# # Copyright (c) Microsoft Corporation. All rights reserved. -# # Licensed under the MIT License. +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. -# import platform -# import unittest -# from unittest import mock +import platform +import unittest +from unittest import mock -# from opentelemetry.sdk.metrics import MeterProvider -# from opentelemetry.sdk.resources import Resource, ResourceAttributes +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.resources import Resource, ResourceAttributes -# from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys -# from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint -# from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( -# _QuickpulseExporter, -# _QuickpulseMetricReader, -# ) -# from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import ( -# enable_live_metrics, -# _QuickpulseManager, -# ) -# from azure.monitor.opentelemetry.exporter._utils import ( -# _get_sdk_version, -# _populate_part_a_fields, -# ) +from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys +from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint +from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( + _QuickpulseExporter, + _QuickpulseMetricReader, +) +from azure.monitor.opentelemetry.exporter._quickpulse._live_metrics import ( + enable_live_metrics, + _QuickpulseManager, +) +from azure.monitor.opentelemetry.exporter._utils import ( + _get_sdk_version, + _populate_part_a_fields, +) -# class TestLiveMetrics(unittest.TestCase): +class TestLiveMetrics(unittest.TestCase): -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._live_metrics._QuickpulseManager") -# def test_enable_live_metrics(self, manager_mock): -# mock_resource = mock.Mock() -# enable_live_metrics( -# connection_string="test_cs", -# resource=mock_resource, -# ) -# manager_mock.assert_called_with("test_cs", mock_resource) + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._live_metrics._QuickpulseManager") + def test_enable_live_metrics(self, manager_mock): + mock_resource = mock.Mock() + enable_live_metrics( + connection_string="test_cs", + resource=mock_resource, + ) + manager_mock.assert_called_with("test_cs", mock_resource) -# class TestQuickpulseManager(unittest.TestCase): +class TestQuickpulseManager(unittest.TestCase): -# @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") -# @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") -# @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") -# def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): -# point_inst_mock = mock.Mock() -# point_mock.return_value = point_inst_mock -# exporter_inst_mock = mock.Mock() -# exporter_mock.return_value = exporter_inst_mock -# reader_inst_mock = mock.Mock() -# reader_mock.return_value = reader_inst_mock -# provider_inst_mock = mock.Mock() -# provider_mock.return_value = provider_inst_mock -# generator_inst_mock = mock.Mock() -# generator_mock.return_value = generator_inst_mock -# generator_inst_mock.generate_trace_id.return_value = "test_trace_id" -# resource = Resource.create( -# { -# ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", -# ResourceAttributes.SERVICE_NAME: "test_service", -# } -# ) -# part_a_fields = _populate_part_a_fields(resource) -# qpm = _QuickpulseManager( -# connection_string="test_cs", -# resource=resource, -# ) -# self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) -# self.assertEqual(qpm._exporter, exporter_inst_mock) -# self.assertEqual(qpm._reader, reader_inst_mock) -# self.assertEqual(qpm._meter_provider, provider_inst_mock) -# point_mock.assert_called_with( -# MonitoringDataPoint, -# version=_get_sdk_version(), -# invariant_version=1, -# instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), -# role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), -# machine_name=platform.node(), -# stream_id="test_trace_id", -# ) -# exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") -# reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) -# provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) + @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") + @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") + def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): + point_inst_mock = mock.Mock() + point_mock.return_value = point_inst_mock + exporter_inst_mock = mock.Mock() + exporter_mock.return_value = exporter_inst_mock + reader_inst_mock = mock.Mock() + reader_mock.return_value = reader_inst_mock + provider_inst_mock = mock.Mock() + provider_mock.return_value = provider_inst_mock + generator_inst_mock = mock.Mock() + generator_mock.return_value = generator_inst_mock + generator_inst_mock.generate_trace_id.return_value = "test_trace_id" + resource = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", + ResourceAttributes.SERVICE_NAME: "test_service", + } + ) + part_a_fields = _populate_part_a_fields(resource) + qpm = _QuickpulseManager( + connection_string="test_cs", + resource=resource, + ) + self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) + self.assertEqual(qpm._exporter, exporter_inst_mock) + self.assertEqual(qpm._reader, reader_inst_mock) + self.assertEqual(qpm._meter_provider, provider_inst_mock) + point_mock.assert_called_with( + MonitoringDataPoint, + version=_get_sdk_version(), + invariant_version=1, + instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), + role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), + machine_name=platform.node(), + stream_id="test_trace_id", + ) + exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") + reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) + provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) From 356351ba2553bf0bb58813def69cbc034d0899a1 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Fri, 23 Feb 2024 08:01:23 -0800 Subject: [PATCH 16/21] tests --- .../tests/quickpulse/test_exporter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index 8cf22455de06..bce95f73c730 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -207,7 +207,7 @@ def test_ping_exception(self): self.assertIsNone(response) - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") def test_quickpulsereader_init(self, task_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -231,7 +231,7 @@ def test_quickpulsereader_init(self, task_mock): task_inst_mock.start.assert_called_once() @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -258,7 +258,7 @@ def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): # TODO: Other ticker cases @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -279,7 +279,7 @@ def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): ) @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._utils.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock From 187e87e3ff134ef368167e8e22a83d9921d2aa08 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Fri, 23 Feb 2024 10:55:24 -0800 Subject: [PATCH 17/21] tests --- .../tests/quickpulse/test_exporter.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index bce95f73c730..7d514018d0be 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -17,7 +17,6 @@ from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.resources import Resource, ResourceAttributes from azure.core.exceptions import HttpResponseError -from azure.monitor.opentelemetry.exporter._utils import PeriodicTask from azure.monitor.opentelemetry.exporter._quickpulse._generated._client import QuickpulseClient from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( @@ -207,7 +206,7 @@ def test_ping_exception(self): self.assertIsNone(response) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask") def test_quickpulsereader_init(self, task_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -222,7 +221,6 @@ def test_quickpulsereader_init(self, task_mock): self.assertEqual(reader._elapsed_num_seconds, 0) self.assertEqual(reader._worker, task_inst_mock) task_mock.assert_called_with( - PeriodicTask, interval=_POST_INTERVAL_SECONDS, function=reader._ticker, name="QuickpulseMetricReader", @@ -231,7 +229,7 @@ def test_quickpulsereader_init(self, task_mock): task_inst_mock.start.assert_called_once() @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter._ping") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask") def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -258,7 +256,7 @@ def test_quickpulsereader_ticker_ping_true(self, task_mock, ping_mock): # TODO: Other ticker cases @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask") def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock @@ -279,7 +277,7 @@ def test_quickpulsereader_receive_metrics(self, task_mock, export_mock): ) @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.export") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter.PeriodicTask") def test_quickpulsereader_receive_metrics_exception(self, task_mock, export_mock): task_inst_mock = mock.Mock() task_mock.return_value = task_inst_mock From dd303683bcd91ad15dbaf184e21e0fa1485326de Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Fri, 23 Feb 2024 12:14:46 -0800 Subject: [PATCH 18/21] tests --- .../tests/quickpulse/test_live_metrics.py | 62 +++++++++---------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py index 9a3a2123cd8b..439c8cbe176f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -38,23 +38,9 @@ def test_enable_live_metrics(self, manager_mock): class TestQuickpulseManager(unittest.TestCase): - @mock.patch("opentelemetry.sdk.metrics.MeterProvider.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseMetricReader.__new__") - @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._exporter._QuickpulseExporter.__new__") - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated.models.MonitoringDataPoint.__new__") - def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, provider_mock): - point_inst_mock = mock.Mock() - point_mock.return_value = point_inst_mock - exporter_inst_mock = mock.Mock() - exporter_mock.return_value = exporter_inst_mock - reader_inst_mock = mock.Mock() - reader_mock.return_value = reader_inst_mock - provider_inst_mock = mock.Mock() - provider_mock.return_value = provider_inst_mock - generator_inst_mock = mock.Mock() - generator_mock.return_value = generator_inst_mock - generator_inst_mock.generate_trace_id.return_value = "test_trace_id" + @mock.patch("opentelemetry.sdk.trace.id_generator.RandomIdGenerator.generate_trace_id") + def test_init(self, generator_mock): + generator_mock.return_value = "test_trace_id" resource = Resource.create( { ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", @@ -63,22 +49,32 @@ def test_init(self, point_mock, exporter_mock, generator_mock, reader_mock, prov ) part_a_fields = _populate_part_a_fields(resource) qpm = _QuickpulseManager( - connection_string="test_cs", + connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/", resource=resource, ) - self.assertEqual(qpm._base_monitoring_data_point, point_inst_mock) - self.assertEqual(qpm._exporter, exporter_inst_mock) - self.assertEqual(qpm._reader, reader_inst_mock) - self.assertEqual(qpm._meter_provider, provider_inst_mock) - point_mock.assert_called_with( - MonitoringDataPoint, - version=_get_sdk_version(), - invariant_version=1, - instance=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, ""), - role_name=part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, ""), - machine_name=platform.node(), - stream_id="test_trace_id", + self.assertTrue(isinstance(qpm._exporter, _QuickpulseExporter)) + self.assertEqual( + qpm._exporter._live_endpoint, + "https://eastus.livediagnostics.monitor.azure.com/", + ) + self.assertEqual( + qpm._exporter._instrumentation_key, + "4321abcd-5678-4efa-8abc-1234567890ac", + ) + self.assertEqual(qpm._base_monitoring_data_point.version, _get_sdk_version()) + self.assertEqual(qpm._base_monitoring_data_point.invariant_version, 1) + self.assertEqual( + qpm._base_monitoring_data_point.instance, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, "") + ) + self.assertEqual( + qpm._base_monitoring_data_point.role_name, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, "") ) - exporter_mock.assert_called_with(_QuickpulseExporter, "test_cs") - reader_mock.assert_called_with(_QuickpulseMetricReader, exporter_inst_mock, point_inst_mock) - provider_mock.assert_called_with(MeterProvider, [reader_inst_mock]) + self.assertEqual(qpm._base_monitoring_data_point.machine_name, platform.node()) + self.assertEqual(qpm._base_monitoring_data_point.stream_id, "test_trace_id") + self.assertTrue(isinstance(qpm._reader, _QuickpulseMetricReader)) + self.assertEqual(qpm._reader._exporter, qpm._exporter) + self.assertEqual(qpm._reader._base_monitoring_data_point, qpm._base_monitoring_data_point) + self.assertTrue(isinstance(qpm._meter_provider, MeterProvider)) + self.assertEqual(qpm._meter_provider._sdk_config.metric_readers, [qpm._reader]) From 376679f4c6ab4675b4589062e69c4c60a02cec41 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 26 Feb 2024 09:36:01 -0800 Subject: [PATCH 19/21] test --- .../tests/quickpulse/test_exporter.py | 12 +++--------- .../tests/quickpulse/test_live_metrics.py | 1 - 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py index 7d514018d0be..d053b3da4e59 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_exporter.py @@ -91,21 +91,15 @@ def setUpClass(cls): ) - @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient.__new__") + @mock.patch("azure.monitor.opentelemetry.exporter._quickpulse._generated._client.QuickpulseClient") def test_init(self, client_mock): - client_inst_mock = mock.Mock() - client_mock.return_value = client_inst_mock exporter = _QuickpulseExporter( connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/" ) - self.assertEqual(exporter._live_endpoint, "https://eastus.livediagnostics.monitor.azure.com/") self.assertEqual(exporter._instrumentation_key, "4321abcd-5678-4efa-8abc-1234567890ab") - self.assertEqual(exporter._client, client_inst_mock) - client_mock.assert_called_with( - QuickpulseClient, - host="https://eastus.livediagnostics.monitor.azure.com/" - ) + self.assertTrue(isinstance(exporter._client, QuickpulseClient)) + self.assertEqual(exporter._client._config.host, "https://eastus.livediagnostics.monitor.azure.com/") def test_export_missing_data_point(self): diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py index 439c8cbe176f..a53b532626a4 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -9,7 +9,6 @@ from opentelemetry.sdk.resources import Resource, ResourceAttributes from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys -from azure.monitor.opentelemetry.exporter._quickpulse._generated.models import MonitoringDataPoint from azure.monitor.opentelemetry.exporter._quickpulse._exporter import ( _QuickpulseExporter, _QuickpulseMetricReader, From dc66875db7980787240f3bbe571b4aa081cf991b Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 26 Feb 2024 10:29:46 -0800 Subject: [PATCH 20/21] tests --- .../samples/logs/test.py | 84 +++++++++++++++++++ .../tests/quickpulse/test_live_metrics.py | 41 +++++++++ .../tests/test_utils.py | 17 ++++ 3 files changed, 142 insertions(+) create mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py new file mode 100644 index 000000000000..b3e6290fcadc --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py @@ -0,0 +1,84 @@ +import os +import logging + +from opentelemetry._logs import ( + get_logger_provider, + set_logger_provider, +) +from opentelemetry.sdk._logs import ( + LoggerProvider, + LoggingHandler, +) +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + +from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +class CustomFilter(logging.Filter): + + def filter(self, record): + if hasattr(record, "ignore"): + return False + return True + + +class LogHandler: + def __init__( + self, + app_name: str, + app_version: str, + **kwargs, + ): + """_summary_ + + Args: + app_name: app name + app_version: app version + kwargs Additional logging parameters + """ + self.app_name = app_name + self.app_version = app_version + self.kwargs = kwargs + + # Create the logging provider + self.logger_provider = LoggerProvider() + set_logger_provider(self.logger_provider) + exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"], + ) + self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + + # Create the logging handler + self.handler = LoggingHandler() + + # Attach LoggingHandler to root logger + self._logger = logging.getLogger(self.app_name) + self._logger.addHandler(self.handler) + self._logger.addFilter(CustomFilter()) + self._logger.setLevel(logging.INFO) + +log_handler = LogHandler(app_name='app-name', app_version=1.0) +logger = log_handler._logger + + +exporter = AzureMonitorTraceExporter.from_connection_string( + os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +) + +tracer_provider = TracerProvider() +trace.set_tracer_provider(tracer_provider) +tracer = trace.get_tracer(__name__) +span_processor = BatchSpanProcessor(exporter, schedule_delay_millis=60000) +trace.get_tracer_provider().add_span_processor(span_processor) + +with tracer.start_as_current_span(name="my-application") as application_span: + logger.info("test logs message") + logger.info("test ignore message", extra={"ignore": "true"}) + logger.exception("test exception message") + +input() \ No newline at end of file diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py index a53b532626a4..0b0f512865bf 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/quickpulse/test_live_metrics.py @@ -77,3 +77,44 @@ def test_init(self, generator_mock): self.assertEqual(qpm._reader._base_monitoring_data_point, qpm._base_monitoring_data_point) self.assertTrue(isinstance(qpm._meter_provider, MeterProvider)) self.assertEqual(qpm._meter_provider._sdk_config.metric_readers, [qpm._reader]) + + + def test_singleton(self): + resource = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance", + ResourceAttributes.SERVICE_NAME: "test_service", + } + ) + part_a_fields = _populate_part_a_fields(resource) + resource2 = Resource.create( + { + ResourceAttributes.SERVICE_INSTANCE_ID: "test_instance2", + ResourceAttributes.SERVICE_NAME: "test_service2", + } + ) + qpm = _QuickpulseManager( + connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/", + resource=resource, + ) + qpm2 = _QuickpulseManager( + connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ac;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/", + resource=resource2, + ) + self.assertEqual(qpm, qpm2) + self.assertEqual( + qpm._base_monitoring_data_point.instance, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, "") + ) + self.assertEqual( + qpm._base_monitoring_data_point.role_name, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, "") + ) + self.assertEqual( + qpm2._base_monitoring_data_point.instance, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE_INSTANCE, "") + ) + self.assertEqual( + qpm2._base_monitoring_data_point.role_name, + part_a_fields.get(ContextTagKeys.AI_CLOUD_ROLE, "") + ) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py index f0b630453306..fc08963ffbec 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import datetime import os import platform +import time import unittest from azure.monitor.opentelemetry.exporter import _utils @@ -44,6 +46,21 @@ def test_nanoseconds_to_duration(self): self.assertEqual(ns_to_duration(3600 * 1000000000), "0.01:00:00.000") self.assertEqual(ns_to_duration(86400 * 1000000000), "1.00:00:00.000") + + @patch("time.time") + def test_ticks_since_dot_net_epoch(self, time_mock): + current_time = time.time() + shift_time = int( + ( + datetime.datetime(1970, 1, 1, 0, 0, 0) - + datetime.datetime(1, 1, 1, 0, 0, 0)).total_seconds() + ) * (10 ** 7) + time_mock.return_value = current_time + ticks = _utils._ticks_since_dot_net_epoch() + expected_ticks = int(current_time * (10**7)) + shift_time + self.assertEqual(ticks, expected_ticks) + + def test_populate_part_a_fields(self): resource = Resource( {"service.name": "testServiceName", From 5a965cee9c8c98968b73d7d874d66bc21417af53 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 26 Feb 2024 10:30:56 -0800 Subject: [PATCH 21/21] remove --- .../samples/logs/test.py | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py deleted file mode 100644 index b3e6290fcadc..000000000000 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/test.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -import logging - -from opentelemetry._logs import ( - get_logger_provider, - set_logger_provider, -) -from opentelemetry.sdk._logs import ( - LoggerProvider, - LoggingHandler, -) -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - -from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter - -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor - -from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter - -class CustomFilter(logging.Filter): - - def filter(self, record): - if hasattr(record, "ignore"): - return False - return True - - -class LogHandler: - def __init__( - self, - app_name: str, - app_version: str, - **kwargs, - ): - """_summary_ - - Args: - app_name: app name - app_version: app version - kwargs Additional logging parameters - """ - self.app_name = app_name - self.app_version = app_version - self.kwargs = kwargs - - # Create the logging provider - self.logger_provider = LoggerProvider() - set_logger_provider(self.logger_provider) - exporter = AzureMonitorLogExporter( - connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"], - ) - self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) - - # Create the logging handler - self.handler = LoggingHandler() - - # Attach LoggingHandler to root logger - self._logger = logging.getLogger(self.app_name) - self._logger.addHandler(self.handler) - self._logger.addFilter(CustomFilter()) - self._logger.setLevel(logging.INFO) - -log_handler = LogHandler(app_name='app-name', app_version=1.0) -logger = log_handler._logger - - -exporter = AzureMonitorTraceExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] -) - -tracer_provider = TracerProvider() -trace.set_tracer_provider(tracer_provider) -tracer = trace.get_tracer(__name__) -span_processor = BatchSpanProcessor(exporter, schedule_delay_millis=60000) -trace.get_tracer_provider().add_span_processor(span_processor) - -with tracer.start_as_current_span(name="my-application") as application_span: - logger.info("test logs message") - logger.info("test ignore message", extra={"ignore": "true"}) - logger.exception("test exception message") - -input() \ No newline at end of file