From c33d42e2d0baf8040399c1f73db97fc56b11cafe Mon Sep 17 00:00:00 2001 From: bambriz Date: Thu, 28 Jul 2022 13:53:57 -0700 Subject: [PATCH 1/9] Create cosmos_diagnostics.py A class that gathers useful diagnostics information. --- .../azure/cosmos/cosmos_diagnostics.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py new file mode 100644 index 000000000000..2f30ee19629c --- /dev/null +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py @@ -0,0 +1,116 @@ +"""Diagnostic tools for Azure Cosmos database service operations. +""" + +from requests.structures import CaseInsensitiveDict + + +class CosmosDiagnostics(object): + """Record Response headers and other useful information from Cosmos read operations. + + The full response headers are stored in the ``headers`` property. + + Calling the class as a function will return a dictionary of all Diagnostics properties. + + Examples: + + >>> col = b.create_container( + ... id="some_container", + ... partition_key=PartitionKey(path='/id', kind='Hash'), + >>> cd = col.diagnostics() #col.diagnostics would return the object itself which can still be used to get the properties. + >>> cd['headers']['x-ms-activity-id'] + '6243eeed-f06a-413d-b913-dcf8122d0642' + >>>cd + {headers: {x-ms-activity-id: '6243eeed-f06a-413d-b913-dcf8122d0642', + ...: ... , + ...: ...}, + body: { ...: ..., + ...: ..., + ...: ...}, + request_charge: #, + error_code: #, + error_message: ... + ...: ...} + + New Properties can be added by adding a getter and setter for them. + + Example: + + @property + def new_property(self): + return self._new_property + + @new_property.setter + def new_property(self, value: type): + self._new_property = value + + """ + + _common = { + "x-ms-activity-id", + "x-ms-session-token", + "x-ms-item-count", + "x-ms-request-quota", + "x-ms-resource-usage", + "x-ms-retry-after-ms", + } + + def __init__(self): + self._headers = CaseInsensitiveDict() + self._body = None + self._request_charge = 0 + self._error_code = 0 + self._error_message = "" + + @property + def headers(self): + return CaseInsensitiveDict(self._headers) + + @headers.setter + def header(self, value: dict): + self._headers = value + self._request_charge += float(value.get("x-ms-request-charge", 0)) + + @property + def body(self): + return self._body + + @body.setter + def body(self, value: dict): + self._body = value + + @property + def error_code(self): + return self._error_code + + @error_code.setter + def error_code(self, value: int): + self._error_code = value + + @property + def error_message(self): + return self._error_message + + @error_message.setter + def error_message(self, value: str): + self._error_message = value + + @property + def request_charge(self): + return self._request_charge + + def clear(self): + self._request_charge = 0 + + def __call__(self): + #This will format all the properties into a dictionary + return { + key: value + for key, value in self.__dict__.items() + if isinstance(value, property) + } + + def __getattr__(self, name): + key = "x-ms-" + name.replace("_", "-") + if key in self._common: + return self._headers[key] + raise AttributeError(name) From 3dbbd10e615fd8365a97ea21d3bc77843aa08cf7 Mon Sep 17 00:00:00 2001 From: bambriz Date: Thu, 11 Aug 2022 14:24:54 -0700 Subject: [PATCH 2/9] Added system info to diagnostics Added system info to diagnostics. This also updates things related to getting exception data for diagnostics --- .../azure/cosmos/_cosmos_client_connection.py | 2 + .../azure/cosmos/_retry_utility.py | 7 +- .../azure/cosmos/_synchronized_request.py | 2 +- .../azure-cosmos/azure/cosmos/container.py | 32 ++++++++ .../azure/cosmos/cosmos_client.py | 13 ++++ .../azure/cosmos/cosmos_diagnostics.py | 75 ++++++++++++++----- .../azure-cosmos/azure/cosmos/database.py | 23 ++++++ sdk/cosmos/azure-cosmos/azure/cosmos/user.py | 18 +++++ 8 files changed, 153 insertions(+), 19 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index ffafc2e397a4..af29ff96d0d2 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -144,6 +144,8 @@ def __init__( # Keeps the latest response headers from the server. self.last_response_headers = None + # Keep Track of Last Exceptions + self.last_exceptions = None self._useMultipleWriteLocations = False self._global_endpoint_manager = global_endpoint_manager._GlobalEndpointManager(self) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py index 601bdeb72ce6..87b667ab3c07 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py @@ -69,6 +69,9 @@ def Execute(client, global_endpoint_manager, function, *args, **kwargs): ) partition_key_range_gone_retry_policy = _gone_retry_policy.PartitionKeyRangeGoneRetryPolicy(client, *args) + #on a new execution, erase last exception from client, currently always ovverides even if there is exception + # client.last_exceptions = None + while True: try: client_timeout = kwargs.get('timeout') @@ -87,10 +90,12 @@ def Execute(client, global_endpoint_manager, function, *args, **kwargs): client.last_response_headers[ HttpHeaders.ThrottleRetryWaitTimeInMs ] = resourceThrottle_retry_policy.cummulative_wait_time_in_milliseconds - + client.last_exceptions = None return result except exceptions.CosmosHttpResponseError as e: retry_policy = None + #Current way of passing status code and message up, only works for exceptions though + client.last_exceptions = e if e.status_code == StatusCodes.FORBIDDEN and e.sub_status == SubStatusCodes.WRITE_FORBIDDEN: retry_policy = endpointDiscovery_retry_policy elif e.status_code == StatusCodes.TOO_MANY_REQUESTS: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py index 6df67d96cdc1..59eb42db7ab1 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py @@ -136,7 +136,7 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin connection_verify=kwargs.pop("connection_verify", is_ssl_enabled), **kwargs ) - + #Find a way to leverage this information response = response.http_response headers = dict(response.headers) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py index 6be5d17c8561..f0c4f51d2826 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py @@ -35,6 +35,7 @@ from .offer import ThroughputProperties from .scripts import ScriptsProxy from .partition_key import NonePartitionKeyValue +from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("ContainerProxy",) @@ -65,6 +66,7 @@ def __init__(self, client_connection, database_link, id, properties=None): # py self.container_link = u"{}/colls/{}".format(database_link, self.id) self._is_system_key = None self._scripts = None # type: Optional[ScriptsProxy] + self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -165,6 +167,8 @@ def read( if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) + return cast('Dict[str, Any]', self._properties) @distributed_trace @@ -226,6 +230,8 @@ def read_item( result = self.client_connection.ReadItem(document_link=doc_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -273,6 +279,8 @@ def read_all_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, + self.client_connection.last_exceptions) return items @distributed_trace @@ -320,6 +328,8 @@ def query_items_change_feed( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -413,6 +423,8 @@ def query_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, + self.client_connection.last_exceptions) return items @distributed_trace @@ -465,6 +477,8 @@ def replace_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -517,6 +531,8 @@ def upsert_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -572,6 +588,8 @@ def create_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -622,6 +640,8 @@ def delete_item( result = self.client_connection.DeleteItem(document_link=document_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) @distributed_trace def read_offer(self, **kwargs): @@ -667,6 +687,8 @@ def get_throughput(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, throughput_properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, + self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=throughput_properties[0]["content"]["offerThroughput"], properties=throughput_properties[0]) @@ -704,6 +726,8 @@ def replace_throughput(self, throughput, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, data) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, + self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) @@ -727,6 +751,8 @@ def list_conflicts(self, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -770,6 +796,8 @@ def query_conflicts( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -794,6 +822,8 @@ def get_conflict(self, conflict, partition_key, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -820,3 +850,5 @@ def delete_conflict(self, conflict, partition_key, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py index 8110afbc787d..665832c5ed29 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py @@ -32,6 +32,7 @@ from .database import DatabaseProxy from .documents import ConnectionPolicy, DatabaseAccount from .exceptions import CosmosResourceNotFoundError +from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("CosmosClient",) @@ -178,6 +179,7 @@ def __init__(self, url, credential, consistency_level=None, **kwargs): self.client_connection = CosmosClientConnection( url, auth=auth, consistency_level=consistency_level, connection_policy=connection_policy, **kwargs ) + self.diagnostics = CosmosDiagnostics() def __repr__(self): # pylint:disable=client-method-name-no-double-underscore # type () -> str @@ -274,6 +276,8 @@ def create_database( # pylint: disable=redefined-builtin result = self.client_connection.CreateDatabase(database=dict(id=id), options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return DatabaseProxy(self.client_connection, id=result["id"], properties=result) @distributed_trace @@ -373,6 +377,8 @@ def list_databases( result = self.client_connection.ReadDatabases(options=feed_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -424,6 +430,8 @@ def query_databases( result = self.client_connection.ReadDatabases(options=feed_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @distributed_trace @@ -461,6 +469,9 @@ def delete_database( self.client_connection.DeleteDatabase(database_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, None, + self.client_connection.last_exceptions) + @distributed_trace def get_database_account(self, **kwargs): @@ -475,4 +486,6 @@ def get_database_account(self, **kwargs): result = self.client_connection.GetDatabaseAccount(**kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py index 2f30ee19629c..17edbede784e 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py @@ -2,6 +2,9 @@ """ from requests.structures import CaseInsensitiveDict +import platform +import os +from . import exceptions class CosmosDiagnostics(object): @@ -58,15 +61,16 @@ def __init__(self): self._headers = CaseInsensitiveDict() self._body = None self._request_charge = 0 - self._error_code = 0 - self._error_message = "" + self._status_code = 200 + self._status_message = "Standard response for successful HTTP requests" + self._system_information = self.get_system_info() @property def headers(self): return CaseInsensitiveDict(self._headers) @headers.setter - def header(self, value: dict): + def headers(self, value: dict): self._headers = value self._request_charge += float(value.get("x-ms-request-charge", 0)) @@ -79,20 +83,20 @@ def body(self, value: dict): self._body = value @property - def error_code(self): - return self._error_code + def status_code(self): + return self._status_code - @error_code.setter - def error_code(self, value: int): - self._error_code = value + @status_code.setter + def status_code(self, value: int): + self._status_code = value @property - def error_message(self): - return self._error_message + def status_message(self): + return self._status_message - @error_message.setter - def error_message(self, value: str): - self._error_message = value + @status_message.setter + def status_message(self, value: str): + self._status_message = value @property def request_charge(self): @@ -101,16 +105,53 @@ def request_charge(self): def clear(self): self._request_charge = 0 + def update_header_and_body(self, header, body): + self.header(header) + self.body(body) + + def update_diagnostics(self, header: dict, body: dict, e: exceptions.CosmosHttpResponseError, **kwargs): + self.headers = header + self.body = body + #Note: Plan is to use kwargs to be able to easily modify this function to update with needed information + #notes figure out how to just get status information instead of just relying on exceptions being passed + if e: + self.status_code = e.status_code + self.status_message = e.message + else: + #cureent hacky way to get succesful status code + self.status_code = 200 + self.status_message = "Standard response for successful HTTP requests" + + #Note: Planning to add a flag to print it in a pretty format instead of just returning a dictionary def __call__(self): #This will format all the properties into a dictionary return { - key: value - for key, value in self.__dict__.items() - if isinstance(value, property) + key: getattr(self, key) + for key in vars(self) } + # return { + # key: value + # for key, value in self.__dict__.items() + # if isinstance(value, property) + # } def __getattr__(self, name): - key = "x-ms-" + name.replace("_", "-") + temp_name = name + key = "x-ms-" + temp_name.replace("_", "-") if key in self._common: return self._headers[key] + else: + return getattr(self, name) raise AttributeError(name) + + #Current System info I was able to get + def get_system_info(self): + ret = {} + ret["system"] = platform.system() + ret["python version"] = platform.python_version() + ret["architecture"] = platform.architecture() + ret["cpu"] = platform.processor() + ret["cpu count"] = os.cpu_count() + ret["machine"] = platform.machine() + + return ret diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 217fcc65955e..569535b960dc 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -35,6 +35,7 @@ from .exceptions import CosmosResourceNotFoundError from .user import UserProxy from .documents import IndexingMode +from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("DatabaseProxy",) @@ -78,6 +79,7 @@ def __init__(self, client_connection, id, properties=None): # pylint: disable=r self.id = id self.database_link = u"dbs/{}".format(self.id) self._properties = properties + self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -144,6 +146,7 @@ def read(self, populate_query_metrics=None, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) return cast('Dict[str, Any]', self._properties) @@ -241,6 +244,7 @@ def create_container( if response_hook: response_hook(self.client_connection.last_response_headers, data) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, self.client_connection.last_exceptions) return ContainerProxy(self.client_connection, self.database_link, data["id"], properties=data) @@ -342,6 +346,7 @@ def delete_container( result = self.client_connection.DeleteContainer(collection_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) def get_container_client(self, container): # type: (Union[str, ContainerProxy, Dict[str, Any]]) -> ContainerProxy @@ -410,6 +415,7 @@ def list_containers(self, max_item_count=None, populate_query_metrics=None, **kw ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result @distributed_trace @@ -452,6 +458,8 @@ def query_containers( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + return result @distributed_trace @@ -530,6 +538,8 @@ def replace_container( if response_hook: response_hook(self.client_connection.last_response_headers, container_properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, container_properties, self.client_connection.last_exceptions) + return ContainerProxy( self.client_connection, self.database_link, container_properties["id"], properties=container_properties ) @@ -554,6 +564,7 @@ def list_users(self, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result @distributed_trace @@ -581,6 +592,7 @@ def query_users(self, query, parameters=None, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result def get_user_client(self, user): @@ -636,6 +648,8 @@ def create_user(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, user) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) + return UserProxy( client_connection=self.client_connection, id=user["id"], database_link=self.database_link, properties=user ) @@ -664,6 +678,8 @@ def upsert_user(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, user) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) + return UserProxy( client_connection=self.client_connection, id=user["id"], database_link=self.database_link, properties=user ) @@ -697,6 +713,8 @@ def replace_user( if response_hook: response_hook(self.client_connection.last_response_headers, replaced_user) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, replaced_user, self.client_connection.last_exceptions) + return UserProxy( client_connection=self.client_connection, id=replaced_user["id"], @@ -725,6 +743,8 @@ def delete_user(self, user, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + @distributed_trace def read_offer(self, **kwargs): # type: (Any) -> ThroughputProperties @@ -769,6 +789,8 @@ def get_throughput(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, throughput_properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, self.client_connection.last_exceptions) + return ThroughputProperties(offer_throughput=throughput_properties[0]["content"]["offerThroughput"], properties=throughput_properties[0]) @@ -805,4 +827,5 @@ def replace_throughput(self, throughput, **kwargs): offer=throughput_properties[0], **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, data) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py index 31b0d2a1e921..e5c5b1c5c3de 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py @@ -31,6 +31,7 @@ from ._cosmos_client_connection import CosmosClientConnection from ._base import build_options from .permission import Permission +from .cosmos_diagnostics import CosmosDiagnostics class UserProxy(object): @@ -46,6 +47,7 @@ def __init__(self, client_connection, id, database_link, properties=None): # py self.id = id self.user_link = u"{}/users/{}".format(database_link, id) self._properties = properties + self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -84,6 +86,8 @@ def read(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, + self.client_connection.last_exceptions) return cast('Dict[str, Any]', self._properties) @@ -106,6 +110,8 @@ def list_permissions(self, max_item_count=None, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @@ -141,6 +147,8 @@ def query_permissions( if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) return result @@ -165,6 +173,8 @@ def get_permission(self, permission, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission_resp) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission_resp, + self.client_connection.last_exceptions) return Permission( id=permission_resp["id"], @@ -196,6 +206,8 @@ def create_permission(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission, + self.client_connection.last_exceptions) return Permission( id=permission["id"], @@ -228,6 +240,8 @@ def upsert_permission(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission, + self.client_connection.last_exceptions) return Permission( id=permission["id"], @@ -262,6 +276,8 @@ def replace_permission(self, permission, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission_resp) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission_resp, + self.client_connection.last_exceptions) return Permission( id=permission_resp["id"], @@ -294,3 +310,5 @@ def delete_permission(self, permission, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) + self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + self.client_connection.last_exceptions) From a31ae83e48850d284abbd2105ba3f4b59d4beefd Mon Sep 17 00:00:00 2001 From: bambriz Date: Fri, 12 Aug 2022 11:17:11 -0700 Subject: [PATCH 3/9] centralized diagnostics to client connection Centralized diagnostics to client connection and we are now getting diagnostics right after we get a response --- .../azure/cosmos/_cosmos_client_connection.py | 3 +- .../azure/cosmos/_synchronized_request.py | 4 +- .../azure-cosmos/azure/cosmos/container.py | 60 ++++++++-------- .../azure/cosmos/cosmos_client.py | 26 +++---- .../azure/cosmos/cosmos_diagnostics.py | 68 ++++++++++++++++--- .../azure-cosmos/azure/cosmos/database.py | 30 ++++---- 6 files changed, 123 insertions(+), 68 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index af29ff96d0d2..c8df73572989 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -59,6 +59,7 @@ from . import _utils from .partition_key import _Undefined, _Empty from ._auth_policy import CosmosBearerTokenCredentialPolicy +from .cosmos_diagnostics import CosmosDiagnostics ClassType = TypeVar("ClassType") # pylint: disable=protected-access @@ -141,7 +142,7 @@ def __init__( # We need to set continuation as not expected. http_constants.HttpHeaders.IsContinuationExpected: False, } - + self.diagnostics = CosmosDiagnostics() # Keeps the latest response headers from the server. self.last_response_headers = None # Keep Track of Last Exceptions diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py index 59eb42db7ab1..c18360fd0ee9 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py @@ -137,13 +137,15 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin **kwargs ) #Find a way to leverage this information + ir = response.http_response.internal_response response = response.http_response headers = dict(response.headers) + data = response.body() if data: data = data.decode("utf-8") - + global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py index f0c4f51d2826..b057052fa9d8 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py @@ -66,7 +66,7 @@ def __init__(self, client_connection, database_link, id, properties=None): # py self.container_link = u"{}/colls/{}".format(database_link, self.id) self._is_system_key = None self._scripts = None # type: Optional[ScriptsProxy] - self.diagnostics = CosmosDiagnostics() + # self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -167,7 +167,7 @@ def read( if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) return cast('Dict[str, Any]', self._properties) @@ -230,8 +230,8 @@ def read_item( result = self.client_connection.ReadItem(document_link=doc_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -279,8 +279,8 @@ def read_all_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, + # self.client_connection.last_exceptions) return items @distributed_trace @@ -328,8 +328,8 @@ def query_items_change_feed( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -423,8 +423,8 @@ def query_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, + # self.client_connection.last_exceptions) return items @distributed_trace @@ -477,8 +477,8 @@ def replace_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -531,8 +531,8 @@ def upsert_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -588,8 +588,8 @@ def create_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -640,8 +640,8 @@ def delete_item( result = self.client_connection.DeleteItem(document_link=document_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) @distributed_trace def read_offer(self, **kwargs): @@ -687,8 +687,8 @@ def get_throughput(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, throughput_properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, + # self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=throughput_properties[0]["content"]["offerThroughput"], properties=throughput_properties[0]) @@ -726,8 +726,8 @@ def replace_throughput(self, throughput, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, data) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, + # self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) @@ -751,8 +751,8 @@ def list_conflicts(self, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -796,8 +796,8 @@ def query_conflicts( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -822,8 +822,8 @@ def get_conflict(self, conflict, partition_key, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -850,5 +850,5 @@ def delete_conflict(self, conflict, partition_key, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py index 665832c5ed29..0ad3c2903627 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py @@ -32,7 +32,6 @@ from .database import DatabaseProxy from .documents import ConnectionPolicy, DatabaseAccount from .exceptions import CosmosResourceNotFoundError -from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("CosmosClient",) @@ -179,7 +178,7 @@ def __init__(self, url, credential, consistency_level=None, **kwargs): self.client_connection = CosmosClientConnection( url, auth=auth, consistency_level=consistency_level, connection_policy=connection_policy, **kwargs ) - self.diagnostics = CosmosDiagnostics() + # self.diagnostics = CosmosDiagnostics() def __repr__(self): # pylint:disable=client-method-name-no-double-underscore # type () -> str @@ -276,8 +275,8 @@ def create_database( # pylint: disable=redefined-builtin result = self.client_connection.CreateDatabase(database=dict(id=id), options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return DatabaseProxy(self.client_connection, id=result["id"], properties=result) @distributed_trace @@ -377,8 +376,8 @@ def list_databases( result = self.client_connection.ReadDatabases(options=feed_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -430,8 +429,8 @@ def query_databases( result = self.client_connection.ReadDatabases(options=feed_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result @distributed_trace @@ -469,8 +468,8 @@ def delete_database( self.client_connection.DeleteDatabase(database_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, None, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, None, + # self.client_connection.last_exceptions) @distributed_trace @@ -486,6 +485,9 @@ def get_database_account(self, **kwargs): result = self.client_connection.GetDatabaseAccount(**kwargs) if response_hook: response_hook(self.client_connection.last_response_headers) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, + # self.client_connection.last_exceptions) return result + + def diagnostics(self, p=False): + return self.client_connection.diagnostics(p=p) \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py index 17edbede784e..157fcd588c60 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py @@ -61,8 +61,12 @@ def __init__(self): self._headers = CaseInsensitiveDict() self._body = None self._request_charge = 0 - self._status_code = 200 - self._status_message = "Standard response for successful HTTP requests" + self._status_code = 0 + self._status_reason = "" + self._substatus_code = 0 + self._status_message = "" + self._elapsed_time = None + self._system_information = self.get_system_info() @property @@ -90,6 +94,27 @@ def status_code(self): def status_code(self, value: int): self._status_code = value + @property + def substatus_code(self): + return self._substatus_code + @substatus_code.setter + def substatus_code(self, value: int): + self._substatus_code = value + + @property + def status_reason(self): + return self._status_reason + @status_reason.setter + def status_reason(self, value: str): + self._status_reason = value + + @property + def elapsed_time(self): + return self._elapsed_time + @elapsed_time.setter + def elapsed_time(self, value): + self._elapsed_time = value + @property def status_message(self): return self._status_message @@ -109,26 +134,51 @@ def update_header_and_body(self, header, body): self.header(header) self.body(body) - def update_diagnostics(self, header: dict, body: dict, e: exceptions.CosmosHttpResponseError, **kwargs): + def update_diagnostics(self, header: dict, body: dict, **kwargs): self.headers = header self.body = body #Note: Plan is to use kwargs to be able to easily modify this function to update with needed information #notes figure out how to just get status information instead of just relying on exceptions being passed + eTime = kwargs.get("elapsed_time") + if eTime: + self.elapsed_time = eTime + e = kwargs.get("exception") + sc = kwargs.get("status_code") + if sc: + self.status_code = sc + sr = kwargs.get("status_reason") + if sr: + self.status_reason = sr + sm = kwargs.get("response_text") + if sm: + self.status_message = sm + ssc = kwargs.get("substatus_code") + if ssc: + self.substatus_code = ssc + else: + self.substatus_code = 0 if e: self.status_code = e.status_code self.status_message = e.message - else: - #cureent hacky way to get succesful status code - self.status_code = 200 - self.status_message = "Standard response for successful HTTP requests" #Note: Planning to add a flag to print it in a pretty format instead of just returning a dictionary - def __call__(self): + def __call__(self, p=False): #This will format all the properties into a dictionary - return { + ret = { key: getattr(self, key) for key in vars(self) } + if p: + print("DIAGNOSTICS") + for key, value in ret.items(): + if type(value) is dict: + print(str(key)+":") + for k, v in value.items(): + print(" " + str(k) + ": " + str(v)) + else: + print(str(key)+": "+str(value)) + else: + return ret # return { # key: value # for key, value in self.__dict__.items() diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 569535b960dc..43878309a78f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -79,7 +79,7 @@ def __init__(self, client_connection, id, properties=None): # pylint: disable=r self.id = id self.database_link = u"dbs/{}".format(self.id) self._properties = properties - self.diagnostics = CosmosDiagnostics() + # self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -146,7 +146,7 @@ def read(self, populate_query_metrics=None, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) return cast('Dict[str, Any]', self._properties) @@ -244,7 +244,7 @@ def create_container( if response_hook: response_hook(self.client_connection.last_response_headers, data) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, self.client_connection.last_exceptions) return ContainerProxy(self.client_connection, self.database_link, data["id"], properties=data) @@ -346,7 +346,7 @@ def delete_container( result = self.client_connection.DeleteContainer(collection_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) def get_container_client(self, container): # type: (Union[str, ContainerProxy, Dict[str, Any]]) -> ContainerProxy @@ -415,7 +415,7 @@ def list_containers(self, max_item_count=None, populate_query_metrics=None, **kw ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result @distributed_trace @@ -458,7 +458,7 @@ def query_containers( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result @@ -538,7 +538,7 @@ def replace_container( if response_hook: response_hook(self.client_connection.last_response_headers, container_properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, container_properties, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, container_properties, self.client_connection.last_exceptions) return ContainerProxy( self.client_connection, self.database_link, container_properties["id"], properties=container_properties @@ -564,7 +564,7 @@ def list_users(self, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result @distributed_trace @@ -592,7 +592,7 @@ def query_users(self, query, parameters=None, max_item_count=None, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) return result def get_user_client(self, user): @@ -648,7 +648,7 @@ def create_user(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, user) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) return UserProxy( client_connection=self.client_connection, id=user["id"], database_link=self.database_link, properties=user @@ -678,7 +678,7 @@ def upsert_user(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, user) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, user, self.client_connection.last_exceptions) return UserProxy( client_connection=self.client_connection, id=user["id"], database_link=self.database_link, properties=user @@ -713,7 +713,7 @@ def replace_user( if response_hook: response_hook(self.client_connection.last_response_headers, replaced_user) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, replaced_user, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, replaced_user, self.client_connection.last_exceptions) return UserProxy( client_connection=self.client_connection, @@ -743,7 +743,7 @@ def delete_user(self, user, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, self.client_connection.last_exceptions) @distributed_trace def read_offer(self, **kwargs): @@ -789,7 +789,7 @@ def get_throughput(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, throughput_properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, self.client_connection.last_exceptions) + # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=throughput_properties[0]["content"]["offerThroughput"], properties=throughput_properties[0]) @@ -827,5 +827,5 @@ def replace_throughput(self, throughput, **kwargs): offer=throughput_properties[0], **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, data) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, data, self.client_connection.last_exceptions) + return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) From bf2a416f817c5cdeacd9157caac81ab6313ab9af Mon Sep 17 00:00:00 2001 From: bambriz Date: Mon, 15 Aug 2022 12:10:10 -0700 Subject: [PATCH 4/9] Added Centralized diagnostics to async Applied diagnostics to async code. Diagnostics should work for async as it does for sync request. --- .../azure/cosmos/_cosmos_client_connection.py | 7 ++- .../azure/cosmos/_synchronized_request.py | 2 +- .../azure/cosmos/aio/_asynchronous_request.py | 4 +- .../azure/cosmos/aio/_cosmos_client.py | 3 + .../aio/_cosmos_client_connection_async.py | 5 +- .../azure/cosmos/cosmos_diagnostics.py | 58 ++++++++++++++----- 6 files changed, 59 insertions(+), 20 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index c8df73572989..d8f252b22bf9 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -142,7 +142,8 @@ def __init__( # We need to set continuation as not expected. http_constants.HttpHeaders.IsContinuationExpected: False, } - self.diagnostics = CosmosDiagnostics() + self._user_agent = _utils.get_user_agent() + self.diagnostics = CosmosDiagnostics(ua=self._user_agent) # Keeps the latest response headers from the server. self.last_response_headers = None # Keep Track of Last Exceptions @@ -178,7 +179,6 @@ def __init__( proxy = host if url.port else host + ":" + str(self.connection_policy.ProxyConfiguration.Port) proxies.update({url.scheme: proxy}) - self._user_agent = _utils.get_user_agent() credentials_policy = None if self.aad_credentials: @@ -214,6 +214,9 @@ def __init__( # Use database_account if no consistency passed in to verify consistency level to be used self._set_client_consistency_level(database_account, consistency_level) + # initialize diagnostics + + def _set_client_consistency_level( self, diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py index c18360fd0ee9..cc5be65ab705 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py @@ -145,7 +145,7 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin data = response.body() if data: data = data.decode("utf-8") - global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text) + global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text, request_headers=ir.request.headers) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py index ca272aa36426..cc48eaceea83 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py @@ -105,14 +105,14 @@ async def _Request(global_endpoint_manager, request_params, connection_policy, p connection_verify=kwargs.pop("connection_verify", is_ssl_enabled), **kwargs ) - + ir = response.http_response.internal_response response = response.http_response headers = dict(response.headers) data = response.body() if data: data = data.decode("utf-8") - + global_endpoint_manager.client.diagnostics.update_diagnostics(header=headers, body=data, status_code=ir.status, status_reason=ir.reason, response_text=ir._body.decode("utf-8"), request_headers=ir.request_info.headers) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py index 9e9707c44356..578009efca8c 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py @@ -394,3 +394,6 @@ async def _get_database_account(self, **kwargs: Any) -> DatabaseAccount: if response_hook: response_hook(self.client_connection.last_response_headers) return result + + async def diagnostics(self,p=False): + return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py index 408cd0acd1e9..d7b0a1d26e54 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py @@ -59,6 +59,7 @@ from .. import _utils from ..partition_key import _Undefined, _Empty from ._auth_policy_async import AsyncCosmosBearerTokenCredentialPolicy +from ..cosmos_diagnostics import CosmosDiagnostics ClassType = TypeVar("ClassType") # pylint: disable=protected-access @@ -145,6 +146,8 @@ def __init__( if consistency_level is not None: self.default_headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level + self._user_agent = _utils.get_user_agent_async() + self.diagnostics = CosmosDiagnostics(ua=self._user_agent) # Keeps the latest response headers from the server. self.last_response_headers = None @@ -178,7 +181,7 @@ def __init__( proxy = host if url.port else host + ":" + str(self.connection_policy.ProxyConfiguration.Port) proxies.update({url.scheme: proxy}) - self._user_agent = _utils.get_user_agent_async() + credentials_policy = None if self.aad_credentials: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py index 157fcd588c60..db947ea95c4d 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py @@ -4,6 +4,7 @@ from requests.structures import CaseInsensitiveDict import platform import os +from ._utils import get_user_agent from . import exceptions @@ -57,10 +58,11 @@ def new_property(self, value: type): "x-ms-retry-after-ms", } - def __init__(self): - self._headers = CaseInsensitiveDict() + def __init__(self, ua): + self._user_agent = ua + self._request_headers = CaseInsensitiveDict() + self._response_headers = CaseInsensitiveDict() self._body = None - self._request_charge = 0 self._status_code = 0 self._status_reason = "" self._substatus_code = 0 @@ -70,13 +72,20 @@ def __init__(self): self._system_information = self.get_system_info() @property - def headers(self): - return CaseInsensitiveDict(self._headers) + def response_headers(self): + return CaseInsensitiveDict(self._response_headers) - @headers.setter - def headers(self, value: dict): - self._headers = value - self._request_charge += float(value.get("x-ms-request-charge", 0)) + @response_headers.setter + def response_headers(self, value: dict): + self._response_headers = value + + @property + def request_headers(self): + return CaseInsensitiveDict(self._request_headers) + + @request_headers.setter + def request_headers(self, value: dict): + self._request_headers = value @property def body(self): @@ -123,19 +132,23 @@ def status_message(self): def status_message(self, value: str): self._status_message = value + + @property - def request_charge(self): - return self._request_charge + def user_agent(self): + return self._user_agent + @user_agent.setter + def user_agent(self, value: str): + self._user_agent = value - def clear(self): - self._request_charge = 0 def update_header_and_body(self, header, body): self.header(header) self.body(body) def update_diagnostics(self, header: dict, body: dict, **kwargs): - self.headers = header + self.clear() + self.response_headers = header self.body = body #Note: Plan is to use kwargs to be able to easily modify this function to update with needed information #notes figure out how to just get status information instead of just relying on exceptions being passed @@ -157,6 +170,12 @@ def update_diagnostics(self, header: dict, body: dict, **kwargs): self.substatus_code = ssc else: self.substatus_code = 0 + ua = kwargs.get("user_agent") + if ua: + self.user_agent = ua + rh = kwargs.get("request_headers") + if rh: + self.request_headers = dict(rh) if e: self.status_code = e.status_code self.status_message = e.message @@ -205,3 +224,14 @@ def get_system_info(self): ret["machine"] = platform.machine() return ret + + def clear(self): + self.response_headers = CaseInsensitiveDict() + self.request_headers = CaseInsensitiveDict() + self.body = None + self.status_code = 0 + self.status_reason = "" + self.substatus_code = 0 + self.status_message = "" + self.elapsed_time = None + pass From b381009f22bb4772e41269e0ee806c190701a6d0 Mon Sep 17 00:00:00 2001 From: bambriz Date: Tue, 23 Aug 2022 12:45:56 -0700 Subject: [PATCH 5/9] Updated Cosmos Diagnostics Added consistency leve;, Operation type, and resource type to Cosmos Diagnostics. Added additional fixes and code cleanup. --- .../azure/cosmos/_cosmos_client_connection.py | 3 +- .../aio/base_execution_context.py | 1 + .../base_execution_context.py | 1 + .../azure/cosmos/_retry_utility.py | 4 +- .../azure/cosmos/_synchronized_request.py | 5 +- .../azure/cosmos/aio/_asynchronous_request.py | 3 +- .../azure/cosmos/aio/_container.py | 16 ++++ .../azure/cosmos/aio/_cosmos_client.py | 15 +++- .../aio/_cosmos_client_connection_async.py | 3 +- .../azure/cosmos/aio/_database.py | 16 ++++ .../azure-cosmos/azure/cosmos/aio/_user.py | 16 ++++ .../azure-cosmos/azure/cosmos/container.py | 42 ++++----- .../azure/cosmos/cosmos_client.py | 13 +++ .../azure/cosmos/cosmos_diagnostics.py | 86 ++++++++++++------- .../azure-cosmos/azure/cosmos/database.py | 16 ++++ sdk/cosmos/azure-cosmos/azure/cosmos/user.py | 33 ++++--- 16 files changed, 191 insertions(+), 82 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index d8f252b22bf9..f66b1e3e7fab 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -238,7 +238,7 @@ def _set_client_consistency_level( else: # Set consistency level header to be used for the client self.default_headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level - + self.diagnostics.consistency_level = consistency_level if consistency_level == documents.ConsistencyLevel.Session: # create a session - this is maintained only if the default consistency level # on the client is set to session, or if the user explicitly sets it as a property @@ -2276,6 +2276,7 @@ def __Post(self, path, request_params, body, req_headers, **kwargs): tuple of (dict, dict) """ + print(request_params) request = self.pipeline_client.post(url=path, headers=req_headers) return synchronized_request.SynchronizedRequest( client=self, diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py index 075a7839350f..ba52823f5039 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py @@ -134,6 +134,7 @@ async def _fetch_items_helper_no_retries(self, fetch_function): else: self._continuation = None if fetched_items: + self._client.diagnostics.update_diagnostics(response_headers, fetched_items) break return fetched_items diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py index 134bad9c876a..1698a2d7acf1 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py @@ -132,6 +132,7 @@ def _fetch_items_helper_no_retries(self, fetch_function): else: self._continuation = None if fetched_items: + self._client.diagnostics.update_diagnostics(response_headers, fetched_items) break return fetched_items diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py index 87b667ab3c07..233303226da8 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py @@ -90,12 +90,10 @@ def Execute(client, global_endpoint_manager, function, *args, **kwargs): client.last_response_headers[ HttpHeaders.ThrottleRetryWaitTimeInMs ] = resourceThrottle_retry_policy.cummulative_wait_time_in_milliseconds - client.last_exceptions = None return result except exceptions.CosmosHttpResponseError as e: retry_policy = None - #Current way of passing status code and message up, only works for exceptions though - client.last_exceptions = e + if e.status_code == StatusCodes.FORBIDDEN and e.sub_status == SubStatusCodes.WRITE_FORBIDDEN: retry_policy = endpointDiscovery_retry_policy elif e.status_code == StatusCodes.TOO_MANY_REQUESTS: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py index cc5be65ab705..07000abb90bf 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py @@ -136,7 +136,7 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin connection_verify=kwargs.pop("connection_verify", is_ssl_enabled), **kwargs ) - #Find a way to leverage this information + #Will make updating diagnostics less verbose ir = response.http_response.internal_response response = response.http_response headers = dict(response.headers) @@ -145,7 +145,8 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin data = response.body() if data: data = data.decode("utf-8") - global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text, request_headers=ir.request.headers) + global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text, + request_headers=ir.request.headers, operation_type=request_params.operation_type, resource_type=request_params.resource_type) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py index cc48eaceea83..3005e697e37a 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py @@ -112,7 +112,8 @@ async def _Request(global_endpoint_manager, request_params, connection_policy, p data = response.body() if data: data = data.decode("utf-8") - global_endpoint_manager.client.diagnostics.update_diagnostics(header=headers, body=data, status_code=ir.status, status_reason=ir.reason, response_text=ir._body.decode("utf-8"), request_headers=ir.request_info.headers) + global_endpoint_manager.client.diagnostics.update_diagnostics(header=headers, body=data, status_code=ir.status, status_reason=ir.reason, response_text=ir._body.decode("utf-8"), request_headers=ir.request_info.headers, + operation_type=request_params.operation_type, resource_type=request_params.resource_type) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py index d9d67adddeea..e5a83f776e05 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py @@ -760,3 +760,19 @@ async def delete_conflict( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) + + async def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py index 578009efca8c..50a2acbaa2aa 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py @@ -395,5 +395,18 @@ async def _get_database_account(self, **kwargs: Any) -> DatabaseAccount: response_hook(self.client_connection.last_response_headers) return result - async def diagnostics(self,p=False): + async def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py index d7b0a1d26e54..f665b86e0091 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py @@ -254,7 +254,7 @@ async def _setup(self): else: # Use database_account if no consistency passed in to verify consistency level to be used user_defined_consistency = self._check_if_account_session_consistency(database_account) - + self.diagnostics.consistency_level = user_defined_consistency if user_defined_consistency == documents.ConsistencyLevel.Session: # create a Session if the user wants Session consistency self.session = _session.Session(self.url_connection) @@ -279,7 +279,6 @@ def _check_if_account_session_consistency( # We only set the header if we're using session consistency in the account in order to keep # the current update_session logic which uses the header self.default_headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level - return consistency_level diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py index 1f47163b666e..1824cefc2a55 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py @@ -783,3 +783,19 @@ async def replace_throughput(self, throughput: int, **kwargs: Any) -> Throughput if response_hook: response_hook(self.client_connection.last_response_headers, data) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) + + async def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py index 58563f9e1b12..946a408bed52 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py @@ -316,3 +316,19 @@ async def delete_permission(self, permission: Union[str, Dict[str, Any], Permiss if response_hook: response_hook(self.client_connection.last_response_headers, result) + + async def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py index b057052fa9d8..d4fe35aadd1c 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py @@ -35,7 +35,6 @@ from .offer import ThroughputProperties from .scripts import ScriptsProxy from .partition_key import NonePartitionKeyValue -from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("ContainerProxy",) @@ -66,7 +65,6 @@ def __init__(self, client_connection, database_link, id, properties=None): # py self.container_link = u"{}/colls/{}".format(database_link, self.id) self._is_system_key = None self._scripts = None # type: Optional[ScriptsProxy] - # self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -167,8 +165,6 @@ def read( if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, self.client_connection.last_exceptions) - return cast('Dict[str, Any]', self._properties) @distributed_trace @@ -230,8 +226,6 @@ def read_item( result = self.client_connection.ReadItem(document_link=doc_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -279,8 +273,6 @@ def read_all_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, - # self.client_connection.last_exceptions) return items @distributed_trace @@ -328,8 +320,6 @@ def query_items_change_feed( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -423,8 +413,6 @@ def query_items( ) if response_hook: response_hook(self.client_connection.last_response_headers, items) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, items, - # self.client_connection.last_exceptions) return items @distributed_trace @@ -477,8 +465,6 @@ def replace_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -531,8 +517,6 @@ def upsert_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -588,8 +572,6 @@ def create_item( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -640,8 +622,6 @@ def delete_item( result = self.client_connection.DeleteItem(document_link=document_link, options=request_options, **kwargs) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) @distributed_trace def read_offer(self, **kwargs): @@ -687,8 +667,6 @@ def get_throughput(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, throughput_properties) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, throughput_properties, - # self.client_connection.last_exceptions) return ThroughputProperties(offer_throughput=throughput_properties[0]["content"]["offerThroughput"], properties=throughput_properties[0]) @@ -796,8 +774,6 @@ def query_conflicts( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) return result @distributed_trace @@ -850,5 +826,19 @@ def delete_conflict(self, conflict, partition_key, **kwargs): ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - # self.client_connection.last_exceptions) + + def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py index 0ad3c2903627..d21525ffc0ca 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py @@ -490,4 +490,17 @@ def get_database_account(self, **kwargs): return result def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ return self.client_connection.diagnostics(p=p) \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py index db947ea95c4d..e659bf920a01 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py @@ -60,13 +60,16 @@ def new_property(self, value: type): def __init__(self, ua): self._user_agent = ua + self._consistency_level = None + self._operation_type = None + self._resource_type = None self._request_headers = CaseInsensitiveDict() self._response_headers = CaseInsensitiveDict() self._body = None - self._status_code = 0 - self._status_reason = "" + self._status_code = None + self._status_reason = None self._substatus_code = 0 - self._status_message = "" + self._status_message = None self._elapsed_time = None self._system_information = self.get_system_info() @@ -89,7 +92,7 @@ def request_headers(self, value: dict): @property def body(self): - return self._body + return dict(self._body) @body.setter def body(self, value: dict): @@ -106,6 +109,7 @@ def status_code(self, value: int): @property def substatus_code(self): return self._substatus_code + @substatus_code.setter def substatus_code(self, value: int): self._substatus_code = value @@ -113,6 +117,7 @@ def substatus_code(self, value: int): @property def status_reason(self): return self._status_reason + @status_reason.setter def status_reason(self, value: str): self._status_reason = value @@ -120,6 +125,7 @@ def status_reason(self, value: str): @property def elapsed_time(self): return self._elapsed_time + @elapsed_time.setter def elapsed_time(self, value): self._elapsed_time = value @@ -132,26 +138,45 @@ def status_message(self): def status_message(self, value: str): self._status_message = value - - @property def user_agent(self): return self._user_agent + @user_agent.setter def user_agent(self, value: str): self._user_agent = value + @property + def consistency_level(self): + return self._consistency_level - def update_header_and_body(self, header, body): - self.header(header) - self.body(body) + @consistency_level.setter + def consistency_level(self, value: str): + self._consistency_level = value + + @property + def operation_type(self): + return self._operation_type + + @operation_type.setter + def operation_type(self, value: str): + self._operation_type = value + + @property + def resource_type(self): + return self._resource_type + + @resource_type.setter + def resource_type(self, value: str): + self._resource_type = value def update_diagnostics(self, header: dict, body: dict, **kwargs): self.clear() self.response_headers = header - self.body = body - #Note: Plan is to use kwargs to be able to easily modify this function to update with needed information - #notes figure out how to just get status information instead of just relying on exceptions being passed + try: + self.body = dict(body) + except ValueError as ve: + self.body = body eTime = kwargs.get("elapsed_time") if eTime: self.elapsed_time = eTime @@ -176,11 +201,16 @@ def update_diagnostics(self, header: dict, body: dict, **kwargs): rh = kwargs.get("request_headers") if rh: self.request_headers = dict(rh) + ot = kwargs.get("operation_type") + if ot: + self.operation_type = ot + rt = kwargs.get("resource_type") + if rt: + self.resource_type = rt if e: self.status_code = e.status_code self.status_message = e.message - #Note: Planning to add a flag to print it in a pretty format instead of just returning a dictionary def __call__(self, p=False): #This will format all the properties into a dictionary ret = { @@ -198,22 +228,19 @@ def __call__(self, p=False): print(str(key)+": "+str(value)) else: return ret - # return { - # key: value - # for key, value in self.__dict__.items() - # if isinstance(value, property) - # } def __getattr__(self, name): temp_name = name key = "x-ms-" + temp_name.replace("_", "-") - if key in self._common: - return self._headers[key] - else: - return getattr(self, name) - raise AttributeError(name) - - #Current System info I was able to get + try: + if key in self._common: + return self._response_headers[key] + else: + return getattr(self, name) + except: + raise AttributeError(name) + + #Current System info def get_system_info(self): ret = {} ret["system"] = platform.system() @@ -229,9 +256,10 @@ def clear(self): self.response_headers = CaseInsensitiveDict() self.request_headers = CaseInsensitiveDict() self.body = None - self.status_code = 0 - self.status_reason = "" + self.status_code = None + self.status_reason = None self.substatus_code = 0 - self.status_message = "" + self.status_message = None self.elapsed_time = None - pass + self.operation_type = None + self.resource_type = None diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 43878309a78f..1e3adef82165 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -829,3 +829,19 @@ def replace_throughput(self, throughput, **kwargs): response_hook(self.client_connection.last_response_headers, data) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) + + def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py index e5c5b1c5c3de..d7990142287b 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py @@ -47,7 +47,6 @@ def __init__(self, client_connection, id, database_link, properties=None): # py self.id = id self.user_link = u"{}/users/{}".format(database_link, id) self._properties = properties - self.diagnostics = CosmosDiagnostics() def __repr__(self): # type () -> str @@ -86,8 +85,6 @@ def read(self, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, self._properties) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, self._properties, - self.client_connection.last_exceptions) return cast('Dict[str, Any]', self._properties) @@ -110,8 +107,6 @@ def list_permissions(self, max_item_count=None, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) return result @@ -147,8 +142,6 @@ def query_permissions( if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) return result @@ -173,8 +166,6 @@ def get_permission(self, permission, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission_resp) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission_resp, - self.client_connection.last_exceptions) return Permission( id=permission_resp["id"], @@ -206,8 +197,6 @@ def create_permission(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission, - self.client_connection.last_exceptions) return Permission( id=permission["id"], @@ -240,8 +229,6 @@ def upsert_permission(self, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission, - self.client_connection.last_exceptions) return Permission( id=permission["id"], @@ -276,8 +263,6 @@ def replace_permission(self, permission, body, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, permission_resp) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, permission_resp, - self.client_connection.last_exceptions) return Permission( id=permission_resp["id"], @@ -310,5 +295,19 @@ def delete_permission(self, permission, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) - self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, - self.client_connection.last_exceptions) + + def diagnostics(self, p=False): + """Returns a dictionary of the diagnostics from the last request. + + Best used when catching an exception. + + Ex: + >>>try: + >>> database = client.create_database_if_not_exists(id="DatabaseTest") + >>>except: + >>> client.diagnostics(p=True) + + :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. + :rtype: dict + """ + return self.client_connection.diagnostics(p=p) From 8e49aff769059d09ac760cb690a1934c2a04140d Mon Sep 17 00:00:00 2001 From: bambriz Date: Tue, 23 Aug 2022 12:48:34 -0700 Subject: [PATCH 6/9] Removed print in cosmos client connection Removed left over print statement. Previous update also included query diagnostics. --- .../azure-cosmos/azure/cosmos/_cosmos_client_connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index f66b1e3e7fab..4fa8ff01468f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -2276,7 +2276,6 @@ def __Post(self, path, request_params, body, req_headers, **kwargs): tuple of (dict, dict) """ - print(request_params) request = self.pipeline_client.post(url=path, headers=req_headers) return synchronized_request.SynchronizedRequest( client=self, From 97a4d97205c1d1f01115372885767762d0dd42dd Mon Sep 17 00:00:00 2001 From: bambriz Date: Mon, 29 Aug 2022 16:55:12 -0700 Subject: [PATCH 7/9] Diagnostics via Azure Core Policy Changed the way we got diagnsotics http data. Instead of the original approach, I am now using the azure core http policy. I made a child class and replaced it in client connection. By default it performs the same, but with a new flag it will include additional information the cosmos diagnostics class would include. --- .../azure/cosmos/_cosmos_client_connection.py | 3 +- .../cosmos/_cosmos_http_logging_policy.py | 135 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index 4fa8ff01468f..c3da60691405 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -60,6 +60,7 @@ from .partition_key import _Undefined, _Empty from ._auth_policy import CosmosBearerTokenCredentialPolicy from .cosmos_diagnostics import CosmosDiagnostics +from ._cosmos_http_logging_policy import CosmosHttpLoggingPolicy ClassType = TypeVar("ClassType") # pylint: disable=protected-access @@ -195,7 +196,7 @@ def __init__( CustomHookPolicy(**kwargs), NetworkTraceLoggingPolicy(**kwargs), DistributedTracingPolicy(**kwargs), - HttpLoggingPolicy(**kwargs), + CosmosHttpLoggingPolicy(enable_diagnostics_logging=kwargs.pop("enable_diagnostics_logging",None), **kwargs), ] transport = kwargs.pop("transport", None) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py new file mode 100644 index 000000000000..2afa86c765f9 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py @@ -0,0 +1,135 @@ +import os +import urllib + +from azure.core.pipeline.policies import HttpLoggingPolicy +import logging +import types + + + +class CosmosHttpLoggingPolicy(HttpLoggingPolicy): + + def __init__(self, logger=None, enable_diagnostics_logging=False, **kwargs): + self._diagnostics_mode = enable_diagnostics_logging + if not enable_diagnostics_logging: + super().__init__(logger, **kwargs) + else: + self.logger = logger or logging.getLogger(__name__) + self.allowed_query_params = set() + + def on_request(self, request): # type: (PipelineRequest) -> None + if self._diagnostics_mode: + http_request = request.http_request + options = request.context.options + # Get logger in my context first (request has been retried) + # then read from kwargs (pop if that's the case) + # then use my instance logger + logger = request.context.setdefault("logger", options.pop("logger", self.logger)) + + if not logger.isEnabledFor(logging.INFO): + return + + try: + parsed_url = list(urllib.parse.urlparse(http_request.url)) + parsed_qp = urllib.parse.parse_qsl(parsed_url[4], keep_blank_values=True) + filtered_qp = [(key, self._redact_query_param(key, value)) for key, value in parsed_qp] + # 4 is query + parsed_url[4] = "&".join(["=".join(part) for part in filtered_qp]) + redacted_url = urllib.parse.urlunparse(parsed_url) + + multi_record = os.environ.get(HttpLoggingPolicy.MULTI_RECORD_LOG, False) + if multi_record: + logger.info("Request URL: %r", redacted_url) + logger.info("Request method: %r", http_request.method) + logger.info("Request headers:") + for header, value in http_request.headers.items(): + #value = self._redact_header(header, value) + logger.info(" %r: %r", header, value) + if isinstance(http_request.body, types.GeneratorType): + logger.info("File upload") + return + try: + if isinstance(http_request.body, types.AsyncGeneratorType): + logger.info("File upload") + return + except AttributeError: + pass + if http_request.body: + logger.info("A body is sent with the request") + return + logger.info("No body was attached to the request") + return + log_string = "Request URL: '{}'".format(redacted_url) + log_string += "\nRequest method: '{}'".format(http_request.method) + log_string += "\nRequest headers:" + for header, value in http_request.headers.items(): + #value = self._redact_header(header, value) + log_string += "\n '{}': '{}'".format(header, value) + if isinstance(http_request.body, types.GeneratorType): + log_string += "\nFile upload" + logger.info(log_string) + return + try: + if isinstance(http_request.body, types.AsyncGeneratorType): + log_string += "\nFile upload" + logger.info(log_string) + return + except AttributeError: + pass + if http_request.body: + log_string += "\nA body is sent with the request" + logger.info(log_string) + return + log_string += "\nNo body was attached to the request" + logger.info(log_string) + + except Exception as err: # pylint: disable=broad-except + logger.warning("Failed to log request: %s", repr(err)) + else: + super().on_request(request) + + + def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None + if self._diagnostics_mode: + http_response = response.http_response + ir = http_response.internal_response + try: + logger = response.context["logger"] + + if not logger.isEnabledFor(logging.INFO): + return + + multi_record = os.environ.get(HttpLoggingPolicy.MULTI_RECORD_LOG, False) + if multi_record: + logger.info("Response status: %r", http_response.status_code) + logger.info("Response status reason: %r", ir.reason) + logger.info("Elapsed Time: %r".ir.elapsed) + if http_response.status_code >= 400: + sm = ir.text + sm.replace("true", "True") + sm.replace("false", "False") + temp_sm = eval(sm) + temp_sm['message'] = temp_sm['message'].replace("\r", " ") + logger.into("Response status error message: %r", temp_sm['message']) + logger.info("Response headers:") + for res_header, value in http_response.headers.items(): + logger.info(" %r: %r", res_header, value) + return + log_string = "Response status: {}".format(http_response.status_code) + log_string += "\nResponse status reason: {}".format(ir.reason) + log_string += "\nElapsed Time: {}".format(ir.elapsed) + if http_response.status_code >= 400: + sm = ir.text + sm.replace("true", "True") + sm.replace("false", "False") + temp_sm = eval(sm) + temp_sm['message'] = temp_sm['message'].replace("\r", " ") + log_string += "\nResponse status error message: {}".format(temp_sm['message']) + log_string += "\nResponse headers:" + for res_header, value in http_response.headers.items(): + log_string += "\n '{}': '{}'".format(res_header, value) + logger.info(log_string) + except Exception as err: # pylint: disable=broad-except + logger.warning("Failed to log response: %s", repr(err)) + else: + super().on_response(request, response) From 7c1a152ddb4f4cb96d7cdbec15044db12ba78f99 Mon Sep 17 00:00:00 2001 From: bambriz Date: Tue, 30 Aug 2022 19:34:35 -0700 Subject: [PATCH 8/9] Removed all references to Cosmos Diagnostics and added Test, Readme entry, and changelog for new changes Removed Old cosmos diagnostics code Added Cosmos Http Logging Policy Test as well as readme entry and changelog. Other minor fixes. --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 1 + sdk/cosmos/azure-cosmos/README.md | 25 ++ .../azure/cosmos/_cosmos_client_connection.py | 9 +- .../cosmos/_cosmos_http_logging_policy.py | 43 ++- .../aio/base_execution_context.py | 1 - .../base_execution_context.py | 1 - .../azure/cosmos/_synchronized_request.py | 5 +- .../azure/cosmos/aio/_asynchronous_request.py | 3 - .../azure/cosmos/aio/_container.py | 16 -- .../azure/cosmos/aio/_cosmos_client.py | 15 - .../aio/_cosmos_client_connection_async.py | 10 +- .../azure/cosmos/aio/_database.py | 15 - .../azure-cosmos/azure/cosmos/aio/_user.py | 15 - .../azure-cosmos/azure/cosmos/container.py | 15 - .../azure/cosmos/cosmos_client.py | 16 -- .../azure/cosmos/cosmos_diagnostics.py | 265 ------------------ .../azure/cosmos/cosmoshttploggingpolicy.md | 3 + .../azure-cosmos/azure/cosmos/database.py | 16 -- sdk/cosmos/azure-cosmos/azure/cosmos/user.py | 15 - sdk/cosmos/azure-cosmos/img.png | Bin 0 -> 477 bytes .../test/test_cosmos_http_logging_policy.py | 133 +++++++++ 21 files changed, 201 insertions(+), 421 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py create mode 100644 sdk/cosmos/azure-cosmos/azure/cosmos/cosmoshttploggingpolicy.md create mode 100644 sdk/cosmos/azure-cosmos/img.png create mode 100644 sdk/cosmos/azure-cosmos/test/test_cosmos_http_logging_policy.py diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 2ec996b0fc5d..07421f611c10 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.3.1 (Unreleased) #### Features Added +- Added `CosmosHttpLoggingPolicy` to replace `HttpLoggingPolicy` for logging HTTP sessions. #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos/README.md b/sdk/cosmos/azure-cosmos/README.md index f79f8f048540..5ac00e471518 100644 --- a/sdk/cosmos/azure-cosmos/README.md +++ b/sdk/cosmos/azure-cosmos/README.md @@ -602,6 +602,31 @@ even when it isn't enabled for the client: database = client.create_database(DATABASE_NAME, logging_enable=True) ``` +Alternatively, you can log using the CosmosHttpLoggingPolicy, which extends from the azure core HttpLoggingPolicy, by passing in your logger to the `logger` argument. +By default, it will use the behaviour from HttpLoggingPolicy. The `enable_diagnostics_logging` argument will add additional diagnostic information to the logger. +```python +import logging +from azure.cosmos import CosmosClient + +#Create a logger for the 'azure' SDK +logger = logging.getLogger('azure') +logger.setLevel(logging.DEBUG) + +# Configure a file output +handler = logging.FileHandler(filename="azure") +logger.addHandler(handler) + +# This client will log diagnostic information from the HTTP session by using the CosmosHttpLoggingPolicy +client = CosmosClient(URL, credential=KEY, logger=logger, enable_diagnostics_logging=True) +``` +| Name | Policy Flavor | Parameters | Accepted in Init? | Accepted in Request? | Description | +|-------------------------|------------------|----------------------------|----------------|------------------|-------------------------------------------------------------------------------| +| CosmosHttpLoggingPolicy | SansIOHTTPPolicy | | | | | +| | | logger | X | X | If specified, it will be used to log information. | +| | | enable_diagnostics_logging | X | X | Used to enable logging additional diagnostic information. Defaults to `False` | +You can learn more about the different policies and how they work [here](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#available-policies) + + ## Next steps For more extensive documentation on the Cosmos DB service, see the [Azure Cosmos DB documentation][cosmos_docs] on docs.microsoft.com. diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index c3da60691405..f14fc9bba96a 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -143,12 +143,9 @@ def __init__( # We need to set continuation as not expected. http_constants.HttpHeaders.IsContinuationExpected: False, } - self._user_agent = _utils.get_user_agent() - self.diagnostics = CosmosDiagnostics(ua=self._user_agent) + # Keeps the latest response headers from the server. self.last_response_headers = None - # Keep Track of Last Exceptions - self.last_exceptions = None self._useMultipleWriteLocations = False self._global_endpoint_manager = global_endpoint_manager._GlobalEndpointManager(self) @@ -180,6 +177,7 @@ def __init__( proxy = host if url.port else host + ":" + str(self.connection_policy.ProxyConfiguration.Port) proxies.update({url.scheme: proxy}) + self._user_agent = _utils.get_user_agent() credentials_policy = None if self.aad_credentials: @@ -196,7 +194,7 @@ def __init__( CustomHookPolicy(**kwargs), NetworkTraceLoggingPolicy(**kwargs), DistributedTracingPolicy(**kwargs), - CosmosHttpLoggingPolicy(enable_diagnostics_logging=kwargs.pop("enable_diagnostics_logging",None), **kwargs), + CosmosHttpLoggingPolicy(**kwargs), ] transport = kwargs.pop("transport", None) @@ -239,7 +237,6 @@ def _set_client_consistency_level( else: # Set consistency level header to be used for the client self.default_headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level - self.diagnostics.consistency_level = consistency_level if consistency_level == documents.ConsistencyLevel.Session: # create a session - this is maintained only if the default consistency level # on the client is set to session, or if the user explicitly sets it as a property diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py index 2afa86c765f9..8e0ca452299f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py @@ -9,18 +9,19 @@ class CosmosHttpLoggingPolicy(HttpLoggingPolicy): - def __init__(self, logger=None, enable_diagnostics_logging=False, **kwargs): - self._diagnostics_mode = enable_diagnostics_logging - if not enable_diagnostics_logging: - super().__init__(logger, **kwargs) - else: + def __init__(self, logger=None, **kwargs): + self._enable_diagnostics_logging = kwargs.pop("enable_diagnostics_logging", None) + super().__init__(logger, **kwargs) + if self._enable_diagnostics_logging: self.logger = logger or logging.getLogger(__name__) - self.allowed_query_params = set() def on_request(self, request): # type: (PipelineRequest) -> None - if self._diagnostics_mode: - http_request = request.http_request - options = request.context.options + http_request = request.http_request + options = request.context.options + self._enable_diagnostics_logging = request.context.setdefault( + "enable_diagnostics_logging", + options.pop("enable_diagnostics_logging", self._enable_diagnostics_logging)) + if self._enable_diagnostics_logging: # Get logger in my context first (request has been retried) # then read from kwargs (pop if that's the case) # then use my instance logger @@ -90,7 +91,7 @@ def on_request(self, request): # type: (PipelineRequest) -> None def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None - if self._diagnostics_mode: + if self._enable_diagnostics_logging: http_response = response.http_response ir = http_response.internal_response try: @@ -103,7 +104,10 @@ def on_response(self, request, response): # type: (PipelineRequest, PipelineRes if multi_record: logger.info("Response status: %r", http_response.status_code) logger.info("Response status reason: %r", ir.reason) - logger.info("Elapsed Time: %r".ir.elapsed) + try: + logger.info("Elapsed Time: %r", ir.elapsed) + except AttributeError: + logger.info("Elapsed Time: %r", None) if http_response.status_code >= 400: sm = ir.text sm.replace("true", "True") @@ -117,7 +121,10 @@ def on_response(self, request, response): # type: (PipelineRequest, PipelineRes return log_string = "Response status: {}".format(http_response.status_code) log_string += "\nResponse status reason: {}".format(ir.reason) - log_string += "\nElapsed Time: {}".format(ir.elapsed) + try: + log_string += "\nElapsed Time: {}".format(ir.elapsed) + except AttributeError: + log_string += "\nElapsed Time: {}".format(None) if http_response.status_code >= 400: sm = ir.text sm.replace("true", "True") @@ -133,3 +140,15 @@ def on_response(self, request, response): # type: (PipelineRequest, PipelineRes logger.warning("Failed to log response: %s", repr(err)) else: super().on_response(request, response) + + # # Current System info + # def get_system_info(self): + # ret = {} + # ret["system"] = platform.system() + # ret["python version"] = platform.python_version() + # ret["architecture"] = platform.architecture() + # ret["cpu"] = platform.processor() + # ret["cpu count"] = os.cpu_count() + # ret["machine"] = platform.machine() + # + # return ret \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py index ba52823f5039..075a7839350f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/base_execution_context.py @@ -134,7 +134,6 @@ async def _fetch_items_helper_no_retries(self, fetch_function): else: self._continuation = None if fetched_items: - self._client.diagnostics.update_diagnostics(response_headers, fetched_items) break return fetched_items diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py index 1698a2d7acf1..134bad9c876a 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/base_execution_context.py @@ -132,7 +132,6 @@ def _fetch_items_helper_no_retries(self, fetch_function): else: self._continuation = None if fetched_items: - self._client.diagnostics.update_diagnostics(response_headers, fetched_items) break return fetched_items diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py index 07000abb90bf..165fd6e5bce3 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_synchronized_request.py @@ -136,8 +136,7 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin connection_verify=kwargs.pop("connection_verify", is_ssl_enabled), **kwargs ) - #Will make updating diagnostics less verbose - ir = response.http_response.internal_response + response = response.http_response headers = dict(response.headers) @@ -145,8 +144,6 @@ def _Request(global_endpoint_manager, request_params, connection_policy, pipelin data = response.body() if data: data = data.decode("utf-8") - global_endpoint_manager.Client.diagnostics.update_diagnostics(header=headers, body=data, elapsed_time=ir.elapsed, status_code=ir.status_code, status_reason=ir.reason, response_text=ir.text, - request_headers=ir.request.headers, operation_type=request_params.operation_type, resource_type=request_params.resource_type) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py index 3005e697e37a..d573d819caa1 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_asynchronous_request.py @@ -105,15 +105,12 @@ async def _Request(global_endpoint_manager, request_params, connection_policy, p connection_verify=kwargs.pop("connection_verify", is_ssl_enabled), **kwargs ) - ir = response.http_response.internal_response response = response.http_response headers = dict(response.headers) data = response.body() if data: data = data.decode("utf-8") - global_endpoint_manager.client.diagnostics.update_diagnostics(header=headers, body=data, status_code=ir.status, status_reason=ir.reason, response_text=ir._body.decode("utf-8"), request_headers=ir.request_info.headers, - operation_type=request_params.operation_type, resource_type=request_params.resource_type) if response.status_code == 404: raise exceptions.CosmosResourceNotFoundError(message=data, response=response) if response.status_code == 409: diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py index e5a83f776e05..d9d67adddeea 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py @@ -760,19 +760,3 @@ async def delete_conflict( ) if response_hook: response_hook(self.client_connection.last_response_headers, result) - - async def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py index 50a2acbaa2aa..4a2a7329cf10 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py @@ -395,18 +395,3 @@ async def _get_database_account(self, **kwargs: Any) -> DatabaseAccount: response_hook(self.client_connection.last_response_headers) return result - async def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py index f665b86e0091..b7f062f5160b 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py @@ -59,7 +59,7 @@ from .. import _utils from ..partition_key import _Undefined, _Empty from ._auth_policy_async import AsyncCosmosBearerTokenCredentialPolicy -from ..cosmos_diagnostics import CosmosDiagnostics +from .._cosmos_http_logging_policy import CosmosHttpLoggingPolicy ClassType = TypeVar("ClassType") # pylint: disable=protected-access @@ -146,8 +146,7 @@ def __init__( if consistency_level is not None: self.default_headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level - self._user_agent = _utils.get_user_agent_async() - self.diagnostics = CosmosDiagnostics(ua=self._user_agent) + # Keeps the latest response headers from the server. self.last_response_headers = None @@ -181,7 +180,7 @@ def __init__( proxy = host if url.port else host + ":" + str(self.connection_policy.ProxyConfiguration.Port) proxies.update({url.scheme: proxy}) - + self._user_agent = _utils.get_user_agent_async() credentials_policy = None if self.aad_credentials: @@ -198,7 +197,7 @@ def __init__( CustomHookPolicy(**kwargs), NetworkTraceLoggingPolicy(**kwargs), DistributedTracingPolicy(**kwargs), - HttpLoggingPolicy(**kwargs), + CosmosHttpLoggingPolicy(**kwargs), ] transport = kwargs.pop("transport", None) @@ -254,7 +253,6 @@ async def _setup(self): else: # Use database_account if no consistency passed in to verify consistency level to be used user_defined_consistency = self._check_if_account_session_consistency(database_account) - self.diagnostics.consistency_level = user_defined_consistency if user_defined_consistency == documents.ConsistencyLevel.Session: # create a Session if the user wants Session consistency self.session = _session.Session(self.url_connection) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py index 1824cefc2a55..382c22ad8b2b 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py @@ -784,18 +784,3 @@ async def replace_throughput(self, throughput: int, **kwargs: Any) -> Throughput response_hook(self.client_connection.last_response_headers, data) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) - async def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py index 946a408bed52..dd495cd40345 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_user.py @@ -317,18 +317,3 @@ async def delete_permission(self, permission: Union[str, Dict[str, Any], Permiss if response_hook: response_hook(self.client_connection.last_response_headers, result) - async def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py index d4fe35aadd1c..bb70a7fc20aa 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/container.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/container.py @@ -827,18 +827,3 @@ def delete_conflict(self, conflict, partition_key, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) - def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py index d21525ffc0ca..702d9df9130f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py @@ -488,19 +488,3 @@ def get_database_account(self, **kwargs): # self.diagnostics.update_diagnostics(self.client_connection.last_response_headers, result, # self.client_connection.last_exceptions) return result - - def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py deleted file mode 100644 index e659bf920a01..000000000000 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_diagnostics.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Diagnostic tools for Azure Cosmos database service operations. -""" - -from requests.structures import CaseInsensitiveDict -import platform -import os -from ._utils import get_user_agent -from . import exceptions - - -class CosmosDiagnostics(object): - """Record Response headers and other useful information from Cosmos read operations. - - The full response headers are stored in the ``headers`` property. - - Calling the class as a function will return a dictionary of all Diagnostics properties. - - Examples: - - >>> col = b.create_container( - ... id="some_container", - ... partition_key=PartitionKey(path='/id', kind='Hash'), - >>> cd = col.diagnostics() #col.diagnostics would return the object itself which can still be used to get the properties. - >>> cd['headers']['x-ms-activity-id'] - '6243eeed-f06a-413d-b913-dcf8122d0642' - >>>cd - {headers: {x-ms-activity-id: '6243eeed-f06a-413d-b913-dcf8122d0642', - ...: ... , - ...: ...}, - body: { ...: ..., - ...: ..., - ...: ...}, - request_charge: #, - error_code: #, - error_message: ... - ...: ...} - - New Properties can be added by adding a getter and setter for them. - - Example: - - @property - def new_property(self): - return self._new_property - - @new_property.setter - def new_property(self, value: type): - self._new_property = value - - """ - - _common = { - "x-ms-activity-id", - "x-ms-session-token", - "x-ms-item-count", - "x-ms-request-quota", - "x-ms-resource-usage", - "x-ms-retry-after-ms", - } - - def __init__(self, ua): - self._user_agent = ua - self._consistency_level = None - self._operation_type = None - self._resource_type = None - self._request_headers = CaseInsensitiveDict() - self._response_headers = CaseInsensitiveDict() - self._body = None - self._status_code = None - self._status_reason = None - self._substatus_code = 0 - self._status_message = None - self._elapsed_time = None - - self._system_information = self.get_system_info() - - @property - def response_headers(self): - return CaseInsensitiveDict(self._response_headers) - - @response_headers.setter - def response_headers(self, value: dict): - self._response_headers = value - - @property - def request_headers(self): - return CaseInsensitiveDict(self._request_headers) - - @request_headers.setter - def request_headers(self, value: dict): - self._request_headers = value - - @property - def body(self): - return dict(self._body) - - @body.setter - def body(self, value: dict): - self._body = value - - @property - def status_code(self): - return self._status_code - - @status_code.setter - def status_code(self, value: int): - self._status_code = value - - @property - def substatus_code(self): - return self._substatus_code - - @substatus_code.setter - def substatus_code(self, value: int): - self._substatus_code = value - - @property - def status_reason(self): - return self._status_reason - - @status_reason.setter - def status_reason(self, value: str): - self._status_reason = value - - @property - def elapsed_time(self): - return self._elapsed_time - - @elapsed_time.setter - def elapsed_time(self, value): - self._elapsed_time = value - - @property - def status_message(self): - return self._status_message - - @status_message.setter - def status_message(self, value: str): - self._status_message = value - - @property - def user_agent(self): - return self._user_agent - - @user_agent.setter - def user_agent(self, value: str): - self._user_agent = value - - @property - def consistency_level(self): - return self._consistency_level - - @consistency_level.setter - def consistency_level(self, value: str): - self._consistency_level = value - - @property - def operation_type(self): - return self._operation_type - - @operation_type.setter - def operation_type(self, value: str): - self._operation_type = value - - @property - def resource_type(self): - return self._resource_type - - @resource_type.setter - def resource_type(self, value: str): - self._resource_type = value - - def update_diagnostics(self, header: dict, body: dict, **kwargs): - self.clear() - self.response_headers = header - try: - self.body = dict(body) - except ValueError as ve: - self.body = body - eTime = kwargs.get("elapsed_time") - if eTime: - self.elapsed_time = eTime - e = kwargs.get("exception") - sc = kwargs.get("status_code") - if sc: - self.status_code = sc - sr = kwargs.get("status_reason") - if sr: - self.status_reason = sr - sm = kwargs.get("response_text") - if sm: - self.status_message = sm - ssc = kwargs.get("substatus_code") - if ssc: - self.substatus_code = ssc - else: - self.substatus_code = 0 - ua = kwargs.get("user_agent") - if ua: - self.user_agent = ua - rh = kwargs.get("request_headers") - if rh: - self.request_headers = dict(rh) - ot = kwargs.get("operation_type") - if ot: - self.operation_type = ot - rt = kwargs.get("resource_type") - if rt: - self.resource_type = rt - if e: - self.status_code = e.status_code - self.status_message = e.message - - def __call__(self, p=False): - #This will format all the properties into a dictionary - ret = { - key: getattr(self, key) - for key in vars(self) - } - if p: - print("DIAGNOSTICS") - for key, value in ret.items(): - if type(value) is dict: - print(str(key)+":") - for k, v in value.items(): - print(" " + str(k) + ": " + str(v)) - else: - print(str(key)+": "+str(value)) - else: - return ret - - def __getattr__(self, name): - temp_name = name - key = "x-ms-" + temp_name.replace("_", "-") - try: - if key in self._common: - return self._response_headers[key] - else: - return getattr(self, name) - except: - raise AttributeError(name) - - #Current System info - def get_system_info(self): - ret = {} - ret["system"] = platform.system() - ret["python version"] = platform.python_version() - ret["architecture"] = platform.architecture() - ret["cpu"] = platform.processor() - ret["cpu count"] = os.cpu_count() - ret["machine"] = platform.machine() - - return ret - - def clear(self): - self.response_headers = CaseInsensitiveDict() - self.request_headers = CaseInsensitiveDict() - self.body = None - self.status_code = None - self.status_reason = None - self.substatus_code = 0 - self.status_message = None - self.elapsed_time = None - self.operation_type = None - self.resource_type = None diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/cosmoshttploggingpolicy.md b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmoshttploggingpolicy.md new file mode 100644 index 000000000000..acd0a90d68d7 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/cosmoshttploggingpolicy.md @@ -0,0 +1,3 @@ +# CosmosHttpLoggingPolicy Documentation + + diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 1e3adef82165..43878309a78f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -829,19 +829,3 @@ def replace_throughput(self, throughput, **kwargs): response_hook(self.client_connection.last_response_headers, data) return ThroughputProperties(offer_throughput=data["content"]["offerThroughput"], properties=data) - - def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py index d7990142287b..d11775fd8986 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py @@ -296,18 +296,3 @@ def delete_permission(self, permission, **kwargs): if response_hook: response_hook(self.client_connection.last_response_headers, result) - def diagnostics(self, p=False): - """Returns a dictionary of the diagnostics from the last request. - - Best used when catching an exception. - - Ex: - >>>try: - >>> database = client.create_database_if_not_exists(id="DatabaseTest") - >>>except: - >>> client.diagnostics(p=True) - - :param p: Defaulted to False, if set to true will print the diagnostics in a nicely formatted style. - :rtype: dict - """ - return self.client_connection.diagnostics(p=p) diff --git a/sdk/cosmos/azure-cosmos/img.png b/sdk/cosmos/azure-cosmos/img.png new file mode 100644 index 0000000000000000000000000000000000000000..8670b9b50fc9708f5fd39898bf40813649790c78 GIT binary patch literal 477 zcmeAS@N?(olHy`uVBq!ia0y~yV4DtP2XZh0$sIp4k1#MWF7b4645^5Fd(e;(D0pbW lSN~l0`~^VfC>Sgu@PpS07*Q8@J+lD0$kWx&Wt~$(69Cz37Cis} literal 0 HcmV?d00001 diff --git a/sdk/cosmos/azure-cosmos/test/test_cosmos_http_logging_policy.py b/sdk/cosmos/azure-cosmos/test/test_cosmos_http_logging_policy.py new file mode 100644 index 000000000000..6f03ec74bee2 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/test/test_cosmos_http_logging_policy.py @@ -0,0 +1,133 @@ +# ------------------------------------ +"""Tests for the HttpLoggingPolicy.""" +import pytest +import logging +import unittest +import unittest +import uuid +import azure.cosmos.cosmos_client as cosmos_client +import types +import test_config +try: + from unittest.mock import Mock +except ImportError: # python < 3.3 + from mock import Mock # type: ignore + + +pytestmark = pytest.mark.cosmosEmulator + +class MockHandler(logging.Handler): + def __init__(self): + super(MockHandler, self).__init__() + self.messages = [] + + def reset(self): + self.messages = [] + + def emit(self, record): + self.messages.append(record) + +@pytest.mark.usefixtures("teardown") +class test_cosmos_http_logger(unittest.TestCase): + + config = test_config._test_config + host = config.host + masterKey = config.masterKey + connectionPolicy = config.connectionPolicy + + @classmethod + def setUpClass(cls): + if (cls.masterKey == '[YOUR_KEY_HERE]' or + cls.host == '[YOUR_ENDPOINT_HERE]'): + raise Exception( + "You must specify your Azure Cosmos account values for " + "'masterKey' and 'host' at the top of this class to run the " + "tests.") + cls.mock_handler_default = MockHandler() + cls.mock_handler_diagnostic = MockHandler() + cls.logger_default = logging.getLogger("testloggerdefault") + cls.logger_default.addHandler(cls.mock_handler_default) + cls.logger_default.setLevel(logging.DEBUG) + cls.logger_diagnostic = logging.getLogger("testloggerdiagnostic") + cls.logger_diagnostic.addHandler(cls.mock_handler_default) + cls.logger_diagnostic.setLevel(logging.DEBUG) + cls.client_default = cosmos_client.CosmosClient(cls.host, cls.masterKey, + consistency_level="Session", connection_policy=cls.connectionPolicy, logger=cls.logger_default) + cls.client_diagnostic = cosmos_client.CosmosClient(cls.host, cls.masterKey, + consistency_level="Session", + connection_policy=cls.connectionPolicy, logger=cls.logger_default, enable_diagnostic_logging=True) + + def test_default_http_logging_policy(self): + #Test if we can log into from creating a database + test_db = self.client_default.create_database_if_not_exists(id="DBTEST") + assert all(m.levelname == 'INFO' for m in self.mock_handler_default.messages) + messages_request = self.mock_handler_default.messages[0].message.split("\n") + messages_response = self.mock_handler_default.messages[1].message.split("\n") + assert "//dbs" in messages_request[0] + assert messages_request[1] == "Request method: 'POST'" + assert 'Request headers:' in messages_request[2] + assert messages_request[3] == 'No body was attached to the request' + assert messages_response[0] == 'Response status: 201' + assert 'Response headers:' in messages_response[1] + + self.mock_handler_default.reset() + #now test in case of an error + try: + test_db_error = self.client_default.create_database(id="DBTEST") + except: + pass + assert all(m.levelname == 'INFO' for m in self.mock_handler_default.messages) + messages_request = self.mock_handler_default.messages[0].message.split("\n") + messages_response = self.mock_handler_default.messages[1].message.split("\n") + assert "//dbs" in messages_request[0] + assert messages_request[1] == "Request method: 'POST'" + assert 'Request headers:' in messages_request[2] + assert messages_request[3] == 'A body is sent with the request' + assert messages_response[0] == 'Response status: 409' + assert 'Response headers:' in messages_response[1] + + #delete database + self.client_default.delete_database("DBTEST") + + self.mock_handler_default.reset() + + def test_cosmos_http_logging_policy(self): + # Test if we can log into from creating a database + test_db = self.client_diagnostic.create_database_if_not_exists(id="DBTEST") + assert all(m.levelname == 'INFO' for m in self.mock_handler_diagnostic.messages) + messages_request = self.mock_handler_diagnostic.messages[0].message.split("\n") + messages_response = self.mock_handler_diagnostic.messages[1].message.split("\n") + assert "//dbs" in messages_request[0] + assert messages_request[1] == "Request method: 'POST'" + assert 'Request headers:' in messages_request[2] + assert messages_request[3] == 'A body is sent with the request' + assert messages_response[0] == 'Response status: 201' + assert messages_response[1] == 'Response status reason: Created' + assert "Elapsed Time:" in messages_response[2] + assert "Response headers" in messages_response[3] + + self.mock_handler_diagnostic.reset() + # now test in case of an error + try: + test_db_error = self.client_diagnostic.create_database(id="DBTEST") + except: + pass + assert all(m.levelname == 'INFO' for m in self.mock_handler_diagnostic.messages) + messages_request = self.mock_handler_diagnostic.messages[0].message.split("\n") + messages_response = self.mock_handler_diagnostic.messages[1].message.split("\n") + assert "//dbs" in messages_request[0] + assert messages_request[1] == "Request method: 'POST'" + assert 'Request headers:' in messages_request[2] + assert messages_request[3] == 'A body is sent with the request' + assert messages_response[0] == 'Response status: 409' + assert messages_response[1] == 'Response status reason: Conflict' + assert "Elapsed Time:" in messages_response[2] + assert "Response headers" in messages_response[3] + + # delete database + self.client_diagnostic.delete_database("DBTEST") + + self.mock_handler_diagnostic.reset() + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From cfb6768e2e67019eb6e351af6db4bcee38d3577a Mon Sep 17 00:00:00 2001 From: bambriz Date: Wed, 31 Aug 2022 15:40:27 -0700 Subject: [PATCH 9/9] small fixes --- .../azure-cosmos/azure/cosmos/_cosmos_client_connection.py | 1 - sdk/cosmos/azure-cosmos/azure/cosmos/database.py | 1 - sdk/cosmos/azure-cosmos/azure/cosmos/user.py | 1 - 3 files changed, 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py index f14fc9bba96a..ed4d811ec9ad 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py @@ -59,7 +59,6 @@ from . import _utils from .partition_key import _Undefined, _Empty from ._auth_policy import CosmosBearerTokenCredentialPolicy -from .cosmos_diagnostics import CosmosDiagnostics from ._cosmos_http_logging_policy import CosmosHttpLoggingPolicy ClassType = TypeVar("ClassType") diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 43878309a78f..8ec2962aff5f 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -35,7 +35,6 @@ from .exceptions import CosmosResourceNotFoundError from .user import UserProxy from .documents import IndexingMode -from .cosmos_diagnostics import CosmosDiagnostics __all__ = ("DatabaseProxy",) diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py index d11775fd8986..e04fb8c26e91 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/user.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/user.py @@ -31,7 +31,6 @@ from ._cosmos_client_connection import CosmosClientConnection from ._base import build_options from .permission import Permission -from .cosmos_diagnostics import CosmosDiagnostics class UserProxy(object):