From f832b243d68a097d83d998c5c166ffe83d587644 Mon Sep 17 00:00:00 2001 From: reibs Date: Thu, 31 Aug 2023 23:57:35 -0700 Subject: [PATCH 1/5] enums --- agentops/agentops.py | 2 +- agentops/event.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/agentops/agentops.py b/agentops/agentops.py index f0d39d25d..a83138361 100644 --- a/agentops/agentops.py +++ b/agentops/agentops.py @@ -63,7 +63,7 @@ def wrapper(*args, **kwargs): arg_names = list(func_args.keys()) arg_values = dict(zip(arg_names, args)) arg_values.update(kwargs) - + # TODO: Rename output to returns try: output = func(*args, **kwargs) diff --git a/agentops/event.py b/agentops/event.py index 135d5d2c6..d08bf3602 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -1,6 +1,3 @@ -from .helpers import get_ISO_time -from typing import Optional, Dict, Literal - """ AgentOps events. @@ -8,6 +5,10 @@ Event: Represents discrete events to be recorded. Session: Represents a session of events, with a start and end state. """ +from .helpers import get_ISO_time +from typing import Optional, Dict, Literal + +Result = Literal["Success", "Fail", "Indeterminate"] class Event: @@ -29,7 +30,7 @@ class Event: def __init__(self, event_type: str, params: Optional[str] = None, output: Optional[str] = None, - result: Optional[str] = None, + result: Result = "Indeterminate", tags: Optional[Dict[str, str]] = None ): self.event_type = event_type @@ -61,7 +62,7 @@ def __init__(self, session_id: str, tags: Optional[Dict[str, str]] = None): self.init_timestamp = get_ISO_time() self.tags = tags - def end_session(self, end_state: Optional[str], rating: Optional[str] = None): + def end_session(self, end_state: Result = "Indeterminate", rating: Optional[str] = None): """ End the session with a specified state and rating. From eb298c200029821b363d575a5232c2c8932bbe89 Mon Sep 17 00:00:00 2001 From: reibs Date: Fri, 1 Sep 2023 11:57:32 -0700 Subject: [PATCH 2/5] renamed output to return --- agentops/agentops.py | 9 ++++----- agentops/event.py | 6 +++--- agentops/logger.py | 15 ++++++++------- tests/test_logger.py | 8 +++----- tests/test_session.py | 4 ++-- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/agentops/agentops.py b/agentops/agentops.py index a83138361..1620e2b11 100644 --- a/agentops/agentops.py +++ b/agentops/agentops.py @@ -9,7 +9,6 @@ from .event import Session, Event from .worker import Worker from uuid import uuid4 -import json from typing import Optional, Dict import functools import inspect @@ -65,12 +64,12 @@ def wrapper(*args, **kwargs): arg_values.update(kwargs) # TODO: Rename output to returns try: - output = func(*args, **kwargs) + returns = func(*args, **kwargs) # Record the event after the function call self.record(Event(event_type=event_name, params=arg_values, - output=output, + returns=returns, result="SUCCESS", tags=tags)) @@ -78,14 +77,14 @@ def wrapper(*args, **kwargs): # Record the event after the function call self.record(Event(event_type=event_name, params=arg_values, - output=None, + returns=None, result='FAIL', tags=tags)) # Re-raise the exception raise - return output + return returns return wrapper diff --git a/agentops/event.py b/agentops/event.py index d08bf3602..dd129fa4d 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -18,7 +18,7 @@ class Event: Args: event_type (str): Type of the event, e.g., "API Call". Required. params (str, optional): The parameters passed to the operation. - output (str, optional): The output of the operation. + returns (str, optional): The output of the operation. result (str, optional): Result of the operation, e.g., "success", "fail", "indeterminate". tags (Dict[str, str], optional): Tags that can be used for grouping or sorting later. e.g. {"llm": "GPT-4"}. @@ -29,13 +29,13 @@ class Event: def __init__(self, event_type: str, params: Optional[str] = None, - output: Optional[str] = None, + returns: Optional[str] = None, result: Result = "Indeterminate", tags: Optional[Dict[str, str]] = None ): self.event_type = event_type self.params = params - self.output = output + self.returns = returns self.result = result self.tags = tags self.timestamp = get_ISO_time() diff --git a/agentops/logger.py b/agentops/logger.py index 43f9fae1a..4f856f8be 100755 --- a/agentops/logger.py +++ b/agentops/logger.py @@ -3,24 +3,25 @@ from .agentops import AgentOps from .event import Event + class AgentOpsLogger(): """ A utility class for creating loggers and handlers configured to work with the AgentOps service. - + This class provides two static methods for creating a logger or a handler that sends log records to the AgentOps service. The logger and handler are configured with a specific AgentOps client and name. - + Example Usage: - + >>> from agentops import AgentOps >>> client = AgentOps(...) >>> logger = AgentOpsLogger.get_agentops_logger(client, 'my_logger') >>> logger.info('This is an info log') - + This will send an 'info' log to the AgentOps service. """ - + @staticmethod def get_agentops_logger(client: AgentOps, name: str, level=logging.DEBUG): """ @@ -40,7 +41,7 @@ def get_agentops_logger(client: AgentOps, name: str, level=logging.DEBUG): handler.setLevel(level) logger.addHandler(handler) return logger - + @staticmethod def get_agentops_handler(client: AgentOps, name: str): """ @@ -108,4 +109,4 @@ def emit(self, record): result = "indeterminate" self.client.record( - Event(f'{self.name}:{record.levelname}', output=log_entry, result=result)) + Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result)) diff --git a/tests/test_logger.py b/tests/test_logger.py index 0b04b6906..016ede5e3 100755 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -20,7 +20,6 @@ def setup_method(self): self.config = Configuration(max_wait_time=5) self.client = AgentOps(api_key=self.api_key, config=self.config) - def teardown_method(self): self.client.end_session(end_state="success") @@ -37,7 +36,6 @@ def test_info(self, mock_req): except Exception as e: pytest.fail(f"test_info failed with {e}") - time.sleep(0.1) # Assert @@ -45,7 +43,7 @@ def test_info(self, mock_req): assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == f"{self.event_type}:INFO" - assert request_json['events'][0]['output'] == test_message + assert request_json['events'][0]['returns'] == test_message def test_error(self, mock_req): # Arrange @@ -67,7 +65,7 @@ def test_error(self, mock_req): assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == f"{self.event_type}:ERROR" - assert request_json['events'][0]['output'] == test_message + assert request_json['events'][0]['returns'] == test_message def test_warn(self, mock_req): # Arrange @@ -89,4 +87,4 @@ def test_warn(self, mock_req): assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == f"{self.event_type}:WARNING" - assert request_json['events'][0]['output'] == test_message + assert request_json['events'][0]['returns'] == test_message diff --git a/tests/test_session.py b/tests/test_session.py index eaa9ff191..3675844cf 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -107,7 +107,7 @@ def dummy_func(x, y): assert request_json['event']['event_type'] == self.event_type assert request_json['event']['params'] == { 'args': [3, 4], 'kwargs': {}} - assert request_json['event']['output'] == 7 + assert request_json['event']['returns'] == 7 assert request_json['event']['result'] == 'SUCCESS' assert request_json['event']['tags'] == {'foo': 'bar'} @@ -127,6 +127,6 @@ def dummy_func(x, y): request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == self.event_type assert request_json['events'][0]['params'] == {'x': 3, 'y': 4} - assert request_json['events'][0]['output'] == 7 + assert request_json['events'][0]['returns'] == 7 assert request_json['events'][0]['result'] == 'SUCCESS' assert request_json['events'][0]['tags'] == {'foo': 'bar'} From 1c3e2530fdba205dddae27cc34f76d44b57a739d Mon Sep 17 00:00:00 2001 From: reibs Date: Fri, 1 Sep 2023 12:37:03 -0700 Subject: [PATCH 3/5] fixed tests --- tests/test_session.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_session.py b/tests/test_session.py index 3675844cf..ecb00c197 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -27,25 +27,26 @@ def test_session(self, mock_req): # Act client.record(Event(self.event_type)) - # Assert - assert len(mock_req.request_history) == 0 + # Assert the session has been initiated and the id has been created on backend. + assert len(mock_req.request_history) == 1 # Act client.record(Event(self.event_type)) time.sleep(0.1) - # Assert - assert len(mock_req.request_history) == 1 + # Assert an event has been added + assert len(mock_req.request_history) == 2 assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == self.event_type # Act - end_state = "Succeed" + end_state = "Success" client.end_session(end_state) time.sleep(0.1) - assert len(mock_req.request_history) == 2 + # Since a session has ended, no more events should be recorded, but end_session should be called + assert len(mock_req.request_history) == 3 assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['session']['rating'] == None @@ -62,18 +63,19 @@ def test_tags(self, mock_req): client.record(Event(self.event_type)) time.sleep(0.1) - # Assert - assert len(mock_req.request_history) == 1 + # Assert 2 requests - 1 for session init, 1 for event + assert len(mock_req.request_history) == 2 assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['events'][0]['event_type'] == self.event_type # Act - end_state = "Succeed" + end_state = "Success" client.end_session() time.sleep(0.1) - assert len(mock_req.request_history) == 2 + # Assert 3 requets, 1 for session init, 1 for event, 1 for end session + assert len(mock_req.request_history) == 3 assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['session']['rating'] == None From 72da3f208cc02d5eed04410b16473271a2b2c5af Mon Sep 17 00:00:00 2001 From: reibs Date: Fri, 1 Sep 2023 13:49:29 -0700 Subject: [PATCH 4/5] added enum to avoid ambiguity on dash --- agentops/agentops.py | 14 +++++++++----- agentops/event.py | 25 ++++++++++++++++++++----- agentops/logger.py | 6 +++--- tests/test_canary.py | 4 +++- tests/test_logger.py | 3 ++- tests/test_session.py | 15 ++++++++------- 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/agentops/agentops.py b/agentops/agentops.py index 1620e2b11..cc194aed0 100644 --- a/agentops/agentops.py +++ b/agentops/agentops.py @@ -6,7 +6,7 @@ """ from .config import Configuration -from .event import Session, Event +from .event import Session, Event, EventState from .worker import Worker from uuid import uuid4 from typing import Optional, Dict @@ -62,7 +62,6 @@ def wrapper(*args, **kwargs): arg_names = list(func_args.keys()) arg_values = dict(zip(arg_names, args)) arg_values.update(kwargs) - # TODO: Rename output to returns try: returns = func(*args, **kwargs) @@ -70,7 +69,7 @@ def wrapper(*args, **kwargs): self.record(Event(event_type=event_name, params=arg_values, returns=returns, - result="SUCCESS", + result="Success", tags=tags)) except Exception as e: @@ -78,7 +77,7 @@ def wrapper(*args, **kwargs): self.record(Event(event_type=event_name, params=arg_values, returns=None, - result='FAIL', + result='Fail', tags=tags)) # Re-raise the exception @@ -101,7 +100,7 @@ def start_session(self, tags: Optional[Dict[str, str]] = None): self.worker = Worker(self.config) self.worker.start_session(self.session) - def end_session(self, end_state: Optional[str] = None, rating: Optional[str] = None): + def end_session(self, end_state: EventState = EventState.INDETERMINATE, rating: Optional[str] = None): """ End the current session with the AgentOps service. @@ -109,5 +108,10 @@ def end_session(self, end_state: Optional[str] = None, rating: Optional[str] = N end_state (str, optional): The final state of the session. rating (str, optional): The rating for the session. """ + valid_results = set(vars(EventState).values()) + if end_state not in valid_results: + raise ValueError( + f"end_state must be one of {EventState.__args__}. Provided: {end_state}") + self.session.end_session(end_state, rating) self.worker.end_session(self.session) diff --git a/agentops/event.py b/agentops/event.py index dd129fa4d..77de1f092 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -6,9 +6,20 @@ Session: Represents a session of events, with a start and end state. """ from .helpers import get_ISO_time -from typing import Optional, Dict, Literal +from typing import Optional, Dict +from enum import Enum -Result = Literal["Success", "Fail", "Indeterminate"] + +class EventState: + SUCCESS = "Success" + FAIL = "Fail" + INDETERMINATE = "Indeterminate" + + +class SessionState: + SUCCESS = "Success" + FAIL = "Fail" + INDETERMINATE = "Indeterminate" class Event: @@ -19,7 +30,7 @@ class Event: event_type (str): Type of the event, e.g., "API Call". Required. params (str, optional): The parameters passed to the operation. returns (str, optional): The output of the operation. - result (str, optional): Result of the operation, e.g., "success", "fail", "indeterminate". + result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". tags (Dict[str, str], optional): Tags that can be used for grouping or sorting later. e.g. {"llm": "GPT-4"}. @@ -30,7 +41,7 @@ class Event: def __init__(self, event_type: str, params: Optional[str] = None, returns: Optional[str] = None, - result: Result = "Indeterminate", + result: EventState = EventState.INDETERMINATE, tags: Optional[Dict[str, str]] = None ): self.event_type = event_type @@ -62,7 +73,7 @@ def __init__(self, session_id: str, tags: Optional[Dict[str, str]] = None): self.init_timestamp = get_ISO_time() self.tags = tags - def end_session(self, end_state: Result = "Indeterminate", rating: Optional[str] = None): + def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None): """ End the session with a specified state and rating. @@ -70,6 +81,10 @@ def end_session(self, end_state: Result = "Indeterminate", rating: Optional[str] end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate" rating (str, optional): The rating for the session. """ + valid_results = set(vars(SessionState).values()) + if end_state not in valid_results: + raise ValueError( + f"end_state must be one of {valid_results}. Provided: {end_state}") self.end_state = end_state self.rating = rating self.end_timestamp = get_ISO_time() diff --git a/agentops/logger.py b/agentops/logger.py index 4f856f8be..2f71cbb56 100755 --- a/agentops/logger.py +++ b/agentops/logger.py @@ -1,7 +1,7 @@ import logging import re from .agentops import AgentOps -from .event import Event +from .event import Event, EventState class AgentOpsLogger(): @@ -104,9 +104,9 @@ def emit(self, record): log_entry = self.remove_color_codes(log_entry) if record.levelno == logging.ERROR: - result = "fail" + result = EventState.FAIL else: - result = "indeterminate" + result = EventState.INDETERMINATE self.client.record( Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result)) diff --git a/tests/test_canary.py b/tests/test_canary.py index 5f268ecae..a6fe44e00 100644 --- a/tests/test_canary.py +++ b/tests/test_canary.py @@ -3,6 +3,7 @@ import time from agentops import AgentOps, Event, Configuration +from agentops.event import EventState @pytest.fixture @@ -13,6 +14,7 @@ def mock_req(): m.post(url + '/sessions', text='ok') yield m + class TestCanary: def setup_method(self): self.url = 'https://agentops-server-v2.fly.dev' @@ -21,7 +23,7 @@ def setup_method(self): self.client = AgentOps(api_key=self.api_key, config=self.config) def teardown_method(self): - self.client.end_session(end_state="success") + self.client.end_session(end_state=EventState.SUCCESS) def test_agent_ops_record(self, mock_req): # Arrange diff --git a/tests/test_logger.py b/tests/test_logger.py index 016ede5e3..7b1c03554 100755 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -2,6 +2,7 @@ import requests_mock import time from agentops import AgentOps, AgentOpsLogger, Configuration +from agentops.event import EventState @pytest.fixture @@ -21,7 +22,7 @@ def setup_method(self): self.client = AgentOps(api_key=self.api_key, config=self.config) def teardown_method(self): - self.client.end_session(end_state="success") + self.client.end_session(end_state=EventState.SUCCESS) def test_info(self, mock_req): # Arrange diff --git a/tests/test_session.py b/tests/test_session.py index ecb00c197..b1a86e297 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -3,6 +3,7 @@ import time from agentops import AgentOps, Event, Configuration +from agentops.event import EventState @pytest.fixture @@ -41,7 +42,7 @@ def test_session(self, mock_req): assert request_json['events'][0]['event_type'] == self.event_type # Act - end_state = "Success" + end_state = EventState.SUCCESS client.end_session(end_state) time.sleep(0.1) @@ -70,8 +71,8 @@ def test_tags(self, mock_req): assert request_json['events'][0]['event_type'] == self.event_type # Act - end_state = "Success" - client.end_session() + end_state = EventState.SUCCESS + client.end_session(end_state) time.sleep(0.1) # Assert 3 requets, 1 for session init, 1 for event, 1 for end session @@ -79,7 +80,7 @@ def test_tags(self, mock_req): assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key request_json = mock_req.last_request.json() assert request_json['session']['rating'] == None - assert request_json['session']['end_state'] == None + assert request_json['session']['end_state'] == end_state assert request_json['session']['tags'] == tags @@ -92,7 +93,7 @@ def setup_method(self): self.client = AgentOps(self.api_key, config=self.config) def teardown_method(self): - self.client.end_session(end_state="success") + self.client.end_session(end_state=EventState.SUCCESS) def test_record_action_decorator(self, mock_req): @self.client.record_action(event_name=self.event_type, tags={'foo': 'bar'}) @@ -110,7 +111,7 @@ def dummy_func(x, y): assert request_json['event']['params'] == { 'args': [3, 4], 'kwargs': {}} assert request_json['event']['returns'] == 7 - assert request_json['event']['result'] == 'SUCCESS' + assert request_json['event']['result'] == EventState.SUCCESS assert request_json['event']['tags'] == {'foo': 'bar'} def test_record_action_decorator(self, mock_req): @@ -130,5 +131,5 @@ def dummy_func(x, y): assert request_json['events'][0]['event_type'] == self.event_type assert request_json['events'][0]['params'] == {'x': 3, 'y': 4} assert request_json['events'][0]['returns'] == 7 - assert request_json['events'][0]['result'] == 'SUCCESS' + assert request_json['events'][0]['result'] == EventState.SUCCESS assert request_json['events'][0]['tags'] == {'foo': 'bar'} From 03d26a80c0eb070105b824112188a26ae432c8a9 Mon Sep 17 00:00:00 2001 From: reibs Date: Fri, 1 Sep 2023 13:51:28 -0700 Subject: [PATCH 5/5] remove enums --- agentops/event.py | 1 - agentops/http.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/agentops/event.py b/agentops/event.py index 77de1f092..bc4cd6237 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -7,7 +7,6 @@ """ from .helpers import get_ISO_time from typing import Optional, Dict -from enum import Enum class EventState: diff --git a/agentops/http.py b/agentops/http.py index 70cf0ed83..c8abda753 100644 --- a/agentops/http.py +++ b/agentops/http.py @@ -69,7 +69,7 @@ def post(url: str, payload: bytes, api_key: str = None, header=None) -> Response JSON_HEADER["X-Agentops-Auth"] = api_key res = request_session.post(url, data=payload, - headers=JSON_HEADER, timeout=20) + headers=JSON_HEADER, timeout=20) result.parse(res) except requests.exceptions.Timeout: