From 3d5257592b440fca8c46b77471e1c34c901ee57f Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Sat, 20 Apr 2024 08:20:13 -0700 Subject: [PATCH] AgentOps Implementation (#411) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implements agentops with a langchain handler, agent tracking and tool call recording * track tool usage * end session after completion * track tool usage time * better tool and llm tracking * code cleanup * make agentops optional * optional dependency usage * remove telemetry code * optional agentops * agentops version bump * remove org key * true dependency * add crew org key to agentops * cleanup * Update pyproject.toml * Revert "true dependency" This reverts commit e52e8e9568c2c00f29c1f800ea00d472413b6067. * Revert "cleanup" This reverts commit 7f5635fb9ef9aaafc4283fdf1a892d8b5c425220. * optional parent key * agentops 0.1.5 * Revert "Revert "cleanup"" This reverts commit cea33d9a5d37289b9e257046c1354057b27fbe7f. * Revert "Revert "true dependency"" This reverts commit 4d1b460b * cleanup * Forcing version 0.1.5 * Update pyproject.toml --------- Co-authored-by: João Moura --- pyproject.toml | 1 + src/crewai/agent.py | 5 +++++ src/crewai/crew.py | 3 +++ src/crewai/tools/tool_usage.py | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 01436da07..179c48293 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ click = "^8.1.7" python-dotenv = "^1.0.0" embedchain = "^0.1.98" appdirs = "^1.4.4" +agentops = "0.1.6" [tool.poetry.extras] tools = ["crewai-tools"] diff --git a/src/crewai/agent.py b/src/crewai/agent.py index 12bc785ce..ac080f2d0 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -24,8 +24,10 @@ from crewai.agents import CacheHandler, CrewAgentExecutor, CrewAgentParser, Tool from crewai.memory.contextual.contextual_memory import ContextualMemory from crewai.utilities import I18N, Logger, Prompts, RPMController from crewai.utilities.token_counter_callback import TokenCalcHandler, TokenProcess +from agentops.agent import track_agent +@track_agent() class Agent(BaseModel): """Represents an agent in a system. @@ -55,6 +57,8 @@ class Agent(BaseModel): _rpm_controller: RPMController = PrivateAttr(default=None) _request_within_rpm_limit: Any = PrivateAttr(default=None) _token_process: TokenProcess = TokenProcess() + agent_ops_agent_name: str = None + agent_ops_agent_id: str = None formatting_errors: int = 0 model_config = ConfigDict(arbitrary_types_allowed=True) @@ -129,6 +133,7 @@ class Agent(BaseModel): def __init__(__pydantic_self__, **data): config = data.pop("config", {}) super().__init__(**config, **data) + __pydantic_self__.agent_ops_agent_name = __pydantic_self__.role @field_validator("id", mode="before") @classmethod diff --git a/src/crewai/crew.py b/src/crewai/crew.py index 0b88d5e4b..6ab6e61c9 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -26,6 +26,7 @@ from crewai.task import Task from crewai.telemetry import Telemetry from crewai.tools.agent_tools import AgentTools from crewai.utilities import I18N, FileHandler, Logger, RPMController +import agentops class Crew(BaseModel): @@ -239,6 +240,7 @@ class Crew(BaseModel): self._set_tasks_callbacks() i18n = I18N(language=self.language, language_file=self.language_file) + agentops.set_parent_key("daebe730-f54d-4af5-98df-e6946fb76d13") for agent in self.agents: agent.i18n = i18n @@ -374,6 +376,7 @@ class Crew(BaseModel): def _finish_execution(self, output) -> None: if self.max_rpm: self._rpm_controller.stop_rpm_counter() + agentops.end_session(end_state="Success", end_state_reason="Finished Execution") self._telemetry.end_crew(self, output) def __repr__(self): diff --git a/src/crewai/tools/tool_usage.py b/src/crewai/tools/tool_usage.py index 954572328..b11f4ec2a 100644 --- a/src/crewai/tools/tool_usage.py +++ b/src/crewai/tools/tool_usage.py @@ -9,6 +9,7 @@ from crewai.agents.tools_handler import ToolsHandler from crewai.telemetry import Telemetry from crewai.tools.tool_calling import InstructorToolCalling, ToolCalling from crewai.utilities import I18N, Converter, ConverterError, Printer +import agentops OPENAI_BIGGER_MODELS = ["gpt-4"] @@ -96,6 +97,7 @@ class ToolUsage: tool: BaseTool, calling: Union[ToolCalling, InstructorToolCalling], ) -> None: + tool_event = agentops.ToolEvent(name=calling.tool_name) if self._check_tool_repeated_usage(calling=calling): try: result = self._i18n.errors("task_repeated_usage").format( @@ -159,6 +161,7 @@ class ToolUsage: self._printer.print(content=f"\n\n{error_message}\n", color="red") return error self.task.increment_tools_errors() + agentops.record(agentops.ErrorEvent(details=e, trigger_event=tool_event)) return self.use(calling=calling, tool_string=tool_string) if self.tools_handler: @@ -179,6 +182,7 @@ class ToolUsage: ) self._printer.print(content=f"\n\n{result}\n", color="purple") + agentops.record(tool_event) self._telemetry.tool_usage( llm=self.function_calling_llm, tool_name=tool.name,