From f47904134b6bfba750f32151dd0a499229816851 Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Tue, 2 Jul 2024 17:52:15 -0700 Subject: [PATCH] Add back AgentOps as Optional Dependency (#543) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implements agentops with a langchain handler, agent tracking and tool call recording * track tool usage * end session after completion * track tool usage time * better tool and llm tracking * code cleanup * make agentops optional * optional dependency usage * remove telemetry code * optional agentops * agentops version bump * remove org key * true dependency * add crew org key to agentops * cleanup * Update pyproject.toml * Revert "true dependency" This reverts commit e52e8e9568c2c00f29c1f800ea00d472413b6067. * Revert "cleanup" This reverts commit 7f5635fb9ef9aaafc4283fdf1a892d8b5c425220. * optional parent key * agentops 0.1.5 * Revert "Revert "cleanup"" This reverts commit cea33d9a5d37289b9e257046c1354057b27fbe7f. * Revert "Revert "true dependency"" This reverts commit 4d1b460b * cleanup * Forcing version 0.1.5 * Update pyproject.toml * agentops update * noop * add crew tag * black formatting * use langchain callback handler to support all LLMs * agentops version bump * track task evaluator * merge upstream * Fix typo in instruction en.json (#676) * Enable search in docs (#663) * Clarify text in docstring (#662) * Update agent.py (#655) Changed default model value from gpt-4 to gpt-4o. Reasoning. gpt-4 costs 30$ per million tokens while gpt-4o costs 5$. This is more cost friendly for default option. * Update README.md (#652) Rework example so that if you use a custom LLM it doesn't throw code errors by uncommenting. * Update BrowserbaseLoadTool.md (#647) * Update crew.py (#644) Fixed Type on line 53 * fixes #665 (#666) * Added timestamp to logger (#646) * Added timestamp to logger Updated the logger.py file to include timestamps when logging output. For example: [2024-05-20 15:32:48][DEBUG]: == Working Agent: Researcher [2024-05-20 15:32:48][INFO]: == Starting Task: Research the topic [2024-05-20 15:33:22][DEBUG]: == [Researcher] Task output: * Update tool_usage.py * Revert "Update tool_usage.py" This reverts commit 95d18d5b6f9bc4b918ce693d2780f6e464d99547. incorrect bramch for this commit * support skip auto end session * conditional protect agentops use * fix crew logger bug * fix crew logger bug * Update crew.py * Update tool_usage.py --------- Co-authored-by: João Moura Co-authored-by: Howard Gil Co-authored-by: Olivier Roberdet Co-authored-by: Paul Sanders Co-authored-by: Anudeep Kolluri <50168940+Anudeep-Kolluri@users.noreply.github.com> Co-authored-by: Mike Heavers Co-authored-by: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Co-authored-by: theCyberTech - Rip&Tear <84775494+theCyberTech@users.noreply.github.com> Co-authored-by: Saif Mahmud <60409889+vmsaif@users.noreply.github.com> --- pyproject.toml | 4 ++- src/crewai/agent.py | 22 +++++++++++++ src/crewai/crew.py | 9 +++++ src/crewai/tools/tool_usage.py | 33 +++++++++++++------ .../utilities/evaluators/task_evaluator.py | 12 +++++++ src/crewai/utilities/logger.py | 2 +- 6 files changed, 70 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ce2bbede7..17c6f526e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,12 @@ click = "^8.1.7" python-dotenv = "^1.0.0" appdirs = "^1.4.4" jsonref = "^1.1.0" +agentops = { version = "^0.1.9", optional = true } embedchain = "^0.1.113" [tool.poetry.extras] tools = ["crewai-tools"] +agentops = ["agentops"] [tool.poetry.group.dev.dependencies] isort = "^5.13.2" @@ -60,4 +62,4 @@ exclude = ["cli/templates/main.py", "cli/templates/crew.py"] [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/src/crewai/agent.py b/src/crewai/agent.py index 67e81f4d0..501d538be 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -19,7 +19,20 @@ from crewai.utilities.token_counter_callback import TokenCalcHandler from crewai.agents.agent_builder.base_agent import BaseAgent from crewai.utilities.training_handler import CrewTrainingHandler +agentops = None +try: + import agentops + from agentops import track_agent +except ImportError: + def track_agent(): + def noop(f): + return f + + return noop + + +@track_agent() class Agent(BaseAgent): """Represents an agent in a system. @@ -48,6 +61,8 @@ class Agent(BaseAgent): default=None, description="Maximum execution time for an agent to execute a task", ) + agent_ops_agent_name: str = None + agent_ops_agent_id: str = None cache_handler: InstanceOf[CacheHandler] = Field( default=None, description="An instance of the CacheHandler class." ) @@ -84,6 +99,7 @@ class Agent(BaseAgent): def __init__(__pydantic_self__, **data): config = data.pop("config", {}) super().__init__(**config, **data) + __pydantic_self__.agent_ops_agent_name = __pydantic_self__.role @model_validator(mode="after") def set_agent_executor(self) -> "Agent": @@ -101,6 +117,12 @@ class Agent(BaseAgent): ): self.llm.callbacks.append(token_handler) + if agentops and not any( + isinstance(handler, agentops.LangchainCallbackHandler) for handler in self.llm.callbacks + ): + agentops.stop_instrumenting() + self.llm.callbacks.append(agentops.LangchainCallbackHandler()) + if not self.agent_executor: if not self.cache_handler: self.cache_handler = CacheHandler() diff --git a/src/crewai/crew.py b/src/crewai/crew.py index e2f927616..636ead48d 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -31,6 +31,11 @@ from crewai.utilities import I18N, FileHandler, Logger, RPMController from crewai.utilities.evaluators.task_evaluator import TaskEvaluator from crewai.utilities.training_handler import CrewTrainingHandler +try: + import agentops +except ImportError: + agentops = None + class Crew(BaseModel): """ @@ -536,6 +541,10 @@ class Crew(BaseModel): def _finish_execution(self, output) -> None: if self.max_rpm: self._rpm_controller.stop_rpm_counter() + if agentops: + agentops.end_session( + end_state="Success", end_state_reason="Finished Execution", is_auto_end=True + ) self._telemetry.end_crew(self, output) def __repr__(self): diff --git a/src/crewai/tools/tool_usage.py b/src/crewai/tools/tool_usage.py index cb7757d84..9bcd1ce91 100644 --- a/src/crewai/tools/tool_usage.py +++ b/src/crewai/tools/tool_usage.py @@ -11,6 +11,12 @@ from crewai.telemetry import Telemetry from crewai.tools.tool_calling import InstructorToolCalling, ToolCalling from crewai.utilities import I18N, Converter, ConverterError, Printer +agentops = None +try: + import agentops +except ImportError: + pass + OPENAI_BIGGER_MODELS = ["gpt-4"] @@ -91,15 +97,16 @@ class ToolUsage: self.task.increment_tools_errors() self._printer.print(content=f"\n\n{error}\n", color="red") return error - return f"{self._use(tool_string=tool_string, tool=tool, calling=calling)}" # type: ignore # BUG?: "_use" of "ToolUsage" does not return a value (it only ever returns None) + return f"{self._use(tool_string=tool_string, tool=tool, calling=calling)}" # type: ignore # BUG?: "_use" of "ToolUsage" does not return a value (it only ever returns None) def _use( self, tool_string: str, tool: BaseTool, calling: Union[ToolCalling, InstructorToolCalling], - ) -> str: # TODO: Fix this return type --> finecwg : I updated return type to str - if self._check_tool_repeated_usage(calling=calling): # type: ignore # _check_tool_repeated_usage of "ToolUsage" does not return a value (it only ever returns None) + ) -> str: # TODO: Fix this return type + tool_event = agentops.ToolEvent(name=calling.tool_name) if agentops else None + if self._check_tool_repeated_usage(calling=calling): # type: ignore # _check_tool_repeated_usage of "ToolUsage" does not return a value (it only ever returns None) try: result = self._i18n.errors("task_repeated_usage").format( tool_names=self.tools_names @@ -110,13 +117,13 @@ class ToolUsage: tool_name=tool.name, attempts=self._run_attempts, ) - result = self._format_result(result=result) # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None) + result = self._format_result(result=result) # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None) return result # type: ignore # Fix the reutrn type of this function except Exception: self.task.increment_tools_errors() - result = None # type: ignore # Incompatible types in assignment (expression has type "None", variable has type "str") + result = None # type: ignore # Incompatible types in assignment (expression has type "None", variable has type "str") if self.tools_handler.cache: result = self.tools_handler.cache.read( # type: ignore # Incompatible types in assignment (expression has type "str | None", variable has type "str") @@ -133,7 +140,7 @@ class ToolUsage: if calling.arguments: try: - acceptable_args = tool.args_schema.schema()["properties"].keys() # type: ignore # Item "None" of "type[BaseModel] | None" has no attribute "schema" + acceptable_args = tool.args_schema.schema()["properties"].keys() # type: ignore # Item "None" of "type[BaseModel] | None" has no attribute "schema" arguments = { k: v for k, v in calling.arguments.items() @@ -145,7 +152,7 @@ class ToolUsage: arguments = calling.arguments result = tool._run(**arguments) else: - arguments = calling.arguments.values() # type: ignore # Incompatible types in assignment (expression has type "dict_values[str, Any]", variable has type "dict[str, Any]") + arguments = calling.arguments.values() # type: ignore # Incompatible types in assignment (expression has type "dict_values[str, Any]", variable has type "dict[str, Any]") result = tool._run(*arguments) else: result = tool._run() @@ -164,6 +171,10 @@ class ToolUsage: return error # type: ignore # No return value expected self.task.increment_tools_errors() + if agentops: + agentops.record( + agentops.ErrorEvent(exception=e, trigger_event=tool_event) + ) return self.use(calling=calling, tool_string=tool_string) # type: ignore # No return value expected if self.tools_handler: @@ -184,18 +195,20 @@ class ToolUsage: ) self._printer.print(content=f"\n\n{result}\n", color="purple") + if agentops: + agentops.record(tool_event) self._telemetry.tool_usage( llm=self.function_calling_llm, tool_name=tool.name, attempts=self._run_attempts, - ) - result = self._format_result(result=result) # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None) + ) + result = self._format_result(result=result) # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None) return result # type: ignore # No return value expected def _format_result(self, result: Any) -> None: self.task.used_tools += 1 if self._should_remember_format(): # type: ignore # "_should_remember_format" of "ToolUsage" does not return a value (it only ever returns None) - result = self._remember_format(result=result) # type: ignore # "_remember_format" of "ToolUsage" does not return a value (it only ever returns None) + result = self._remember_format(result=result) # type: ignore # "_remember_format" of "ToolUsage" does not return a value (it only ever returns None) return result def _should_remember_format(self) -> None: diff --git a/src/crewai/utilities/evaluators/task_evaluator.py b/src/crewai/utilities/evaluators/task_evaluator.py index 299590ba5..252541b66 100644 --- a/src/crewai/utilities/evaluators/task_evaluator.py +++ b/src/crewai/utilities/evaluators/task_evaluator.py @@ -5,6 +5,17 @@ from pydantic import BaseModel, Field from crewai.utilities import Converter from crewai.utilities.pydantic_schema_parser import PydanticSchemaParser +agentops = None +try: + import agentops + from agentops import track_agent +except ImportError: + + def track_agent(name): + def noop(f): + return f + + return noop class Entity(BaseModel): @@ -38,6 +49,7 @@ class TrainingTaskEvaluation(BaseModel): ) +@track_agent(name="Task Evaluator") class TaskEvaluator: def __init__(self, original_agent): self.llm = original_agent.llm diff --git a/src/crewai/utilities/logger.py b/src/crewai/utilities/logger.py index e0c74d5ce..b54595c9c 100644 --- a/src/crewai/utilities/logger.py +++ b/src/crewai/utilities/logger.py @@ -1,7 +1,7 @@ from datetime import datetime from crewai.utilities.printer import Printer - +from datetime import datetime class Logger: _printer = Printer()