From 5afe3921d234dd65283f8a25b7251404f431607c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 17:47:12 +0000 Subject: [PATCH] feat: restore AgentOps documentation and integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add AgentOps optional dependency to pyproject.toml - Implement AgentOpsListener with event handling for crew kickoff, tool usage, and task evaluation - Add comprehensive AgentOps documentation in English, Portuguese, and Korean - Update docs.json navigation to include AgentOps in observability sections - Add agentops.log to .gitignore - Include comprehensive tests for AgentOps integration with mocking - Ensure graceful handling when AgentOps package is not installed Addresses issue #3348 by restoring AgentOps integration removed in PR #3334 Co-Authored-By: João --- .gitignore | 3 + docs/docs.json | 3 + docs/en/observability/agentops.mdx | 184 ++++++++++++++++ docs/ko/observability/agentops.mdx | 131 ++++++++++++ docs/pt-BR/observability/agentops.mdx | 131 ++++++++++++ pyproject.toml | 1 + src/crewai/utilities/events/__init__.py | 2 + .../utilities/events/third_party/__init__.py | 1 + .../events/third_party/agentops_listener.py | 134 ++++++++++++ .../utilities/events/third_party/__init__.py | 199 ++++++++++++++++++ .../third_party/test_agentops_listener.py | 196 +++++++++++++++++ 11 files changed, 985 insertions(+) create mode 100644 docs/en/observability/agentops.mdx create mode 100644 docs/ko/observability/agentops.mdx create mode 100644 docs/pt-BR/observability/agentops.mdx create mode 100644 src/crewai/utilities/events/third_party/agentops_listener.py create mode 100644 tests/utilities/events/third_party/__init__.py create mode 100644 tests/utilities/events/third_party/test_agentops_listener.py diff --git a/.gitignore b/.gitignore index 1e4e7bf6c..cd0228ac5 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ plan.md conceptual_plan.md build_image chromadb-*.lock + +# AgentOps +agentops.log diff --git a/docs/docs.json b/docs/docs.json index 907c9d174..4c9ed481a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -226,6 +226,7 @@ "group": "Observability", "pages": [ "en/observability/overview", + "en/observability/agentops", "en/observability/arize-phoenix", "en/observability/langdb", "en/observability/langfuse", @@ -565,6 +566,7 @@ "group": "Observabilidade", "pages": [ "pt-BR/observability/overview", + "pt-BR/observability/agentops", "pt-BR/observability/arize-phoenix", "pt-BR/observability/langdb", "pt-BR/observability/langfuse", @@ -912,6 +914,7 @@ "group": "Observability", "pages": [ "ko/observability/overview", + "ko/observability/agentops", "ko/observability/arize-phoenix", "ko/observability/langdb", "ko/observability/langfuse", diff --git a/docs/en/observability/agentops.mdx b/docs/en/observability/agentops.mdx new file mode 100644 index 000000000..f7eb4c0cd --- /dev/null +++ b/docs/en/observability/agentops.mdx @@ -0,0 +1,184 @@ +--- +title: "AgentOps Integration" +description: "Monitor and analyze your CrewAI agents with AgentOps observability platform" +--- + +# AgentOps Integration + +AgentOps is a powerful observability platform designed specifically for AI agents. It provides comprehensive monitoring, analytics, and debugging capabilities for your CrewAI crews. + +## Features + +- **Real-time Monitoring**: Track agent performance and behavior in real-time +- **Session Replay**: Review complete agent sessions with detailed execution traces +- **Performance Analytics**: Analyze crew efficiency, tool usage, and task completion rates +- **Error Tracking**: Identify and debug issues in agent workflows +- **Cost Tracking**: Monitor LLM usage and associated costs +- **Team Collaboration**: Share insights and collaborate on agent optimization + +## Installation + +Install AgentOps alongside CrewAI: + +```bash +pip install crewai[agentops] +``` + +Or install AgentOps separately: + +```bash +pip install agentops +``` + +## Setup + +1. **Get your API Key**: Sign up at [AgentOps](https://agentops.ai) and get your API key + +2. **Configure your environment**: Set your AgentOps API key as an environment variable: + +```bash +export AGENTOPS_API_KEY="your-api-key-here" +``` + +3. **Initialize AgentOps**: Add this to your CrewAI script: + +```python +import agentops +from crewai import Agent, Task, Crew + +# Initialize AgentOps +agentops.init() + +# Your CrewAI code here +agent = Agent( + role="Data Analyst", + goal="Analyze data and provide insights", + backstory="You are an expert data analyst...", +) + +task = Task( + description="Analyze the sales data and provide insights", + agent=agent, +) + +crew = Crew( + agents=[agent], + tasks=[task], +) + +# Run your crew +result = crew.kickoff() + +# End the AgentOps session +agentops.end_session("Success") +``` + +## Automatic Integration + +CrewAI automatically integrates with AgentOps when the library is installed. The integration captures: + +- **Crew Kickoff Events**: Start and completion of crew executions +- **Tool Usage**: All tool calls and their results +- **Task Evaluations**: Task performance metrics and feedback +- **Error Events**: Any errors that occur during execution + +## Configuration Options + +You can customize the AgentOps integration: + +```python +import agentops + +# Configure AgentOps with custom settings +agentops.init( + api_key="your-api-key", + tags=["production", "data-analysis"], + auto_start_session=True, + instrument_llm_calls=True, +) +``` + +## Viewing Your Data + +1. **Dashboard**: Visit the AgentOps dashboard to view your agent sessions +2. **Session Details**: Click on any session to see detailed execution traces +3. **Analytics**: Use the analytics tab to identify performance trends +4. **Errors**: Monitor the errors tab for debugging information + +## Best Practices + +- **Tag Your Sessions**: Use meaningful tags to organize your agent runs +- **Monitor Costs**: Keep track of LLM usage and associated costs +- **Review Errors**: Regularly check for and address any errors +- **Optimize Performance**: Use analytics to identify bottlenecks and optimization opportunities + +## Troubleshooting + +### AgentOps Not Recording Data + +1. Verify your API key is set correctly +2. Check that AgentOps is properly initialized +3. Ensure you're calling `agentops.end_session()` at the end of your script + +### Missing Events + +If some events aren't being captured: + +1. Make sure you have the latest version of both CrewAI and AgentOps +2. Check that the AgentOps listener is properly registered +3. Review the logs for any error messages + +## Example: Complete Integration + +```python +import os +import agentops +from crewai import Agent, Task, Crew, Process + +# Initialize AgentOps +agentops.init( + api_key=os.getenv("AGENTOPS_API_KEY"), + tags=["example", "tutorial"], +) + +# Define your agents +researcher = Agent( + role="Research Specialist", + goal="Conduct thorough research on given topics", + backstory="You are an expert researcher with access to various tools...", +) + +writer = Agent( + role="Content Writer", + goal="Create engaging content based on research", + backstory="You are a skilled writer who can transform research into compelling content...", +) + +# Define your tasks +research_task = Task( + description="Research the latest trends in AI and machine learning", + agent=researcher, +) + +writing_task = Task( + description="Write a blog post about AI trends based on the research", + agent=writer, +) + +# Create and run your crew +crew = Crew( + agents=[researcher, writer], + tasks=[research_task, writing_task], + process=Process.sequential, +) + +try: + result = crew.kickoff() + print(result) + agentops.end_session("Success") +except Exception as e: + print(f"Error: {e}") + agentops.end_session("Fail") +``` + +This integration provides comprehensive observability for your CrewAI agents, helping you monitor, debug, and optimize your AI workflows. diff --git a/docs/ko/observability/agentops.mdx b/docs/ko/observability/agentops.mdx new file mode 100644 index 000000000..796abb2aa --- /dev/null +++ b/docs/ko/observability/agentops.mdx @@ -0,0 +1,131 @@ +--- +title: "AgentOps 통합" +description: "AgentOps 관찰 가능성 플랫폼으로 CrewAI 에이전트를 모니터링하고 분석하세요" +--- + +# AgentOps 통합 + +AgentOps는 AI 에이전트를 위해 특별히 설계된 강력한 관찰 가능성 플랫폼입니다. CrewAI 크루를 위한 포괄적인 모니터링, 분석 및 디버깅 기능을 제공합니다. + +## 기능 + +- **실시간 모니터링**: 에이전트 성능과 동작을 실시간으로 추적 +- **세션 재생**: 상세한 실행 추적과 함께 완전한 에이전트 세션 검토 +- **성능 분석**: 크루 효율성, 도구 사용량 및 작업 완료율 분석 +- **오류 추적**: 에이전트 워크플로우의 문제 식별 및 디버그 +- **비용 추적**: LLM 사용량 및 관련 비용 모니터링 +- **팀 협업**: 인사이트 공유 및 에이전트 최적화 협업 + +## 설치 + +CrewAI와 함께 AgentOps 설치: + +```bash +pip install crewai[agentops] +``` + +또는 AgentOps를 별도로 설치: + +```bash +pip install agentops +``` + +## 설정 + +1. **API 키 받기**: [AgentOps](https://agentops.ai)에 가입하고 API 키를 받으세요 + +2. **환경 구성**: AgentOps API 키를 환경 변수로 설정: + +```bash +export AGENTOPS_API_KEY="여기에-api-키-입력" +``` + +3. **AgentOps 초기화**: CrewAI 스크립트에 다음을 추가: + +```python +import agentops +from crewai import Agent, Task, Crew + +# AgentOps 초기화 +agentops.init() + +# 여기에 CrewAI 코드 +agent = Agent( + role="데이터 분석가", + goal="데이터를 분석하고 인사이트 제공", + backstory="당신은 전문 데이터 분석가입니다...", +) + +task = Task( + description="판매 데이터를 분석하고 인사이트를 제공하세요", + agent=agent, +) + +crew = Crew( + agents=[agent], + tasks=[task], +) + +# 크루 실행 +result = crew.kickoff() + +# AgentOps 세션 종료 +agentops.end_session("Success") +``` + +## 자동 통합 + +CrewAI는 라이브러리가 설치되면 AgentOps와 자동으로 통합됩니다. 통합은 다음을 캡처합니다: + +- **크루 킥오프 이벤트**: 크루 실행의 시작과 완료 +- **도구 사용**: 모든 도구 호출과 결과 +- **작업 평가**: 작업 성능 메트릭과 피드백 +- **오류 이벤트**: 실행 중 발생하는 모든 오류 + +## 구성 옵션 + +AgentOps 통합을 사용자 정의할 수 있습니다: + +```python +import agentops + +# 사용자 정의 설정으로 AgentOps 구성 +agentops.init( + api_key="당신의-api-키", + tags=["프로덕션", "데이터-분석"], + auto_start_session=True, + instrument_llm_calls=True, +) +``` + +## 데이터 보기 + +1. **대시보드**: AgentOps 대시보드를 방문하여 에이전트 세션 보기 +2. **세션 세부사항**: 세션을 클릭하여 상세한 실행 추적 보기 +3. **분석**: 분석 탭을 사용하여 성능 트렌드 식별 +4. **오류**: 디버깅 정보를 위해 오류 탭 모니터링 + +## 모범 사례 + +- **세션 태그 지정**: 의미 있는 태그를 사용하여 에이전트 실행 정리 +- **비용 모니터링**: LLM 사용량과 관련 비용 추적 +- **오류 검토**: 정기적으로 오류 확인 및 해결 +- **성능 최적화**: 분석을 사용하여 병목 현상과 최적화 기회 식별 + +## 문제 해결 + +### AgentOps가 데이터를 기록하지 않음 + +1. API 키가 올바르게 설정되었는지 확인 +2. AgentOps가 제대로 초기화되었는지 확인 +3. 스크립트 끝에서 `agentops.end_session()`을 호출하는지 확인 + +### 누락된 이벤트 + +일부 이벤트가 캡처되지 않는 경우: + +1. CrewAI와 AgentOps의 최신 버전이 있는지 확인 +2. AgentOps 리스너가 제대로 등록되었는지 확인 +3. 오류 메시지에 대한 로그 검토 + +이 통합은 CrewAI 에이전트에 대한 포괄적인 관찰 가능성을 제공하여 AI 워크플로우를 모니터링, 디버그 및 최적화하는 데 도움이 됩니다. diff --git a/docs/pt-BR/observability/agentops.mdx b/docs/pt-BR/observability/agentops.mdx new file mode 100644 index 000000000..db7a2a04d --- /dev/null +++ b/docs/pt-BR/observability/agentops.mdx @@ -0,0 +1,131 @@ +--- +title: "Integração AgentOps" +description: "Monitore e analise seus agentes CrewAI com a plataforma de observabilidade AgentOps" +--- + +# Integração AgentOps + +AgentOps é uma poderosa plataforma de observabilidade projetada especificamente para agentes de IA. Ela fornece capacidades abrangentes de monitoramento, análise e depuração para suas crews CrewAI. + +## Recursos + +- **Monitoramento em Tempo Real**: Acompanhe o desempenho e comportamento dos agentes em tempo real +- **Replay de Sessão**: Revise sessões completas de agentes com rastreamentos detalhados de execução +- **Análise de Desempenho**: Analise eficiência da crew, uso de ferramentas e taxas de conclusão de tarefas +- **Rastreamento de Erros**: Identifique e depure problemas em fluxos de trabalho de agentes +- **Rastreamento de Custos**: Monitore o uso de LLM e custos associados +- **Colaboração em Equipe**: Compartilhe insights e colabore na otimização de agentes + +## Instalação + +Instale o AgentOps junto com o CrewAI: + +```bash +pip install crewai[agentops] +``` + +Ou instale o AgentOps separadamente: + +```bash +pip install agentops +``` + +## Configuração + +1. **Obtenha sua Chave API**: Cadastre-se no [AgentOps](https://agentops.ai) e obtenha sua chave API + +2. **Configure seu ambiente**: Defina sua chave API do AgentOps como uma variável de ambiente: + +```bash +export AGENTOPS_API_KEY="sua-chave-api-aqui" +``` + +3. **Inicialize o AgentOps**: Adicione isso ao seu script CrewAI: + +```python +import agentops +from crewai import Agent, Task, Crew + +# Inicializar AgentOps +agentops.init() + +# Seu código CrewAI aqui +agent = Agent( + role="Analista de Dados", + goal="Analisar dados e fornecer insights", + backstory="Você é um analista de dados especialista...", +) + +task = Task( + description="Analise os dados de vendas e forneça insights", + agent=agent, +) + +crew = Crew( + agents=[agent], + tasks=[task], +) + +# Execute sua crew +result = crew.kickoff() + +# Finalize a sessão AgentOps +agentops.end_session("Success") +``` + +## Integração Automática + +O CrewAI se integra automaticamente com o AgentOps quando a biblioteca está instalada. A integração captura: + +- **Eventos de Kickoff da Crew**: Início e conclusão de execuções da crew +- **Uso de Ferramentas**: Todas as chamadas de ferramentas e seus resultados +- **Avaliações de Tarefas**: Métricas de desempenho de tarefas e feedback +- **Eventos de Erro**: Quaisquer erros que ocorram durante a execução + +## Opções de Configuração + +Você pode personalizar a integração do AgentOps: + +```python +import agentops + +# Configure AgentOps com configurações personalizadas +agentops.init( + api_key="sua-chave-api", + tags=["producao", "analise-dados"], + auto_start_session=True, + instrument_llm_calls=True, +) +``` + +## Visualizando Seus Dados + +1. **Dashboard**: Visite o dashboard do AgentOps para ver suas sessões de agentes +2. **Detalhes da Sessão**: Clique em qualquer sessão para ver rastreamentos detalhados de execução +3. **Análises**: Use a aba de análises para identificar tendências de desempenho +4. **Erros**: Monitore a aba de erros para informações de depuração + +## Melhores Práticas + +- **Marque Suas Sessões**: Use tags significativas para organizar suas execuções de agentes +- **Monitore Custos**: Acompanhe o uso de LLM e custos associados +- **Revise Erros**: Verifique e resolva regularmente quaisquer erros +- **Otimize Desempenho**: Use análises para identificar gargalos e oportunidades de otimização + +## Solução de Problemas + +### AgentOps Não Está Gravando Dados + +1. Verifique se sua chave API está definida corretamente +2. Verifique se o AgentOps está inicializado adequadamente +3. Certifique-se de estar chamando `agentops.end_session()` no final do seu script + +### Eventos Ausentes + +Se alguns eventos não estão sendo capturados: + +1. Certifique-se de ter a versão mais recente do CrewAI e AgentOps +2. Verifique se o listener do AgentOps está registrado adequadamente +3. Revise os logs para quaisquer mensagens de erro + +Esta integração fornece observabilidade abrangente para seus agentes CrewAI, ajudando você a monitorar, depurar e otimizar seus fluxos de trabalho de IA. diff --git a/pyproject.toml b/pyproject.toml index 2c7c9770a..5e03c5479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ docling = [ aisuite = [ "aisuite>=0.1.10", ] +agentops = ["agentops==0.3.18"] [tool.uv] dev-dependencies = [ diff --git a/src/crewai/utilities/events/__init__.py b/src/crewai/utilities/events/__init__.py index b5269959b..641e0d820 100644 --- a/src/crewai/utilities/events/__init__.py +++ b/src/crewai/utilities/events/__init__.py @@ -67,6 +67,7 @@ from .memory_events import ( # events from .event_listener import EventListener +from .third_party.agentops_listener import agentops_listener __all__ = [ "EventListener", @@ -121,4 +122,5 @@ __all__ = [ "ToolSelectionErrorEvent", "ToolUsageEvent", "ToolValidateInputErrorEvent", + "agentops_listener", ] diff --git a/src/crewai/utilities/events/third_party/__init__.py b/src/crewai/utilities/events/third_party/__init__.py index e69de29bb..e9de52477 100644 --- a/src/crewai/utilities/events/third_party/__init__.py +++ b/src/crewai/utilities/events/third_party/__init__.py @@ -0,0 +1 @@ +from .agentops_listener import agentops_listener diff --git a/src/crewai/utilities/events/third_party/agentops_listener.py b/src/crewai/utilities/events/third_party/agentops_listener.py new file mode 100644 index 000000000..f34c0a5b6 --- /dev/null +++ b/src/crewai/utilities/events/third_party/agentops_listener.py @@ -0,0 +1,134 @@ +import logging + +from crewai.utilities.events.base_event_listener import BaseEventListener +from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus +from crewai.utilities.events.crew_events import ( + CrewKickoffCompletedEvent, + CrewKickoffStartedEvent, +) +from crewai.utilities.events.task_events import TaskEvaluationEvent +from crewai.utilities.events.tool_usage_events import ( + ToolUsageErrorEvent, + ToolUsageStartedEvent, +) + +logger = logging.getLogger(__name__) + + +class AgentOpsListener(BaseEventListener): + def __init__(self): + self.agentops = None + try: + import agentops + + self.agentops = agentops + logger.info("AgentOps integration enabled") + except ImportError: + logger.debug("AgentOps not installed, skipping AgentOps integration") + + super().__init__() + + def setup_listeners(self, crewai_event_bus: CrewAIEventsBus): + if self.agentops is None: + return + + crewai_event_bus.register_handler( + CrewKickoffStartedEvent, self._handle_crew_kickoff_started + ) + crewai_event_bus.register_handler( + CrewKickoffCompletedEvent, self._handle_crew_kickoff_completed + ) + crewai_event_bus.register_handler( + ToolUsageStartedEvent, self._handle_tool_usage_started + ) + crewai_event_bus.register_handler( + ToolUsageErrorEvent, self._handle_tool_usage_error + ) + crewai_event_bus.register_handler( + TaskEvaluationEvent, self._handle_task_evaluation + ) + + def _handle_crew_kickoff_started(self, event: CrewKickoffStartedEvent): + if self.agentops is None: + return + + try: + self.agentops.start_session( + tags=["crewai", "crew_kickoff"], + config=self.agentops.Configuration( + auto_start_session=False, + instrument_llm_calls=True, + ), + ) + logger.debug("AgentOps session started for crew kickoff") + except Exception as e: + logger.warning(f"Failed to start AgentOps session: {e}") + + def _handle_crew_kickoff_completed(self, event: CrewKickoffCompletedEvent): + if self.agentops is None: + return + + try: + self.agentops.end_session("Success") + logger.debug("AgentOps session ended for crew kickoff completion") + except Exception as e: + logger.warning(f"Failed to end AgentOps session: {e}") + + def _handle_tool_usage_started(self, event: ToolUsageStartedEvent): + if self.agentops is None: + return + + try: + self.agentops.record( + self.agentops.ActionEvent( + action_type="tool_usage", + params={ + "tool_name": event.tool_name, + "arguments": event.arguments, + }, + ) + ) + logger.debug(f"AgentOps recorded tool usage: {event.tool_name}") + except Exception as e: + logger.warning(f"Failed to record tool usage in AgentOps: {e}") + + def _handle_tool_usage_error(self, event: ToolUsageErrorEvent): + if self.agentops is None: + return + + try: + self.agentops.record( + self.agentops.ErrorEvent( + message=f"Tool usage error: {event.error}", + error_type="ToolUsageError", + details={ + "tool_name": event.tool_name, + "arguments": event.arguments, + }, + ) + ) + logger.debug(f"AgentOps recorded tool usage error: {event.tool_name}") + except Exception as e: + logger.warning(f"Failed to record tool usage error in AgentOps: {e}") + + def _handle_task_evaluation(self, event: TaskEvaluationEvent): + if self.agentops is None: + return + + try: + self.agentops.record( + self.agentops.ActionEvent( + action_type="task_evaluation", + params={ + "task_id": str(event.task_id), + "score": event.score, + "feedback": event.feedback, + }, + ) + ) + logger.debug(f"AgentOps recorded task evaluation: {event.task_id}") + except Exception as e: + logger.warning(f"Failed to record task evaluation in AgentOps: {e}") + + +agentops_listener = AgentOpsListener() diff --git a/tests/utilities/events/third_party/__init__.py b/tests/utilities/events/third_party/__init__.py new file mode 100644 index 000000000..27b68d4d0 --- /dev/null +++ b/tests/utilities/events/third_party/__init__.py @@ -0,0 +1,199 @@ + + +import pytest +from unittest.mock import Mock, patch, MagicMock +from crewai.utilities.events.third_party.agentops_listener import AgentOpsListener +from crewai.utilities.events.crew_events import ( + CrewKickoffStartedEvent, + CrewKickoffCompletedEvent, +) +from crewai.utilities.events.task_events import TaskEvaluationEvent +from crewai.utilities.events.tool_usage_events import ( + ToolUsageStartedEvent, + ToolUsageErrorEvent, +) +from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus + + +class TestAgentOpsListener: + def test_agentops_listener_initialization_with_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + assert listener.agentops is not None + + def test_agentops_listener_initialization_without_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + assert listener.agentops is None + + def test_setup_listeners_with_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + mock_event_bus = Mock(spec=CrewAIEventsBus) + + listener.setup_listeners(mock_event_bus) + + assert mock_event_bus.register_handler.call_count == 5 + mock_event_bus.register_handler.assert_any_call( + CrewKickoffStartedEvent, listener._handle_crew_kickoff_started + ) + mock_event_bus.register_handler.assert_any_call( + CrewKickoffCompletedEvent, listener._handle_crew_kickoff_completed + ) + mock_event_bus.register_handler.assert_any_call( + ToolUsageStartedEvent, listener._handle_tool_usage_started + ) + mock_event_bus.register_handler.assert_any_call( + ToolUsageErrorEvent, listener._handle_tool_usage_error + ) + mock_event_bus.register_handler.assert_any_call( + TaskEvaluationEvent, listener._handle_task_evaluation + ) + + def test_setup_listeners_without_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + mock_event_bus = Mock(spec=CrewAIEventsBus) + + listener.setup_listeners(mock_event_bus) + + mock_event_bus.register_handler.assert_not_called() + + def test_handle_crew_kickoff_started_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + mock_agentops.start_session.assert_called_once() + call_args = mock_agentops.start_session.call_args + assert call_args[1]["tags"] == ["crewai", "crew_kickoff"] + + def test_handle_crew_kickoff_started_without_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + def test_handle_crew_kickoff_completed_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + mock_agentops.end_session.assert_called_once_with("Success") + + def test_handle_crew_kickoff_completed_without_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + def test_handle_tool_usage_started_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = ToolUsageStartedEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_started(event) + + mock_agentops.record.assert_called_once() + call_args = mock_agentops.record.call_args[0][0] + assert hasattr(call_args, "action_type") + + def test_handle_tool_usage_error_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = ToolUsageErrorEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + error="Test error", + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_error(event) + + mock_agentops.record.assert_called_once() + + def test_handle_task_evaluation_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = TaskEvaluationEvent( + task_id="test-task", + score=0.85, + feedback="Good performance" + ) + + listener._handle_task_evaluation(event) + + mock_agentops.record.assert_called_once() + + def test_handle_crew_kickoff_started_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.start_session.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + def test_handle_crew_kickoff_completed_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.end_session.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + def test_handle_tool_usage_started_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = ToolUsageStartedEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_started(event) + + def test_handle_tool_usage_error_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = ToolUsageErrorEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + error="Test error", + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_error(event) + + def test_handle_task_evaluation_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = TaskEvaluationEvent( + task_id="test-task", + score=0.85, + feedback="Good performance" + ) + + listener._handle_task_evaluation(event) + + def test_agentops_listener_instance_creation(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops"): + from crewai.utilities.events.third_party.agentops_listener import agentops_listener + assert agentops_listener is not None + assert isinstance(agentops_listener, AgentOpsListener) diff --git a/tests/utilities/events/third_party/test_agentops_listener.py b/tests/utilities/events/third_party/test_agentops_listener.py new file mode 100644 index 000000000..ebc269622 --- /dev/null +++ b/tests/utilities/events/third_party/test_agentops_listener.py @@ -0,0 +1,196 @@ +from unittest.mock import Mock, patch +from crewai.utilities.events.third_party.agentops_listener import AgentOpsListener +from crewai.utilities.events.crew_events import ( + CrewKickoffStartedEvent, + CrewKickoffCompletedEvent, +) +from crewai.utilities.events.task_events import TaskEvaluationEvent +from crewai.utilities.events.tool_usage_events import ( + ToolUsageStartedEvent, + ToolUsageErrorEvent, +) +from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus + + +class TestAgentOpsListener: + def test_agentops_listener_initialization_with_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops"): + listener = AgentOpsListener() + assert listener.agentops is not None + + def test_agentops_listener_initialization_without_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + assert listener.agentops is None + + def test_setup_listeners_with_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops"): + listener = AgentOpsListener() + mock_event_bus = Mock(spec=CrewAIEventsBus) + + listener.setup_listeners(mock_event_bus) + + assert mock_event_bus.register_handler.call_count == 5 + mock_event_bus.register_handler.assert_any_call( + CrewKickoffStartedEvent, listener._handle_crew_kickoff_started + ) + mock_event_bus.register_handler.assert_any_call( + CrewKickoffCompletedEvent, listener._handle_crew_kickoff_completed + ) + mock_event_bus.register_handler.assert_any_call( + ToolUsageStartedEvent, listener._handle_tool_usage_started + ) + mock_event_bus.register_handler.assert_any_call( + ToolUsageErrorEvent, listener._handle_tool_usage_error + ) + mock_event_bus.register_handler.assert_any_call( + TaskEvaluationEvent, listener._handle_task_evaluation + ) + + def test_setup_listeners_without_agentops_installed(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + mock_event_bus = Mock(spec=CrewAIEventsBus) + + listener.setup_listeners(mock_event_bus) + + mock_event_bus.register_handler.assert_not_called() + + def test_handle_crew_kickoff_started_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + mock_agentops.start_session.assert_called_once() + call_args = mock_agentops.start_session.call_args + assert call_args[1]["tags"] == ["crewai", "crew_kickoff"] + + def test_handle_crew_kickoff_started_without_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + def test_handle_crew_kickoff_completed_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + mock_agentops.end_session.assert_called_once_with("Success") + + def test_handle_crew_kickoff_completed_without_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops", side_effect=ImportError): + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + def test_handle_tool_usage_started_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = ToolUsageStartedEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_started(event) + + mock_agentops.record.assert_called_once() + call_args = mock_agentops.record.call_args[0][0] + assert hasattr(call_args, "action_type") + + def test_handle_tool_usage_error_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = ToolUsageErrorEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + error="Test error", + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_error(event) + + mock_agentops.record.assert_called_once() + + def test_handle_task_evaluation_with_agentops(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + listener = AgentOpsListener() + event = TaskEvaluationEvent( + task_id="test-task", + score=0.85, + feedback="Good performance" + ) + + listener._handle_task_evaluation(event) + + mock_agentops.record.assert_called_once() + + def test_handle_crew_kickoff_started_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.start_session.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = CrewKickoffStartedEvent(crew_id="test-crew") + + listener._handle_crew_kickoff_started(event) + + def test_handle_crew_kickoff_completed_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.end_session.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = CrewKickoffCompletedEvent(crew_id="test-crew", crew_output=Mock()) + + listener._handle_crew_kickoff_completed(event) + + def test_handle_tool_usage_started_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = ToolUsageStartedEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_started(event) + + def test_handle_tool_usage_error_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = ToolUsageErrorEvent( + tool_name="test_tool", + arguments={"arg1": "value1"}, + error="Test error", + agent_id="test-agent", + task_id="test-task" + ) + + listener._handle_tool_usage_error(event) + + def test_handle_task_evaluation_with_exception(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops") as mock_agentops: + mock_agentops.record.side_effect = Exception("Test exception") + listener = AgentOpsListener() + event = TaskEvaluationEvent( + task_id="test-task", + score=0.85, + feedback="Good performance" + ) + + listener._handle_task_evaluation(event) + + def test_agentops_listener_instance_creation(self): + with patch("crewai.utilities.events.third_party.agentops_listener.agentops"): + from crewai.utilities.events.third_party.agentops_listener import agentops_listener + assert agentops_listener is not None + assert isinstance(agentops_listener, AgentOpsListener)