diff --git a/src/crewai/llm.py b/src/crewai/llm.py index aac1af3b7..a0378cd2d 100644 --- a/src/crewai/llm.py +++ b/src/crewai/llm.py @@ -21,6 +21,12 @@ from typing import ( from dotenv import load_dotenv from pydantic import BaseModel +from crewai.utilities.events.llm_events import ( + LLMCallCompletedEvent, + LLMCallFailedEvent, + LLMCallStartedEvent, + LLMCallType, +) from crewai.utilities.events.tool_usage_events import ToolExecutionErrorEvent with warnings.catch_warnings(): @@ -259,6 +265,15 @@ class LLM: >>> print(response) "The capital of France is Paris." """ + crewai_event_bus.emit( + self, + event=LLMCallStartedEvent( + messages=messages, + tools=tools, + callbacks=callbacks, + available_functions=available_functions, + ), + ) # Validate parameters before proceeding with the call. self._validate_call_params() @@ -333,12 +348,13 @@ class LLM: # --- 4) If no tool calls, return the text response if not tool_calls or not available_functions: + self._handle_emit_call_events(text_response, LLMCallType.LLM_CALL) return text_response # --- 5) Handle the tool call tool_call = tool_calls[0] function_name = tool_call.function.name - print("function_name", function_name) + if function_name in available_functions: try: function_args = json.loads(tool_call.function.arguments) @@ -350,6 +366,7 @@ class LLM: try: # Call the actual tool function result = fn(**function_args) + self._handle_emit_call_events(result, LLMCallType.TOOL_CALL) return result except Exception as e: @@ -365,6 +382,12 @@ class LLM: error=str(e), ), ) + crewai_event_bus.emit( + self, + event=LLMCallFailedEvent( + error=f"Tool execution error: {str(e)}" + ), + ) return text_response else: @@ -374,12 +397,28 @@ class LLM: return text_response except Exception as e: + crewai_event_bus.emit( + self, + event=LLMCallFailedEvent(error=str(e)), + ) if not LLMContextLengthExceededException( str(e) )._is_context_limit_error(str(e)): logging.error(f"LiteLLM call failed: {str(e)}") raise + def _handle_emit_call_events(self, response: Any, call_type: LLMCallType): + """Handle the events for the LLM call. + + Args: + response (str): The response from the LLM call. + call_type (str): The type of call, either "tool_call" or "llm_call". + """ + crewai_event_bus.emit( + self, + event=LLMCallCompletedEvent(response=response, call_type=call_type), + ) + def _format_messages_for_provider( self, messages: List[Dict[str, str]] ) -> List[Dict[str, str]]: diff --git a/src/crewai/utilities/events/__init__.py b/src/crewai/utilities/events/__init__.py index 7f3442360..aa4a24ac5 100644 --- a/src/crewai/utilities/events/__init__.py +++ b/src/crewai/utilities/events/__init__.py @@ -34,6 +34,7 @@ from .tool_usage_events import ( ToolUsageEvent, ToolValidateInputErrorEvent, ) +from .llm_events import LLMCallCompletedEvent, LLMCallFailedEvent, LLMCallStartedEvent # events from .event_listener import EventListener diff --git a/src/crewai/utilities/events/event_listener.py b/src/crewai/utilities/events/event_listener.py index 0dcefcd3d..793528240 100644 --- a/src/crewai/utilities/events/event_listener.py +++ b/src/crewai/utilities/events/event_listener.py @@ -4,6 +4,11 @@ from crewai.telemetry.telemetry import Telemetry from crewai.utilities import Logger from crewai.utilities.constants import EMITTER_COLOR from crewai.utilities.events.base_event_listener import BaseEventListener +from crewai.utilities.events.llm_events import ( + LLMCallCompletedEvent, + LLMCallFailedEvent, + LLMCallStartedEvent, +) from .agent_events import AgentExecutionCompletedEvent, AgentExecutionStartedEvent from .crew_events import ( @@ -253,5 +258,28 @@ class EventListener(BaseEventListener): # ) + # ----------- LLM EVENTS ----------- + + @crewai_event_bus.on(LLMCallStartedEvent) + def on_llm_call_started(source, event: LLMCallStartedEvent): + self.logger.log( + f"🤖 LLM Call Started", + event.timestamp, + ) + + @crewai_event_bus.on(LLMCallCompletedEvent) + def on_llm_call_completed(source, event: LLMCallCompletedEvent): + self.logger.log( + f"✅ LLM Call Completed", + event.timestamp, + ) + + @crewai_event_bus.on(LLMCallFailedEvent) + def on_llm_call_failed(source, event: LLMCallFailedEvent): + self.logger.log( + f"❌ LLM Call Failed: '{event.error}'", + event.timestamp, + ) + event_listener = EventListener() diff --git a/src/crewai/utilities/events/llm_events.py b/src/crewai/utilities/events/llm_events.py new file mode 100644 index 000000000..8c2554a21 --- /dev/null +++ b/src/crewai/utilities/events/llm_events.py @@ -0,0 +1,36 @@ +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from crewai.utilities.events.base_events import CrewEvent + + +class LLMCallType(Enum): + """Type of LLM call being made""" + + TOOL_CALL = "tool_call" + LLM_CALL = "llm_call" + + +class LLMCallStartedEvent(CrewEvent): + """Event emitted when a LLM call starts""" + + type: str = "llm_call_started" + messages: Union[str, List[Dict[str, str]]] + tools: Optional[List[dict]] = None + callbacks: Optional[List[Any]] = None + available_functions: Optional[Dict[str, Any]] = None + + +class LLMCallCompletedEvent(CrewEvent): + """Event emitted when a LLM call completes""" + + type: str = "llm_call_completed" + response: Any + call_type: LLMCallType + + +class LLMCallFailedEvent(CrewEvent): + """Event emitted when a LLM call fails""" + + error: str + type: str = "llm_call_failed" diff --git a/src/crewai/utilities/events/task_events.py b/src/crewai/utilities/events/task_events.py index f69e77d6a..d81b7ce2d 100644 --- a/src/crewai/utilities/events/task_events.py +++ b/src/crewai/utilities/events/task_events.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Optional from crewai.tasks.task_output import TaskOutput from crewai.utilities.events.base_events import CrewEvent diff --git a/tests/utilities/cassettes/test_llm_emits_call_failed_event.yaml b/tests/utilities/cassettes/test_llm_emits_call_failed_event.yaml new file mode 100644 index 000000000..2222ad933 --- /dev/null +++ b/tests/utilities/cassettes/test_llm_emits_call_failed_event.yaml @@ -0,0 +1,103 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": + "gpt-4o-mini", "stop": []}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '102' + content-type: + - application/json + cookie: + - _cfuvid=IY8ppO70AMHr2skDSUsGh71zqHHdCQCZ3OvkPi26NBc-1740424913267-0.0.1.1-604800000; + __cf_bm=fU6K5KZoDmgcEuF8_yWAYKUO5fKHh6q5.wDPnna393g-1740424913-1.0.1.1-2iOaq3JVGWs439V0HxJee0IC9HdJm7dPkeJorD.AGw0YwkngRPM8rrTzn_7ht1BkbOauEezj.wPKcBz18gIYUg + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.61.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.61.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-B4YLA2SrC2rwdVQ3U87G5a0P5lsLw\",\n \"object\": + \"chat.completion\",\n \"created\": 1740425016,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Hello! I'm just a computer program, so + I don't have feelings, but I'm here and ready to help you. How can I assist + you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_709714d124\"\n}\n" + headers: + CF-RAY: + - 9171d4c0ed44236e-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 24 Feb 2025 19:23:38 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '1954' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999978' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_ea2703502b8827e4297cd2a7bae9d9c8 + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/utilities/cassettes/test_llm_emits_call_started_event.yaml b/tests/utilities/cassettes/test_llm_emits_call_started_event.yaml new file mode 100644 index 000000000..0120aa1b3 --- /dev/null +++ b/tests/utilities/cassettes/test_llm_emits_call_started_event.yaml @@ -0,0 +1,108 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": + "gpt-4o-mini", "stop": []}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '102' + content-type: + - application/json + cookie: + - _cfuvid=GefCcEtb_Gem93E4a9Hvt3Xyof1YQZVJAXBb9I6pEUs-1739398417375-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.61.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.61.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-B4YJU8IWKGyBQtAyPDRd3SFI2flYR\",\n \"object\": + \"chat.completion\",\n \"created\": 1740424912,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Hello! I'm just a computer program, so + I don't have feelings, but I'm here and ready to help you. How can I assist + you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_7fcd609668\"\n}\n" + headers: + CF-RAY: + - 9171d230d8ed7ae0-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 24 Feb 2025 19:21:53 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=fU6K5KZoDmgcEuF8_yWAYKUO5fKHh6q5.wDPnna393g-1740424913-1.0.1.1-2iOaq3JVGWs439V0HxJee0IC9HdJm7dPkeJorD.AGw0YwkngRPM8rrTzn_7ht1BkbOauEezj.wPKcBz18gIYUg; + path=/; expires=Mon, 24-Feb-25 19:51:53 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=IY8ppO70AMHr2skDSUsGh71zqHHdCQCZ3OvkPi26NBc-1740424913267-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '993' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999978' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d9c4d49185e97b1797061efc1e55d811 + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/utilities/test_events.py b/tests/utilities/test_events.py index 68bda7bec..aa65d82b2 100644 --- a/tests/utilities/test_events.py +++ b/tests/utilities/test_events.py @@ -1,6 +1,5 @@ -import json from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest from pydantic import Field @@ -9,6 +8,7 @@ from crewai.agent import Agent from crewai.agents.crew_agent_executor import CrewAgentExecutor from crewai.crew import Crew from crewai.flow.flow import Flow, listen, start +from crewai.llm import LLM from crewai.task import Task from crewai.tools.base_tool import BaseTool from crewai.tools.tool_usage import ToolUsage @@ -31,6 +31,12 @@ from crewai.utilities.events.flow_events import ( MethodExecutionFailedEvent, MethodExecutionStartedEvent, ) +from crewai.utilities.events.llm_events import ( + LLMCallCompletedEvent, + LLMCallFailedEvent, + LLMCallStartedEvent, + LLMCallType, +) from crewai.utilities.events.task_events import ( TaskCompletedEvent, TaskFailedEvent, @@ -495,3 +501,43 @@ def test_flow_emits_method_execution_failed_event(): assert received_events[0].flow_name == "TestFlow" assert received_events[0].type == "method_execution_failed" assert received_events[0].error == error + + +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_llm_emits_call_started_event(): + received_events = [] + + @crewai_event_bus.on(LLMCallStartedEvent) + def handle_llm_call_started(source, event): + received_events.append(event) + + @crewai_event_bus.on(LLMCallCompletedEvent) + def handle_llm_call_completed(source, event): + received_events.append(event) + + llm = LLM(model="gpt-4o-mini") + llm.call("Hello, how are you?") + + assert len(received_events) == 2 + assert received_events[0].type == "llm_call_started" + assert received_events[1].type == "llm_call_completed" + + +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_llm_emits_call_failed_event(): + received_events = [] + + @crewai_event_bus.on(LLMCallFailedEvent) + def handle_llm_call_failed(source, event): + received_events.append(event) + + error_message = "Simulated LLM call failure" + with patch.object(LLM, "_call_llm", side_effect=Exception(error_message)): + llm = LLM(model="gpt-4o-mini") + with pytest.raises(Exception) as exc_info: + llm.call("Hello, how are you?") + + assert str(exc_info.value) == error_message + assert len(received_events) == 1 + assert received_events[0].type == "llm_call_failed" + assert received_events[0].error == error_message