diff --git a/src/crewai/cli/plus_api.py b/src/crewai/cli/plus_api.py index 2b0e4a7d7..77b7fe5fd 100644 --- a/src/crewai/cli/plus_api.py +++ b/src/crewai/cli/plus_api.py @@ -166,3 +166,13 @@ class PlusAPI: json=payload, timeout=30, ) + + def mark_trace_batch_as_failed( + self, trace_batch_id: str, error_message: str + ) -> requests.Response: + return self._make_request( + "PATCH", + f"{self.TRACING_RESOURCE}/batches/{trace_batch_id}", + json={"status": "failed", "failure_reason": error_message}, + timeout=30, + ) diff --git a/src/crewai/events/listeners/tracing/trace_batch_manager.py b/src/crewai/events/listeners/tracing/trace_batch_manager.py index 826524d25..386cab91c 100644 --- a/src/crewai/events/listeners/tracing/trace_batch_manager.py +++ b/src/crewai/events/listeners/tracing/trace_batch_manager.py @@ -200,6 +200,9 @@ class TraceBatchManager: if self.event_buffer: events_sent_to_backend_status = self._send_events_to_backend() if events_sent_to_backend_status == 500: + self.plus_api.mark_trace_batch_as_failed( + self.trace_batch_id, "Error sending events to backend" + ) return None self._finalize_backend_batch() @@ -273,10 +276,13 @@ class TraceBatchManager: logger.error( f"❌ Failed to finalize trace batch: {response.status_code} - {response.text}" ) + self.plus_api.mark_trace_batch_as_failed( + self.trace_batch_id, response.text + ) except Exception as e: logger.error(f"❌ Error finalizing trace batch: {e}") - # TODO: send error to app marking as failed + self.plus_api.mark_trace_batch_as_failed(self.trace_batch_id, str(e)) def _cleanup_batch_data(self): """Clean up batch data after successful finalization to free memory""" diff --git a/tests/cassettes/TestTraceListenerSetup.test_trace_batch_marked_as_failed_on_finalize_error.yaml b/tests/cassettes/TestTraceListenerSetup.test_trace_batch_marked_as_failed_on_finalize_error.yaml new file mode 100644 index 000000000..b0161e2cd --- /dev/null +++ b/tests/cassettes/TestTraceListenerSetup.test_trace_batch_marked_as_failed_on_finalize_error.yaml @@ -0,0 +1,298 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are Test Agent. Test backstory\nYour + personal goal is: Test goal\nTo give my best complete final answer to the task + respond using the exact following format:\n\nThought: I now can give a great + answer\nFinal Answer: Your final answer must be the great and the most complete + as possible, it must be outcome described.\n\nI MUST use these formats, my job + depends on it!"}, {"role": "user", "content": "\nCurrent Task: Say hello to + the world\n\nThis is the expected criteria for your final answer: hello world\nyou + MUST return the actual complete content as the final answer, not a summary.\n\nBegin! + This is VERY important to you, use the tools available and give your best Final + Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": + ["\nObservation:"]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '825' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.93.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.93.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.9 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFJNj9MwEL3nV4x8blC/0nRzA8QKuCBxQFrBKnLtSeLF8Vi2s11Y9b8j + O90m5UPiEinz5j2/NzPPGQBTklXARMeD6K3O33588/nm/d27T5sv6wdp1OGu/9l1T9tbWzQlW0QG + HR5QhBfWK0G91RgUmREWDnnAqLoqi/1uv19vVgnoSaKOtNaGfEt5r4zK18v1Nl+W+Wp/ZnekBHpW + wdcMAOA5faNPI/GJVbBcvFR69J63yKpLEwBzpGOFce+VD9wEtphAQSagSdY/gKEjCG6gVY8IHNpo + G7jxR3QA38ytMlzD6/RfQYdaExzJaTkXdNgMnsdQZtB6BnBjKPA4lBTl/oycLuY1tdbRwf9GZY0y + yne1Q+7JRKM+kGUJPWUA92lIw1VuZh31NtSBvmN6blWUox6bdjNDN2cwUOB6Vi/Po73WqyUGrrSf + jZkJLjqUE3XaCR+kohmQzVL/6eZv2mNyZdr/kZ8AIdAGlLV1KJW4Tjy1OYyn+6+2y5STYebRPSqB + dVDo4iYkNnzQ40Ex/8MH7OtGmRaddWq8qsbWxW7Jmx0WxQ3LTtkvAAAA//8DAIkIBqtjAwAA + headers: + CF-RAY: + - 983f8c061b6ec487-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 24 Sep 2025 04:30:32 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=JDjpnzx5y8PJaJDQcCeX6MeBt8BOGuL79pd.ca5mqvE-1758688232-1.0.1.1-5VN5hj5LzEZFfkotBaZ_dbUITo_YB7RLsFOlQc.0MdSZOsz7WhNkH.s7H700L12Yi8nHGW44BgIwCF3uWx1w4PRBqrb1IVH3FkeV.QwCTaA; + path=/; expires=Wed, 24-Sep-25 05:00:32 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=b5n8BZZDRtHA4TrxQ1RDeEdtQBzhstjP6u21LYM8L94-1758688232142-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - crewai-iuxna1 + openai-processing-ms: + - '535' + openai-project: + - proj_xitITlrFeen7zjNSzML82h9x + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '562' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-project-tokens: + - '150000000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-project-tokens: + - '149999827' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999830' + x-ratelimit-reset-project-tokens: + - 0s + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_af61ab9d53bf400baf30c5bc5a7e2102 + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - api.scarf.sh + User-Agent: + - CrewAI-Python/0.193.2 + method: GET + uri: https://api.scarf.sh/v2/packages/CrewAI/crewai/docs/00f2dad1-8334-4a39-934e-003b2e1146db + response: + body: + string: '' + headers: + Connection: + - close + Date: + - Wed, 24 Sep 2025 04:47:59 GMT + Strict-Transport-Security: + - max-age=15724800; includeSubDomains + Transfer-Encoding: + - chunked + x-scarf-request-id: + - 4158376f-cb1c-46fe-a14c-dee366b955e2 + status: + code: 401 + message: Unauthorized +- request: + body: '{"trace_id": "06e1250e-6d88-4c64-abe5-deabde573ae1", "execution_type": + "crew", "user_identifier": null, "execution_context": {"crew_fingerprint": null, + "crew_name": "crew", "flow_name": null, "crewai_version": "0.193.2", "privacy_level": + "standard"}, "execution_metadata": {"expected_duration_estimate": 300, "agent_count": + 0, "task_count": 0, "flow_method_count": 0, "execution_started_at": "2025-09-24T04:50:23.219835+00:00"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '428' + Content-Type: + - application/json + User-Agent: + - CrewAI-CLI/0.193.2 + X-Crewai-Organization-Id: + - d3a3d10c-35db-423f-a7a4-c026030ba64d + X-Crewai-Version: + - 0.193.2 + method: POST + uri: http://localhost:3000/crewai_plus/api/v1/tracing/batches + response: + body: + string: '{"error":"bad_credentials","message":"Bad credentials"}' + headers: + Content-Length: + - '55' + cache-control: + - no-cache + content-security-policy: + - 'default-src ''self'' *.crewai.com crewai.com; script-src ''self'' ''unsafe-inline'' + *.crewai.com crewai.com https://cdn.jsdelivr.net/npm/apexcharts https://www.gstatic.com + https://run.pstmn.io https://share.descript.com/; style-src ''self'' ''unsafe-inline'' + *.crewai.com crewai.com https://cdn.jsdelivr.net/npm/apexcharts; img-src ''self'' + data: *.crewai.com crewai.com https://zeus.tools.crewai.com https://dashboard.tools.crewai.com + https://cdn.jsdelivr.net; font-src ''self'' data: *.crewai.com crewai.com; + connect-src ''self'' *.crewai.com crewai.com https://zeus.tools.crewai.com + https://connect.useparagon.com/ https://zeus.useparagon.com/* https://*.useparagon.com/* + https://run.pstmn.io https://connect.tools.crewai.com/ ws://localhost:3036 + wss://localhost:3036; frame-src ''self'' *.crewai.com crewai.com https://connect.useparagon.com/ + https://zeus.tools.crewai.com https://zeus.useparagon.com/* https://connect.tools.crewai.com/ + https://www.youtube.com https://share.descript.com' + content-type: + - application/json; charset=utf-8 + permissions-policy: + - camera=(), microphone=(self), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server-timing: + - cache_read.active_support;dur=0.37, sql.active_record;dur=30.81, cache_generate.active_support;dur=29.14, + cache_write.active_support;dur=0.14, cache_read_multi.active_support;dur=0.19, + start_processing.action_controller;dur=0.00, process_action.action_controller;dur=2.74 + vary: + - Accept + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-permitted-cross-domain-policies: + - none + x-request-id: + - 2420790e-9669-4235-851c-468185b6ef40 + x-runtime: + - '0.102516' + x-xss-protection: + - 1; mode=block + status: + code: 401 + message: Unauthorized +- request: + body: '{"status": "failed", "failure_reason": "Error sending events to backend"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '73' + Content-Type: + - application/json + User-Agent: + - CrewAI-CLI/0.193.2 + X-Crewai-Organization-Id: + - d3a3d10c-35db-423f-a7a4-c026030ba64d + X-Crewai-Version: + - 0.193.2 + method: PATCH + uri: http://localhost:3000/crewai_plus/api/v1/tracing/batches/None + response: + body: + string: '{"error":"bad_credentials","message":"Bad credentials"}' + headers: + Content-Length: + - '55' + cache-control: + - no-cache + content-security-policy: + - 'default-src ''self'' *.crewai.com crewai.com; script-src ''self'' ''unsafe-inline'' + *.crewai.com crewai.com https://cdn.jsdelivr.net/npm/apexcharts https://www.gstatic.com + https://run.pstmn.io https://share.descript.com/; style-src ''self'' ''unsafe-inline'' + *.crewai.com crewai.com https://cdn.jsdelivr.net/npm/apexcharts; img-src ''self'' + data: *.crewai.com crewai.com https://zeus.tools.crewai.com https://dashboard.tools.crewai.com + https://cdn.jsdelivr.net; font-src ''self'' data: *.crewai.com crewai.com; + connect-src ''self'' *.crewai.com crewai.com https://zeus.tools.crewai.com + https://connect.useparagon.com/ https://zeus.useparagon.com/* https://*.useparagon.com/* + https://run.pstmn.io https://connect.tools.crewai.com/ ws://localhost:3036 + wss://localhost:3036; frame-src ''self'' *.crewai.com crewai.com https://connect.useparagon.com/ + https://zeus.tools.crewai.com https://zeus.useparagon.com/* https://connect.tools.crewai.com/ + https://www.youtube.com https://share.descript.com' + content-type: + - application/json; charset=utf-8 + permissions-policy: + - camera=(), microphone=(self), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server-timing: + - cache_read.active_support;dur=0.06, sql.active_record;dur=3.86, cache_generate.active_support;dur=4.28, + cache_write.active_support;dur=0.15, cache_read_multi.active_support;dur=0.12, + start_processing.action_controller;dur=0.00, process_action.action_controller;dur=1.70 + vary: + - Accept + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-permitted-cross-domain-policies: + - none + x-request-id: + - 1750d141-c48f-47f1-b8b4-130195437d22 + x-runtime: + - '0.043849' + x-xss-protection: + - 1; mode=block + status: + code: 401 + message: Unauthorized +version: 1 diff --git a/tests/tracing/test_tracing.py b/tests/tracing/test_tracing.py index 2ac07eb6d..629fb7a87 100644 --- a/tests/tracing/test_tracing.py +++ b/tests/tracing/test_tracing.py @@ -80,10 +80,6 @@ class TestTraceListenerSetup: patch("requests.get") as mock_get, patch("requests.put") as mock_put, patch("requests.delete") as mock_delete, - patch.object(TraceBatchManager, "initialize_batch", return_value=None), - patch.object( - TraceBatchManager, "_finalize_backend_batch", return_value=True - ), patch.object(TraceBatchManager, "_cleanup_batch_data", return_value=True), ): mock_response = MagicMock() @@ -100,11 +96,15 @@ class TestTraceListenerSetup: mock_put.return_value = mock_response mock_delete.return_value = mock_response + mock_mark_failed = MagicMock() + mock_mark_failed.return_value = mock_response + yield { "post": mock_post, "get": mock_get, "put": mock_put, "delete": mock_delete, + "mark_trace_batch_as_failed": mock_mark_failed, } @pytest.mark.vcr(filter_headers=["authorization"]) @@ -657,3 +657,51 @@ class TestTraceListenerSetup: handler.handle_execution_completion() mock_mark_completed.assert_called_once() + + @pytest.mark.vcr(filter_headers=["authorization"]) + def test_trace_batch_marked_as_failed_on_finalize_error(self, mock_plus_api_calls): + """Test that trace batch is marked as failed when finalization returns non-200 status""" + + with patch.dict(os.environ, {"CREWAI_TRACING_ENABLED": "true"}): + agent = Agent( + role="Test Agent", + goal="Test goal", + backstory="Test backstory", + llm="gpt-4o-mini", + ) + task = Task( + description="Say hello to the world", + expected_output="hello world", + agent=agent, + ) + crew = Crew(agents=[agent], tasks=[task], verbose=True) + + trace_listener = TraceCollectionListener() + from crewai.events.event_bus import crewai_event_bus + + trace_listener.setup_listeners(crewai_event_bus) + + with ( + patch.object( + trace_listener.batch_manager.plus_api, + "send_trace_events", + return_value=MagicMock(status_code=200), + ), + patch.object( + trace_listener.batch_manager.plus_api, + "finalize_trace_batch", + return_value=MagicMock( + status_code=500, text="Internal Server Error" + ), + ), + patch.object( + trace_listener.batch_manager.plus_api, + "mark_trace_batch_as_failed", + wraps=mock_plus_api_calls["mark_trace_batch_as_failed"], + ) as mock_mark_failed, + ): + crew.kickoff() + + mock_mark_failed.assert_called_once() + call_args = mock_mark_failed.call_args_list[0] + assert call_args[0][1] == "Error sending events to backend"