fix: remove stdout prints, improve test determinism, and update trace handling

Removed `print` statements from the `LLMStreamChunkEvent` handler to prevent LLM response chunks from being written directly to stdout. The listener now only tracks chunks internally. Fixes #3715 Added explicit return statements for trace-related tests. Updated cassette for `test_failed_evaluation` to reflect new behavior where an empty trace dict is used instead of returning early. Ensured deterministic cleanup order in test fixtures by making `clear_event_bus_handlers` depend on `setup_test_environment`. This guarantees event bus shutdown and file handle cleanup occur before temporary directory deletion, resolving intermittent “Directory not empty” errors in CI.
2026-01-08 15:48:29 +00:00 · 2025-10-18 12:27:16 -04:00
parent 436f1b4639
commit b680065c45
4 changed files with 108 additions and 11 deletions
--- a/lib/crewai/src/crewai/events/event_listener.py
+++ b/lib/crewai/src/crewai/events/event_listener.py
@@ -53,7 +53,6 @@ from crewai.task import Task
 from crewai.telemetry.telemetry import Telemetry
 from crewai.utilities import Logger
 from crewai.utilities.constants import EMITTER_COLOR
-from crewai.utilities.printer import Printer

 from .listeners.memory_listener import MemoryListener
 from .types.flow_events import (
@@ -76,8 +75,6 @@ from .types.tool_usage_events import (
    ToolUsageStartedEvent,
 )

-_printer = Printer()
-

 class EventListener(BaseEventListener):
    _instance = None
@@ -381,12 +378,8 @@ class EventListener(BaseEventListener):
        @crewai_event_bus.on(LLMStreamChunkEvent)
        def on_llm_stream_chunk(source, event: LLMStreamChunkEvent):
            self.text_stream.write(event.chunk)
-
            self.text_stream.seek(self.next_chunk)
-
-            # Read from the in-memory stream
-            content = self.text_stream.read()
-            _printer.print(content)
+            self.text_stream.read()
            self.next_chunk = self.text_stream.tell()

        # ----------- LLM GUARDRAIL EVENTS -----------
--- a/lib/crewai/src/crewai/experimental/evaluation/agent_evaluator.py
+++ b/lib/crewai/src/crewai/experimental/evaluation/agent_evaluator.py
@@ -97,7 +97,7 @@ class AgentEvaluator:
            )

            if not trace:
-                return
+                trace = {}

            result = self.evaluate(
                agent=agent,
@@ -151,7 +151,7 @@ class AgentEvaluator:
            )

            if not trace:
-                return
+                trace = {}

            result = self.evaluate(
                agent=target_agent,
--- a/lib/crewai/tests/cassettes/TestAgentEvaluator.test_failed_evaluation.yaml
+++ b/lib/crewai/tests/cassettes/TestAgentEvaluator.test_failed_evaluation.yaml
@@ -120,4 +120,105 @@ interactions:
    status:
      code: 200
      message: OK
+- request:
+    body: '{"trace_id": "b0237c14-8cd1-4453-920d-608a63d4b7ef", "execution_type":
+      "crew", "user_identifier": null, "execution_context": {"crew_fingerprint": null,
+      "crew_name": "crew", "flow_name": null, "crewai_version": "1.0.0b2", "privacy_level":
+      "standard"}, "execution_metadata": {"expected_duration_estimate": 300, "agent_count":
+      0, "task_count": 0, "flow_method_count": 0, "execution_started_at": "2025-10-18T15:21:00.300365+00:00"},
+      "ephemeral_trace_id": "b0237c14-8cd1-4453-920d-608a63d4b7ef"}'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '490'
+      Content-Type:
+      - application/json
+      User-Agent:
+      - CrewAI-CLI/1.0.0b2
+      X-Crewai-Organization-Id:
+      - 60577da1-895c-4675-8135-62e9010bdcf3
+      X-Crewai-Version:
+      - 1.0.0b2
+    method: POST
+    uri: https://app.crewai.com/crewai_plus/api/v1/tracing/ephemeral/batches
+  response:
+    body:
+      string: '{"id":"703e1e1b-7cca-4cc6-9d03-95d5ab7461e2","ephemeral_trace_id":"b0237c14-8cd1-4453-920d-608a63d4b7ef","execution_type":"crew","crew_name":"crew","flow_name":null,"status":"running","duration_ms":null,"crewai_version":"1.0.0b2","total_events":0,"execution_context":{"crew_fingerprint":null,"crew_name":"crew","flow_name":null,"crewai_version":"1.0.0b2","privacy_level":"standard"},"created_at":"2025-10-18T15:21:01.551Z","updated_at":"2025-10-18T15:21:01.551Z","access_code":"TRACE-91322fd9f9","user_identifier":null}'
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '519'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Sat, 18 Oct 2025 15:21:01 GMT
+      cache-control:
+      - no-store
+      content-security-policy:
+      - 'default-src ''self'' *.app.crewai.com app.crewai.com; script-src ''self''
+        ''unsafe-inline'' *.app.crewai.com app.crewai.com https://cdn.jsdelivr.net/npm/apexcharts
+        https://www.gstatic.com https://run.pstmn.io https://apis.google.com https://apis.google.com/js/api.js
+        https://accounts.google.com https://accounts.google.com/gsi/client https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css.map
+        https://*.google.com https://docs.google.com https://slides.google.com https://js.hs-scripts.com
+        https://js.sentry-cdn.com https://browser.sentry-cdn.com https://www.googletagmanager.com
+        https://js-na1.hs-scripts.com https://js.hubspot.com http://js-na1.hs-scripts.com
+        https://bat.bing.com https://cdn.amplitude.com https://cdn.segment.com https://d1d3n03t5zntha.cloudfront.net/
+        https://descriptusercontent.com https://edge.fullstory.com https://googleads.g.doubleclick.net
+        https://js.hs-analytics.net https://js.hs-banner.com https://js.hsadspixel.net
+        https://js.hscollectedforms.net https://js.usemessages.com https://snap.licdn.com
+        https://static.cloudflareinsights.com https://static.reo.dev https://www.google-analytics.com
+        https://share.descript.com/; style-src ''self'' ''unsafe-inline'' *.app.crewai.com
+        app.crewai.com https://cdn.jsdelivr.net/npm/apexcharts; img-src ''self'' data:
+        *.app.crewai.com app.crewai.com https://zeus.tools.crewai.com https://dashboard.tools.crewai.com
+        https://cdn.jsdelivr.net https://forms.hsforms.com https://track.hubspot.com
+        https://px.ads.linkedin.com https://px4.ads.linkedin.com https://www.google.com
+        https://www.google.com.br; font-src ''self'' data: *.app.crewai.com app.crewai.com;
+        connect-src ''self'' *.app.crewai.com app.crewai.com https://zeus.tools.crewai.com
+        https://connect.useparagon.com/ https://zeus.useparagon.com/* https://*.useparagon.com/*
+        https://run.pstmn.io https://connect.tools.crewai.com/ https://*.sentry.io
+        https://www.google-analytics.com https://edge.fullstory.com https://rs.fullstory.com
+        https://api.hubspot.com https://forms.hscollectedforms.net https://api.hubapi.com
+        https://px.ads.linkedin.com https://px4.ads.linkedin.com https://google.com/pagead/form-data/16713662509
+        https://google.com/ccm/form-data/16713662509 https://www.google.com/ccm/collect
+        https://worker-actionkit.tools.crewai.com https://api.reo.dev; frame-src ''self''
+        *.app.crewai.com app.crewai.com https://connect.useparagon.com/ https://zeus.tools.crewai.com
+        https://zeus.useparagon.com/* https://connect.tools.crewai.com/ https://docs.google.com
+        https://drive.google.com https://slides.google.com https://accounts.google.com
+        https://*.google.com https://app.hubspot.com/ https://td.doubleclick.net https://www.googletagmanager.com/
+        https://www.youtube.com https://share.descript.com'
+      etag:
+      - W/"9e9becfaa0607314159093ffcadb0713"
+      expires:
+      - '0'
+      permissions-policy:
+      - camera=(), microphone=(self), geolocation=()
+      pragma:
+      - no-cache
+      referrer-policy:
+      - strict-origin-when-cross-origin
+      strict-transport-security:
+      - max-age=63072000; includeSubDomains
+      vary:
+      - Accept
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+      x-permitted-cross-domain-policies:
+      - none
+      x-request-id:
+      - 7dd520cd-8e74-4648-968b-90b1dc2e81d8
+      x-runtime:
+      - '0.099253'
+      x-xss-protection:
+      - 1; mode=block
+    status:
+      code: 201
+      message: Created
 version: 1
--- a/lib/crewai/tests/conftest.py
+++ b/lib/crewai/tests/conftest.py
@@ -160,11 +160,13 @@ def mock_opentelemetry_components():


@pytest.fixture(autouse=True)
-def clear_event_bus_handlers():
+def clear_event_bus_handlers(setup_test_environment):
    """Clear event bus handlers after each test for isolation.

    Handlers registered during the test are allowed to run, then cleaned up
    after the test completes.
+
+    Depends on setup_test_environment to ensure cleanup happens in correct order.
    """
    from crewai.events.event_bus import crewai_event_bus
    from crewai.experimental.evaluation.evaluation_listener import (
@@ -173,6 +175,7 @@ def clear_event_bus_handlers():

    yield

+    # Shutdown event bus and wait for all handlers to complete
    crewai_event_bus.shutdown(wait=True)
    crewai_event_bus._initialize()