crewAI/lib/crewai-core/tests/test_telemetry_deploy.py

"""Deployment telemetry: attribution by origin, and an origin-independent count.

``Create Crew Deployment`` and ``Start Deployment`` answer "which deployment,
from where"; ``deploy:created`` / ``deploy:pushed`` answer "how many
deployments", from the feature-usage aggregation, regardless of origin.
"""

from __future__ import annotations

from collections.abc import Iterator
from typing import Any, cast
from unittest.mock import MagicMock, patch

from crewai_core.telemetry import Telemetry
import pytest


@pytest.fixture
def telemetry() -> Iterator[tuple[Telemetry, MagicMock]]:
    """A Telemetry whose spans are captured instead of exported.

    The singleton is disabled in tests and never builds a provider, so both the
    gate and the provider are supplied here.
    """
    instance = Telemetry()
    span = MagicMock()
    provider = MagicMock()
    provider.get_tracer.return_value.start_span.return_value = span

    with (
        patch.object(instance, "provider", provider, create=True),
        patch.object(instance, "_should_execute_telemetry", return_value=True),
        patch("crewai_core.telemetry.close_span"),
    ):
        yield instance, span


def _attributes(span: MagicMock) -> dict[str, Any]:
    return {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}


def _span_names(provider: MagicMock) -> list[str]:
    tracer = provider.get_tracer.return_value
    return [call.args[0] for call in tracer.start_span.call_args_list]


class TestCreateDeployment:
    def test_defaults_to_cli(self, telemetry: tuple[Telemetry, MagicMock]) -> None:
        instance, span = telemetry
        instance.create_crew_deployment_span()
        assert _attributes(span)["source"] == "cli"

    def test_records_tui_when_started_from_the_run_ui(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.create_crew_deployment_span(source="tui")
        assert _attributes(span)["source"] == "tui"

    def test_also_counts_the_deployment_as_a_feature(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, _ = telemetry
        with patch.object(instance, "feature_usage_span") as feature:
            instance.create_crew_deployment_span()
        feature.assert_called_once_with("deploy:created")

    def test_feature_count_is_the_same_from_either_origin(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """The whole point: one number for deployments, whatever started them."""
        instance, _ = telemetry
        with patch.object(instance, "feature_usage_span") as feature:
            instance.create_crew_deployment_span(source="cli")
            instance.create_crew_deployment_span(source="tui")
        assert [call.args[0] for call in feature.call_args_list] == [
            "deploy:created",
            "deploy:created",
        ]


class TestCrewDeploymentCreated:
    """The post-success span: the only one that can carry the created uuid."""

    def test_carries_the_uuid_and_defaults_to_cli(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.crew_deployment_created_span("dep-abc")
        attributes = _attributes(span)
        assert attributes["uuid"] == "dep-abc"
        assert attributes["source"] == "cli"

    def test_is_a_separate_span_from_the_attempt(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """Two names, because they count different things.

        ``Create Crew Deployment`` counts attempts and fires before the API call;
        this one fires only after one succeeded. Collapsing them would turn the
        attempt metric into a success metric, which the churn figures rely on.
        """
        instance, _ = telemetry
        # The fixture patches `provider` with a MagicMock; the declared type is a
        # real TracerProvider, so narrow it rather than reaching through it.
        tracer = cast(MagicMock, instance.provider).get_tracer.return_value
        # feature_usage_span opens a span of its own; patch it out so this asserts
        # on the two deployment spans rather than on emission order in general.
        with patch.object(instance, "feature_usage_span"):
            instance.create_crew_deployment_span()
            instance.crew_deployment_created_span("dep-abc")
        assert [call.args[0] for call in tracer.start_span.call_args_list] == [
            "Create Crew Deployment",
            "Crew Deployment Created",
        ]

    def test_does_not_emit_a_second_deploy_created_count(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """A second emit would double every deployment in the feature count."""
        instance, _ = telemetry
        with patch.object(instance, "feature_usage_span") as feature:
            instance.crew_deployment_created_span("dep-abc")
        feature.assert_not_called()

    def test_omits_the_uuid_key_rather_than_writing_an_empty_one(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """Absent must stay distinguishable from empty, as on the sibling spans."""
        instance, span = telemetry
        instance.crew_deployment_created_span(None, source="tui")
        attributes = _attributes(span)
        assert "uuid" not in attributes
        assert attributes["source"] == "tui"


class TestStartDeployment:
    def test_defaults_to_cli_and_keeps_the_uuid(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.start_deployment_span("dep-123")
        attributes = _attributes(span)
        assert attributes["source"] == "cli"
        assert attributes["uuid"] == "dep-123"

    def test_records_tui_when_started_from_the_run_ui(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.start_deployment_span("dep-123", source="tui")
        assert _attributes(span)["source"] == "tui"

    def test_source_is_recorded_even_without_a_uuid(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """uuid is optional; source must not be conditional on it."""
        instance, span = telemetry
        instance.start_deployment_span(None, source="tui")
        attributes = _attributes(span)
        assert attributes["source"] == "tui"
        assert "uuid" not in attributes

    def test_also_counts_the_deployment_as_a_feature(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, _ = telemetry
        with patch.object(instance, "feature_usage_span") as feature:
            instance.start_deployment_span("dep-123")
        feature.assert_called_once_with("deploy:pushed")


class TestCrewDeploymentFailed:
    """The third deployment span: why an attempt did not become a success."""

    def test_records_the_reason_and_defaults_to_cli(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.crew_deployment_failed_span("network_error")
        attributes = _attributes(span)
        assert attributes["reason"] == "network_error"
        assert attributes["source"] == "cli"

    def test_carries_the_http_status_when_the_api_answered(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, span = telemetry
        instance.crew_deployment_failed_span("api_4xx", status_code=422)
        assert _attributes(span)["status_code"] == 422

    def test_omits_the_status_key_when_there_was_no_response(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """A missing key means "no response"; a 0 would read as a status."""
        instance, span = telemetry
        instance.crew_deployment_failed_span("user_declined", source="tui")
        attributes = _attributes(span)
        assert "status_code" not in attributes
        assert attributes["source"] == "tui"

    def test_is_a_separate_span_from_the_attempt_and_the_success(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        instance, _span = telemetry
        # The attempt also emits its deploy:created feature span; that count is
        # covered elsewhere and only clutters the sequence asserted here.
        with patch.object(instance, "feature_usage_span"):
            instance.create_crew_deployment_span()
            instance.crew_deployment_failed_span("api_5xx", status_code=500)
        provider = cast(MagicMock, instance.provider)
        assert _span_names(provider) == [
            "Create Crew Deployment",
            "Crew Deployment Failed",
        ]

    def test_does_not_emit_a_feature_count(
        self, telemetry: tuple[Telemetry, MagicMock]
    ) -> None:
        """deploy:created is already counted by the attempt span."""
        instance, _span = telemetry
        with patch.object(instance, "feature_usage_span") as feature:
            instance.crew_deployment_failed_span("zip_error")
        feature.assert_not_called()


class TestDisabledTelemetry:
    def test_opted_out_users_emit_nothing(self) -> None:
        """No span and no feature count when telemetry is off."""
        instance = Telemetry()
        provider = MagicMock()

        with (
            patch.object(instance, "provider", provider, create=True),
            patch.object(instance, "_should_execute_telemetry", return_value=False),
            patch.object(instance, "feature_usage_span") as feature,
        ):
            instance.create_crew_deployment_span(source="tui")
            instance.start_deployment_span("dep-123", source="tui")
            instance.crew_deployment_failed_span("api_4xx", status_code=401)

        assert _span_names(provider) == []
        # feature_usage_span is itself gated, so it is still called; it is the
        # export that must not happen. Assert it was not bypassed some other way.
        assert [call.args[0] for call in feature.call_args_list] == [
            "deploy:created",
            "deploy:pushed",
        ]


class TestReleaseAttribution:
    """Every span this emitter produces must carry the running release.

    A span without ``crewai_version`` cannot be attributed to a version, so a
    version-filtered question returns nothing for it rather than something
    visibly wrong. Covers all ten spans this module emits, not only the ones
    that were missing it, so a regression on the others is caught too.
    """

    @pytest.mark.parametrize(
        ("method", "args"),
        [
            ("deploy_signup_error_span", ()),
            ("start_deployment_span", ("dep-123",)),
            ("create_crew_deployment_span", ()),
            ("crew_deployment_created_span", ("dep-123",)),
            ("crew_deployment_failed_span", ("api_5xx",)),
            ("get_crew_logs_span", ("dep-123", "deployment")),
            ("remove_crew_span", ("dep-123",)),
            ("feature_usage_span", ("memory:query",)),
            ("flow_creation_span", ("ResearchFlow",)),
            ("template_installed_span", ("my-template",)),
        ],
    )
    def test_span_records_the_release(
        self,
        telemetry: tuple[Telemetry, MagicMock],
        method: str,
        args: tuple[object, ...],
    ) -> None:
        instance, span = telemetry
        sentinel = "0.0.0-release-sentinel"

        # Patched at the source module, not at crewai_core.telemetry: these
        # methods import get_crewai_version inside the call, so a patch on the
        # importing module would never be seen. An arbitrary sentinel also means
        # a hard-coded literal cannot satisfy the assertion.
        with patch("crewai_core.version.get_crewai_version", return_value=sentinel):
            getattr(instance, method)(*args)

        assert _attributes(span).get("crewai_version") == sentinel, (
            f"{method} did not record the value returned by get_crewai_version()"
        )