fix: allow messages be empty on LLMCallCompletedEvent

This is a experimental feature we still need refine it further

src/crewai/crew.py

@@ -1313,6 +1313,7 @@ class Crew(FlowTrackable, BaseModel):
         n_iterations: int,
         eval_llm: Union[str, InstanceOf[BaseLLM]],
         inputs: Optional[Dict[str, Any]] = None,
+        include_agent_eval: Optional[bool] = False
     ) -> None:
         """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures."""
         try:
@@ -1331,13 +1332,29 @@ class Crew(FlowTrackable, BaseModel):
                 ),
             )
             test_crew = self.copy()
+
+            # TODO: Refator to use a single Evaluator Manage class
             evaluator = CrewEvaluator(test_crew, llm_instance)
 
+            if include_agent_eval:
+                from crewai.evaluation import create_default_evaluator
+                agent_evaluator = create_default_evaluator(crew=test_crew)
+
             for i in range(1, n_iterations + 1):
                 evaluator.set_iteration(i)
+
+                if include_agent_eval:
+                    agent_evaluator.set_iteration(i)
+
                 test_crew.kickoff(inputs=inputs)
 
+                # TODO: Refactor to use ListenerEvents instead of trigger each iteration manually
+                if include_agent_eval:
+                    agent_evaluator.evaluate_current_iteration()
+
             evaluator.print_crew_evaluation_result()
+            if include_agent_eval:
+                agent_evaluator.get_agent_evaluation(include_evaluation_feedback=True)
 
             crewai_event_bus.emit(
                 self,

src/crewai/evaluation/__init__.py

@@ -0,0 +1,53 @@
+from crewai.evaluation.base_evaluator import (
+    BaseEvaluator,
+    EvaluationScore,
+    MetricCategory,
+    AgentEvaluationResult
+)
+
+from crewai.evaluation.metrics.semantic_quality_metrics import (
+    SemanticQualityEvaluator
+)
+
+from crewai.evaluation.metrics.goal_metrics import (
+    GoalAlignmentEvaluator
+)
+
+from crewai.evaluation.metrics.reasoning_metrics import (
+    ReasoningEfficiencyEvaluator
+)
+
+
+from crewai.evaluation.metrics.tools_metrics import (
+    ToolSelectionEvaluator,
+    ParameterExtractionEvaluator,
+    ToolInvocationEvaluator
+)
+
+from crewai.evaluation.evaluation_listener import (
+    EvaluationTraceCallback,
+    create_evaluation_callbacks
+)
+
+
+from crewai.evaluation.agent_evaluator import (
+    AgentEvaluator,
+    create_default_evaluator
+)
+
+__all__ = [
+    "BaseEvaluator",
+    "EvaluationScore",
+    "MetricCategory",
+    "AgentEvaluationResult",
+    "SemanticQualityEvaluator",
+    "GoalAlignmentEvaluator",
+    "ReasoningEfficiencyEvaluator",
+    "ToolSelectionEvaluator",
+    "ParameterExtractionEvaluator",
+    "ToolInvocationEvaluator",
+    "EvaluationTraceCallback",
+    "create_evaluation_callbacks",
+    "AgentEvaluator",
+    "create_default_evaluator"
+]

src/crewai/evaluation/agent_evaluator.py

@@ -0,0 +1,178 @@
+from crewai.evaluation.base_evaluator import AgentEvaluationResult, AggregationStrategy
+from crewai.agent import Agent
+from crewai.task import Task
+from crewai.evaluation.evaluation_display import EvaluationDisplayFormatter
+
+from typing import Any, Dict
+from collections import defaultdict
+from crewai.evaluation import BaseEvaluator, create_evaluation_callbacks
+from collections.abc import Sequence
+from crewai.crew import Crew
+from crewai.utilities.events.crewai_event_bus import crewai_event_bus
+from crewai.utilities.events.utils.console_formatter import ConsoleFormatter
+
+class AgentEvaluator:
+    def __init__(
+        self,
+        evaluators: Sequence[BaseEvaluator] | None = None,
+        crew: Crew | None = None,
+    ):
+        self.crew: Crew | None = crew
+        self.evaluators: Sequence[BaseEvaluator] | None = evaluators
+
+        self.agent_evaluators: dict[str, Sequence[BaseEvaluator] | None] = {}
+        if crew is not None:
+            assert crew and crew.agents is not None
+            for agent in crew.agents:
+                self.agent_evaluators[str(agent.id)] = self.evaluators
+
+        self.callback = create_evaluation_callbacks()
+        self.console_formatter = ConsoleFormatter()
+        self.display_formatter = EvaluationDisplayFormatter()
+
+        self.iteration = 1
+        self.iterations_results: dict[int, dict[str, list[AgentEvaluationResult]]] = {}
+
+    def set_iteration(self, iteration: int) -> None:
+        self.iteration = iteration
+
+    def evaluate_current_iteration(self) -> dict[str, list[AgentEvaluationResult]]:
+        if not self.crew:
+            raise ValueError("Cannot evaluate: no crew was provided to the evaluator.")
+
+        if not self.callback:
+            raise ValueError("Cannot evaluate: no callback was set. Use set_callback() method first.")
+
+        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
+        evaluation_results: defaultdict[str, list[AgentEvaluationResult]] = defaultdict(list)
+
+        total_evals = 0
+        for agent in self.crew.agents:
+            for task in self.crew.tasks:
+                if task.agent and task.agent.id == agent.id and self.agent_evaluators.get(str(agent.id)):
+                    total_evals += 1
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]{task.description}[/bold blue]"),
+            BarColumn(),
+            TextColumn("{task.percentage:.0f}% completed"),
+            console=self.console_formatter.console
+        ) as progress:
+            eval_task = progress.add_task(f"Evaluating agents (iteration {self.iteration})...", total=total_evals)
+
+            for agent in self.crew.agents:
+                evaluator = self.agent_evaluators.get(str(agent.id))
+                if not evaluator:
+                    continue
+
+                for task in self.crew.tasks:
+
+                    if task.agent and str(task.agent.id) != str(agent.id):
+                        continue
+
+                    trace = self.callback.get_trace(str(agent.id), str(task.id))
+                    if not trace:
+                        self.console_formatter.print(f"[yellow]Warning: No trace found for agent {agent.role} on task {task.description[:30]}...[/yellow]")
+                        progress.update(eval_task, advance=1)
+                        continue
+
+                    with crewai_event_bus.scoped_handlers():
+                        result = self.evaluate(
+                            agent=agent,
+                            task=task,
+                            execution_trace=trace,
+                            final_output=task.output
+                        )
+                        evaluation_results[agent.role].append(result)
+                        progress.update(eval_task, advance=1)
+
+        self.iterations_results[self.iteration] = evaluation_results
+        return evaluation_results
+
+    def get_evaluation_results(self):
+        if self.iteration in self.iterations_results:
+            return self.iterations_results[self.iteration]
+
+        return self.evaluate_current_iteration()
+
+    def display_results_with_iterations(self):
+        self.display_formatter.display_summary_results(self.iterations_results)
+
+    def get_agent_evaluation(self, strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE, include_evaluation_feedback: bool = False):
+        agent_results = {}
+        with crewai_event_bus.scoped_handlers():
+            task_results = self.get_evaluation_results()
+            for agent_role, results in task_results.items():
+                if not results:
+                    continue
+
+                agent_id = results[0].agent_id
+
+                aggregated_result = self.display_formatter._aggregate_agent_results(
+                    agent_id=agent_id,
+                    agent_role=agent_role,
+                    results=results,
+                    strategy=strategy
+                )
+
+                agent_results[agent_role] = aggregated_result
+
+
+            if self.iteration == max(self.iterations_results.keys()):
+                self.display_results_with_iterations()
+
+            if include_evaluation_feedback:
+                self.display_evaluation_with_feedback()
+
+        return agent_results
+
+    def display_evaluation_with_feedback(self):
+        self.display_formatter.display_evaluation_with_feedback(self.iterations_results)
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: Any
+    ) -> AgentEvaluationResult:
+        result = AgentEvaluationResult(
+            agent_id=str(agent.id),
+            task_id=str(task.id)
+        )
+        assert self.evaluators is not None
+        for evaluator in self.evaluators:
+            try:
+                score = evaluator.evaluate(
+                    agent=agent,
+                    task=task,
+                    execution_trace=execution_trace,
+                    final_output=final_output
+                )
+                result.metrics[evaluator.metric_category] = score
+            except Exception as e:
+                self.console_formatter.print(f"Error in {evaluator.metric_category.value} evaluator: {str(e)}")
+
+        return result
+
+def create_default_evaluator(crew, llm=None):
+    from crewai.evaluation import (
+        GoalAlignmentEvaluator,
+        SemanticQualityEvaluator,
+        ToolSelectionEvaluator,
+        ParameterExtractionEvaluator,
+        ToolInvocationEvaluator,
+        ReasoningEfficiencyEvaluator
+    )
+
+    evaluators = [
+        GoalAlignmentEvaluator(llm=llm),
+        SemanticQualityEvaluator(llm=llm),
+        ToolSelectionEvaluator(llm=llm),
+        ParameterExtractionEvaluator(llm=llm),
+        ToolInvocationEvaluator(llm=llm),
+        ReasoningEfficiencyEvaluator(llm=llm),
+    ]
+
+    return AgentEvaluator(evaluators=evaluators, crew=crew)

src/crewai/evaluation/base_evaluator.py

@@ -0,0 +1,125 @@
+import abc
+import enum
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from crewai.agent import Agent
+from crewai.task import Task
+from crewai.llm import BaseLLM
+from crewai.utilities.llm_utils import create_llm
+
+class MetricCategory(enum.Enum):
+    GOAL_ALIGNMENT = "goal_alignment"
+    SEMANTIC_QUALITY = "semantic_quality"
+    REASONING_EFFICIENCY = "reasoning_efficiency"
+    TOOL_SELECTION = "tool_selection"
+    PARAMETER_EXTRACTION = "parameter_extraction"
+    TOOL_INVOCATION = "tool_invocation"
+
+    def title(self):
+        return self.value.replace('_', ' ').title()
+
+
+class EvaluationScore(BaseModel):
+    score: float | None = Field(
+        default=5.0,
+        description="Numeric score from 0-10 where 0 is worst and 10 is best, None if not applicable",
+        ge=0.0,
+        le=10.0
+    )
+    feedback: str = Field(
+        default="",
+        description="Detailed feedback explaining the evaluation score"
+    )
+    raw_response: str | None = Field(
+        default=None,
+        description="Raw response from the evaluator (e.g., LLM)"
+    )
+
+    def __str__(self) -> str:
+        if self.score is None:
+            return f"Score: N/A - {self.feedback}"
+        return f"Score: {self.score:.1f}/10 - {self.feedback}"
+
+
+class BaseEvaluator(abc.ABC):
+    def __init__(self, llm: BaseLLM | None = None):
+        self.llm: BaseLLM | None = create_llm(llm)
+
+    @property
+    @abc.abstractmethod
+    def metric_category(self) -> MetricCategory:
+        pass
+
+    @abc.abstractmethod
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: Any,
+    ) -> EvaluationScore:
+        pass
+
+
+class AgentEvaluationResult(BaseModel):
+    agent_id: str = Field(description="ID of the evaluated agent")
+    task_id: str = Field(description="ID of the task that was executed")
+    metrics: Dict[MetricCategory, EvaluationScore] = Field(
+        default_factory=dict,
+        description="Evaluation scores for each metric category"
+    )
+
+
+class AggregationStrategy(Enum):
+    SIMPLE_AVERAGE = "simple_average"  # Equal weight to all tasks
+    WEIGHTED_BY_COMPLEXITY = "weighted_by_complexity"  # Weight by task complexity
+    BEST_PERFORMANCE = "best_performance"  # Use best scores across tasks
+    WORST_PERFORMANCE = "worst_performance"  # Use worst scores across tasks
+
+
+class AgentAggregatedEvaluationResult(BaseModel):
+    agent_id: str = Field(
+        default="",
+        description="ID of the agent"
+    )
+    agent_role: str = Field(
+        default="",
+        description="Role of the agent"
+    )
+    task_count: int = Field(
+        default=0,
+        description="Number of tasks included in this aggregation"
+    )
+    aggregation_strategy: AggregationStrategy = Field(
+        default=AggregationStrategy.SIMPLE_AVERAGE,
+        description="Strategy used for aggregation"
+    )
+    metrics: Dict[MetricCategory, EvaluationScore] = Field(
+        default_factory=dict,
+        description="Aggregated metrics across all tasks"
+    )
+    task_results: List[str] = Field(
+        default_factory=list,
+        description="IDs of tasks included in this aggregation"
+    )
+    overall_score: Optional[float] = Field(
+        default=None,
+        description="Overall score for this agent"
+    )
+
+    def __str__(self) -> str:
+        result = f"Agent Evaluation: {self.agent_role}\n"
+        result += f"Strategy: {self.aggregation_strategy.value}\n"
+        result += f"Tasks evaluated: {self.task_count}\n"
+
+        for category, score in self.metrics.items():
+            result += f"\n\n- {category.value.upper()}: {score.score}/10\n"
+
+            if score.feedback:
+                detailed_feedback = "\n  ".join(score.feedback.split('\n'))
+                result += f"  {detailed_feedback}\n"
+
+        return result

src/crewai/evaluation/evaluation_display.py

@@ -0,0 +1,341 @@
+from collections import defaultdict
+from typing import Dict, Any, List
+from rich.table import Table
+from rich.box import HEAVY_EDGE, ROUNDED
+from collections.abc import Sequence
+from crewai.evaluation.base_evaluator import AgentAggregatedEvaluationResult, AggregationStrategy, AgentEvaluationResult, MetricCategory
+from crewai.evaluation import EvaluationScore
+from crewai.utilities.events.utils.console_formatter import ConsoleFormatter
+from crewai.utilities.llm_utils import create_llm
+
+class EvaluationDisplayFormatter:
+    def __init__(self):
+        self.console_formatter = ConsoleFormatter()
+
+    def display_evaluation_with_feedback(self, iterations_results: Dict[int, Dict[str, List[Any]]]):
+        if not iterations_results:
+            self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")
+            return
+
+        # Get all agent roles across all iterations
+        all_agent_roles: set[str] = set()
+        for iter_results in iterations_results.values():
+            all_agent_roles.update(iter_results.keys())
+
+        for agent_role in sorted(all_agent_roles):
+            self.console_formatter.print(f"\n[bold cyan]Agent: {agent_role}[/bold cyan]")
+
+            # Process each iteration
+            for iter_num, results in sorted(iterations_results.items()):
+                if agent_role not in results or not results[agent_role]:
+                    continue
+
+                agent_results = results[agent_role]
+                agent_id = agent_results[0].agent_id
+
+                # Aggregate results for this agent in this iteration
+                aggregated_result = self._aggregate_agent_results(
+                    agent_id=agent_id,
+                    agent_role=agent_role,
+                    results=agent_results,
+                )
+
+                # Display iteration header
+                self.console_formatter.print(f"\n[bold]Iteration {iter_num}[/bold]")
+
+                # Create table for this iteration
+                table = Table(box=ROUNDED)
+                table.add_column("Metric", style="cyan")
+                table.add_column("Score (1-10)", justify="center")
+                table.add_column("Feedback", style="green")
+
+                # Add metrics to table
+                if aggregated_result.metrics:
+                    for metric, evaluation_score in aggregated_result.metrics.items():
+                        score = evaluation_score.score
+
+                        if isinstance(score, (int, float)):
+                            if score >= 8.0:
+                                score_text = f"[green]{score:.1f}[/green]"
+                            elif score >= 6.0:
+                                score_text = f"[cyan]{score:.1f}[/cyan]"
+                            elif score >= 4.0:
+                                score_text = f"[yellow]{score:.1f}[/yellow]"
+                            else:
+                                score_text = f"[red]{score:.1f}[/red]"
+                        else:
+                            score_text = "[dim]N/A[/dim]"
+
+                        table.add_section()
+                        table.add_row(
+                            metric.title(),
+                            score_text,
+                            evaluation_score.feedback or ""
+                        )
+
+                if aggregated_result.overall_score is not None:
+                    overall_score = aggregated_result.overall_score
+                    if overall_score >= 8.0:
+                        overall_color = "green"
+                    elif overall_score >= 6.0:
+                        overall_color = "cyan"
+                    elif overall_score >= 4.0:
+                        overall_color = "yellow"
+                    else:
+                        overall_color = "red"
+
+                    table.add_section()
+                    table.add_row(
+                        "Overall Score",
+                        f"[{overall_color}]{overall_score:.1f}[/]",
+                        "Overall agent evaluation score"
+                    )
+
+                # Print the table for this iteration
+                self.console_formatter.print(table)
+
+    def display_summary_results(self, iterations_results: Dict[int, Dict[str, List[AgentAggregatedEvaluationResult]]]):
+        if not iterations_results:
+            self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")
+            return
+
+        self.console_formatter.print("\n")
+
+        table = Table(title="Agent Performance Scores \n (1-10 Higher is better)", box=HEAVY_EDGE)
+
+        table.add_column("Agent/Metric", style="cyan")
+
+        for iter_num in sorted(iterations_results.keys()):
+            run_label = f"Run {iter_num}"
+            table.add_column(run_label, justify="center")
+
+        table.add_column("Avg. Total", justify="center")
+
+        all_agent_roles: set[str] = set()
+        for results in iterations_results.values():
+            all_agent_roles.update(results.keys())
+
+        for agent_role in sorted(all_agent_roles):
+            agent_scores_by_iteration = {}
+            agent_metrics_by_iteration = {}
+
+            for iter_num, results in sorted(iterations_results.items()):
+                if agent_role not in results or not results[agent_role]:
+                    continue
+
+                agent_results = results[agent_role]
+                agent_id = agent_results[0].agent_id
+
+                aggregated_result = self._aggregate_agent_results(
+                    agent_id=agent_id,
+                    agent_role=agent_role,
+                    results=agent_results,
+                    strategy=AggregationStrategy.SIMPLE_AVERAGE
+                )
+
+                valid_scores = [score.score for score in aggregated_result.metrics.values()
+                               if score.score is not None]
+                if valid_scores:
+                    avg_score = sum(valid_scores) / len(valid_scores)
+                    agent_scores_by_iteration[iter_num] = avg_score
+
+                agent_metrics_by_iteration[iter_num] = aggregated_result.metrics
+
+            if not agent_scores_by_iteration:
+                continue
+
+            avg_across_iterations = sum(agent_scores_by_iteration.values()) / len(agent_scores_by_iteration)
+
+            row = [f"[bold]{agent_role}[/bold]"]
+
+            for iter_num in sorted(iterations_results.keys()):
+                if iter_num in agent_scores_by_iteration:
+                    score = agent_scores_by_iteration[iter_num]
+                    if score >= 8.0:
+                        color = "green"
+                    elif score >= 6.0:
+                        color = "cyan"
+                    elif score >= 4.0:
+                        color = "yellow"
+                    else:
+                        color = "red"
+                    row.append(f"[bold {color}]{score:.1f}[/]")
+                else:
+                    row.append("-")
+
+            if avg_across_iterations >= 8.0:
+                color = "green"
+            elif avg_across_iterations >= 6.0:
+                color = "cyan"
+            elif avg_across_iterations >= 4.0:
+                color = "yellow"
+            else:
+                color = "red"
+            row.append(f"[bold {color}]{avg_across_iterations:.1f}[/]")
+
+            table.add_row(*row)
+
+            all_metrics: set[Any] = set()
+            for metrics in agent_metrics_by_iteration.values():
+                all_metrics.update(metrics.keys())
+
+            for metric in sorted(all_metrics, key=lambda x: x.value):
+                metric_scores = []
+
+                row = [f"  - {metric.title()}"]
+
+                for iter_num in sorted(iterations_results.keys()):
+                    if (iter_num in agent_metrics_by_iteration and
+                            metric in agent_metrics_by_iteration[iter_num]):
+                        metric_score = agent_metrics_by_iteration[iter_num][metric].score
+                        if metric_score is not None:
+                            metric_scores.append(metric_score)
+                            if metric_score >= 8.0:
+                                color = "green"
+                            elif metric_score >= 6.0:
+                                color = "cyan"
+                            elif metric_score >= 4.0:
+                                color = "yellow"
+                            else:
+                                color = "red"
+                            row.append(f"[{color}]{metric_score:.1f}[/]")
+                        else:
+                            row.append("[dim]N/A[/dim]")
+                    else:
+                        row.append("-")
+
+                if metric_scores:
+                    avg = sum(metric_scores) / len(metric_scores)
+                    if avg >= 8.0:
+                        color = "green"
+                    elif avg >= 6.0:
+                        color = "cyan"
+                    elif avg >= 4.0:
+                        color = "yellow"
+                    else:
+                        color = "red"
+                    row.append(f"[{color}]{avg:.1f}[/]")
+                else:
+                    row.append("-")
+
+                table.add_row(*row)
+
+            table.add_row(*[""] * (len(sorted(iterations_results.keys())) + 2))
+
+        self.console_formatter.print(table)
+        self.console_formatter.print("\n")
+
+    def _aggregate_agent_results(
+        self,
+        agent_id: str,
+        agent_role: str,
+        results: Sequence[AgentEvaluationResult],
+        strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE,
+    ) -> AgentAggregatedEvaluationResult:
+        metrics_by_category: dict[MetricCategory, list[EvaluationScore]] = defaultdict(list)
+
+        for result in results:
+            for metric_name, evaluation_score in result.metrics.items():
+                metrics_by_category[metric_name].append(evaluation_score)
+
+        aggregated_metrics: dict[MetricCategory, EvaluationScore] = {}
+        for category, scores in metrics_by_category.items():
+            valid_scores = [s.score for s in scores if s.score is not None]
+            avg_score = sum(valid_scores) / len(valid_scores) if valid_scores else None
+
+            feedbacks = [s.feedback for s in scores if s.feedback]
+
+            feedback_summary = None
+            if feedbacks:
+                if len(feedbacks) > 1:
+                    # Use the summarization method for multiple feedbacks
+                    feedback_summary = self._summarize_feedbacks(
+                        agent_role=agent_role,
+                        metric=category.title(),
+                        feedbacks=feedbacks,
+                        scores=[s.score for s in scores],
+                        strategy=strategy
+                    )
+                else:
+                    feedback_summary = feedbacks[0]
+
+            aggregated_metrics[category] = EvaluationScore(
+                score=avg_score,
+                feedback=feedback_summary
+            )
+
+        overall_score = None
+        if aggregated_metrics:
+            valid_scores = [m.score for m in aggregated_metrics.values() if m.score is not None]
+            if valid_scores:
+                overall_score = sum(valid_scores) / len(valid_scores)
+
+        return AgentAggregatedEvaluationResult(
+            agent_id=agent_id,
+            agent_role=agent_role,
+            metrics=aggregated_metrics,
+            overall_score=overall_score,
+            task_count=len(results),
+            aggregation_strategy=strategy
+        )
+
+    def _summarize_feedbacks(
+        self,
+        agent_role: str,
+        metric: str,
+        feedbacks: List[str],
+        scores: List[float | None],
+        strategy: AggregationStrategy
+    ) -> str:
+        if len(feedbacks) <= 2 and all(len(fb) < 200 for fb in feedbacks):
+            return "\n\n".join([f"Feedback {i+1}: {fb}" for i, fb in enumerate(feedbacks)])
+
+        try:
+            llm = create_llm()
+
+            formatted_feedbacks = []
+            for i, (feedback, score) in enumerate(zip(feedbacks, scores)):
+                if len(feedback) > 500:
+                    feedback = feedback[:500] + "..."
+                score_text = f"{score:.1f}" if score is not None else "N/A"
+                formatted_feedbacks.append(f"Feedback #{i+1} (Score: {score_text}):\n{feedback}")
+
+            all_feedbacks = "\n\n" + "\n\n---\n\n".join(formatted_feedbacks)
+
+            strategy_guidance = ""
+            if strategy == AggregationStrategy.BEST_PERFORMANCE:
+                strategy_guidance = "Focus on the highest-scoring aspects and strengths demonstrated."
+            elif strategy == AggregationStrategy.WORST_PERFORMANCE:
+                strategy_guidance = "Focus on areas that need improvement and common issues across tasks."
+            else:  # Default/average strategies
+                strategy_guidance = "Provide a balanced analysis of strengths and weaknesses across all tasks."
+
+            prompt = [
+                {"role": "system", "content": f"""You are an expert evaluator creating a comprehensive summary of agent performance feedback.
+                Your job is to synthesize multiple feedback points about the same metric across different tasks.
+
+                Create a concise, insightful summary that captures the key patterns and themes from all feedback.
+                {strategy_guidance}
+
+                Your summary should be:
+                1. Specific and concrete (not vague or general)
+                2. Focused on actionable insights
+                3. Highlighting patterns across tasks
+                4. 150-250 words in length
+
+                The summary should be directly usable as final feedback for the agent's performance on this metric."""},
+                {"role": "user", "content": f"""I need a synthesized summary of the following feedback for:
+
+                Agent Role: {agent_role}
+                Metric: {metric.title()}
+
+                {all_feedbacks}
+                """}
+            ]
+            assert llm is not None
+            response = llm.call(prompt)
+
+            return response
+
+        except Exception:
+            return "Synthesized from multiple tasks: " + "\n\n".join([f"- {fb[:500]}..." for fb in feedbacks])

src/crewai/evaluation/evaluation_listener.py

@@ -0,0 +1,190 @@
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from collections.abc import Sequence
+
+from crewai.agent import Agent
+from crewai.task import Task
+from crewai.utilities.events.base_event_listener import BaseEventListener
+from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus
+from crewai.utilities.events.agent_events import (
+    AgentExecutionStartedEvent,
+    AgentExecutionCompletedEvent
+)
+from crewai.utilities.events.tool_usage_events import (
+    ToolUsageFinishedEvent,
+    ToolUsageErrorEvent,
+    ToolExecutionErrorEvent,
+    ToolSelectionErrorEvent,
+    ToolValidateInputErrorEvent
+)
+from crewai.utilities.events.llm_events import (
+    LLMCallStartedEvent,
+    LLMCallCompletedEvent
+)
+
+class EvaluationTraceCallback(BaseEventListener):
+    """Event listener for collecting execution traces for evaluation.
+
+    This listener attaches to the event bus to collect detailed information
+    about the execution process, including agent steps, tool uses, knowledge
+    retrievals, and final output - all for use in agent evaluation.
+    """
+
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if not hasattr(self, "_initialized") or not self._initialized:
+            super().__init__()
+            self.traces = {}
+            self.current_agent_id = None
+            self.current_task_id = None
+            self._initialized = True
+
+    def setup_listeners(self, event_bus: CrewAIEventsBus):
+        @event_bus.on(AgentExecutionStartedEvent)
+        def on_agent_started(source, event: AgentExecutionStartedEvent):
+            self.on_agent_start(event.agent, event.task)
+
+        @event_bus.on(AgentExecutionCompletedEvent)
+        def on_agent_completed(source, event: AgentExecutionCompletedEvent):
+            self.on_agent_finish(event.agent, event.task, event.output)
+
+        @event_bus.on(ToolUsageFinishedEvent)
+        def on_tool_completed(source, event: ToolUsageFinishedEvent):
+            self.on_tool_use(event.tool_name, event.tool_args, event.output, success=True)
+
+        @event_bus.on(ToolUsageErrorEvent)
+        def on_tool_usage_error(source, event: ToolUsageErrorEvent):
+            self.on_tool_use(event.tool_name, event.tool_args, event.error,
+                           success=False, error_type="usage_error")
+
+        @event_bus.on(ToolExecutionErrorEvent)
+        def on_tool_execution_error(source, event: ToolExecutionErrorEvent):
+            self.on_tool_use(event.tool_name, event.tool_args, event.error,
+                           success=False, error_type="execution_error")
+
+        @event_bus.on(ToolSelectionErrorEvent)
+        def on_tool_selection_error(source, event: ToolSelectionErrorEvent):
+            self.on_tool_use(event.tool_name, event.tool_args, event.error,
+                           success=False, error_type="selection_error")
+
+        @event_bus.on(ToolValidateInputErrorEvent)
+        def on_tool_validate_input_error(source, event: ToolValidateInputErrorEvent):
+            self.on_tool_use(event.tool_name, event.tool_args, event.error,
+                           success=False, error_type="validation_error")
+
+        @event_bus.on(LLMCallStartedEvent)
+        def on_llm_call_started(source, event: LLMCallStartedEvent):
+            self.on_llm_call_start(event.messages, event.tools)
+
+        @event_bus.on(LLMCallCompletedEvent)
+        def on_llm_call_completed(source, event: LLMCallCompletedEvent):
+            self.on_llm_call_end(event.messages, event.response)
+
+    def on_agent_start(self, agent: Agent, task: Task):
+        self.current_agent_id = agent.id
+        self.current_task_id = task.id
+
+        trace_key = f"{agent.id}_{task.id}"
+        self.traces[trace_key] = {
+            "agent_id": agent.id,
+            "task_id": task.id,
+            "tool_uses": [],
+            "llm_calls": [],
+            "start_time": datetime.now(),
+            "final_output": None
+        }
+
+    def on_agent_finish(self, agent: Agent, task: Task, output: Any):
+        trace_key = f"{agent.id}_{task.id}"
+        if trace_key in self.traces:
+            self.traces[trace_key]["final_output"] = output
+            self.traces[trace_key]["end_time"] = datetime.now()
+
+        self.current_agent_id = None
+        self.current_task_id = None
+
+    def on_tool_use(self, tool_name: str, tool_args: dict[str, Any] | str, result: Any,
+                   success: bool = True, error_type: str | None = None):
+        if not self.current_agent_id or not self.current_task_id:
+            return
+
+        trace_key = f"{self.current_agent_id}_{self.current_task_id}"
+        if trace_key in self.traces:
+            tool_use = {
+                "tool": tool_name,
+                "args": tool_args,
+                "result": result,
+                "success": success,
+                "timestamp": datetime.now()
+            }
+
+            # Add error information if applicable
+            if not success and error_type:
+                tool_use["error"] = True
+                tool_use["error_type"] = error_type
+
+            self.traces[trace_key]["tool_uses"].append(tool_use)
+
+    def on_llm_call_start(self, messages: str | Sequence[dict[str, Any]] | None, tools: Sequence[dict[str, Any]] | None = None):
+        if not self.current_agent_id or not self.current_task_id:
+            return
+
+        trace_key = f"{self.current_agent_id}_{self.current_task_id}"
+        if trace_key not in self.traces:
+            return
+
+        self.current_llm_call = {
+            "messages": messages,
+            "tools": tools,
+            "start_time": datetime.now(),
+            "response": None,
+            "end_time": None
+        }
+
+    def on_llm_call_end(self, messages: str | list[dict[str, Any]] | None, response: Any):
+        if not self.current_agent_id or not self.current_task_id:
+            return
+
+        trace_key = f"{self.current_agent_id}_{self.current_task_id}"
+        if trace_key not in self.traces:
+            return
+
+        total_tokens = 0
+        if hasattr(response, "usage") and hasattr(response.usage, "total_tokens"):
+            total_tokens = response.usage.total_tokens
+
+        current_time = datetime.now()
+        start_time = None
+        if hasattr(self, "current_llm_call") and self.current_llm_call:
+            start_time = self.current_llm_call.get("start_time")
+
+        if not start_time:
+            start_time = current_time
+        llm_call = {
+            "messages": messages,
+            "response": response,
+            "start_time": start_time,
+            "end_time": current_time,
+            "total_tokens": total_tokens
+        }
+
+        self.traces[trace_key]["llm_calls"].append(llm_call)
+
+        if hasattr(self, "current_llm_call"):
+            self.current_llm_call = {}
+
+    def get_trace(self, agent_id: str, task_id: str) -> Optional[Dict[str, Any]]:
+        trace_key = f"{agent_id}_{task_id}"
+        return self.traces.get(trace_key)
+
+
+def create_evaluation_callbacks() -> EvaluationTraceCallback:
+    return EvaluationTraceCallback()

src/crewai/evaluation/json_parser.py

@@ -0,0 +1,30 @@
+"""Robust JSON parsing utilities for evaluation responses."""
+
+import json
+import re
+from typing import Any
+
+
+def extract_json_from_llm_response(text: str) -> dict[str, Any]:
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    json_patterns = [
+        # Standard markdown code blocks with json
+        r'```json\s*([\s\S]*?)\s*```',
+        # Code blocks without language specifier
+        r'```\s*([\s\S]*?)\s*```',
+        # Inline code with JSON
+        r'`([{\\[].*[}\]])`',
+    ]
+
+    for pattern in json_patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE | re.DOTALL)
+        for match in matches:
+            try:
+                return json.loads(match.strip())
+            except json.JSONDecodeError:
+                continue
+    raise ValueError("No valid JSON found in the response")

src/crewai/evaluation/metrics/goal_metrics.py

@@ -0,0 +1,66 @@
+from typing import Any, Dict
+
+from crewai.agent import Agent
+from crewai.task import Task
+
+from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory
+from crewai.evaluation.json_parser import extract_json_from_llm_response
+
+class GoalAlignmentEvaluator(BaseEvaluator):
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.GOAL_ALIGNMENT
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: Any,
+    ) -> EvaluationScore:
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing how well an AI agent's output aligns with its assigned task goal.
+
+Score the agent's goal alignment on a scale from 0-10 where:
+- 0: Complete misalignment, agent did not understand or attempt the task goal
+- 5: Partial alignment, agent attempted the task but missed key requirements
+- 10: Perfect alignment, agent fully satisfied all task requirements
+
+Consider:
+1. Did the agent correctly interpret the task goal?
+2. Did the final output directly address the requirements?
+3. Did the agent focus on relevant aspects of the task?
+4. Did the agent provide all requested information or deliverables?
+
+Return your evaluation as JSON with fields 'score' (number) and 'feedback' (string).
+"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Agent goal: {agent.goal}
+Task description: {task.description}
+Expected output: {task.expected_output}
+
+Agent's final output:
+{final_output}
+
+Evaluate how well the agent's output aligns with the assigned task goal.
+"""}
+        ]
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data: dict[str, Any] = extract_json_from_llm_response(response)
+            assert evaluation_data is not None
+
+            return EvaluationScore(
+                score=evaluation_data.get("score", 0),
+                feedback=evaluation_data.get("feedback", response),
+                raw_response=response
+            )
+        except Exception:
+            return EvaluationScore(
+                score=None,
+                feedback=f"Failed to parse evaluation. Raw response: {response}",
+                raw_response=response
+            )

src/crewai/evaluation/metrics/reasoning_metrics.py

@@ -0,0 +1,355 @@
+"""Agent reasoning efficiency evaluators.
+
+This module provides evaluator implementations for:
+- Reasoning efficiency
+- Loop detection
+- Thinking-to-action ratio
+"""
+
+import logging
+import re
+from enum import Enum
+from typing import Any, Dict, List, Tuple
+import numpy as np
+from collections.abc import Sequence
+
+from crewai.agent import Agent
+from crewai.task import Task
+
+from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory
+from crewai.evaluation.json_parser import extract_json_from_llm_response
+from crewai.tasks.task_output import TaskOutput
+
+class ReasoningPatternType(Enum):
+    EFFICIENT = "efficient"  # Good reasoning flow
+    LOOP = "loop"  # Agent is stuck in a loop
+    VERBOSE = "verbose"  # Agent is unnecessarily verbose
+    INDECISIVE = "indecisive"  # Agent struggles to make decisions
+    SCATTERED = "scattered"  # Agent jumps between topics without focus
+
+
+class ReasoningEfficiencyEvaluator(BaseEvaluator):
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.REASONING_EFFICIENCY
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: TaskOutput,
+    ) -> EvaluationScore:
+        llm_calls = execution_trace.get("llm_calls", [])
+
+        if not llm_calls or len(llm_calls) < 2:
+            return EvaluationScore(
+                score=None,
+                feedback="Insufficient LLM calls to evaluate reasoning efficiency."
+            )
+
+        total_calls = len(llm_calls)
+        total_tokens = sum(call.get("total_tokens", 0) for call in llm_calls)
+        avg_tokens_per_call = total_tokens / total_calls if total_calls > 0 else 0
+        time_intervals = []
+        has_reliable_timing = True
+        for i in range(1, len(llm_calls)):
+            start_time = llm_calls[i-1].get("end_time")
+            end_time = llm_calls[i].get("start_time")
+            if start_time and end_time and start_time != end_time:
+                try:
+                    interval = end_time - start_time
+                    time_intervals.append(interval.total_seconds() if hasattr(interval, 'total_seconds') else 0)
+                except Exception:
+                    has_reliable_timing = False
+            else:
+                has_reliable_timing = False
+
+        loop_detected, loop_details = self._detect_loops(llm_calls)
+        pattern_analysis = self._analyze_reasoning_patterns(llm_calls)
+
+        efficiency_metrics = {
+            "total_llm_calls": total_calls,
+            "total_tokens": total_tokens,
+            "avg_tokens_per_call": avg_tokens_per_call,
+            "reasoning_pattern": pattern_analysis["primary_pattern"].value,
+            "loops_detected": loop_detected,
+        }
+
+        if has_reliable_timing and time_intervals:
+            efficiency_metrics["avg_time_between_calls"] = np.mean(time_intervals)
+
+        loop_info = f"Detected {len(loop_details)} potential reasoning loops." if loop_detected else "No significant reasoning loops detected."
+
+        call_samples = self._get_call_samples(llm_calls)
+
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing the reasoning efficiency of an AI agent's thought process.
+
+Evaluate the agent's reasoning efficiency across these five key subcategories:
+
+1. Focus (0-10): How well the agent stays on topic and avoids unnecessary tangents
+2. Progression (0-10): How effectively the agent builds on previous thoughts rather than repeating or circling
+3. Decision Quality (0-10): How decisively and appropriately the agent makes decisions
+4. Conciseness (0-10): How efficiently the agent communicates without unnecessary verbosity
+5. Loop Avoidance (0-10): How well the agent avoids getting stuck in repetitive thinking patterns
+
+For each subcategory, provide a score from 0-10 where:
+- 0: Completely inefficient
+- 5: Moderately efficient
+- 10: Highly efficient
+
+The overall score should be a weighted average of these subcategories.
+
+Return your evaluation as JSON with the following structure:
+{
+    "overall_score": float,
+    "scores": {
+        "focus": float,
+        "progression": float,
+        "decision_quality": float,
+        "conciseness": float,
+        "loop_avoidance": float
+    },
+    "feedback": string (general feedback about overall reasoning efficiency),
+    "optimization_suggestions": string (concrete suggestions for improving reasoning efficiency),
+    "detected_patterns": string (describe any inefficient reasoning patterns you observe)
+}"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Task description: {task.description}
+
+Reasoning efficiency metrics:
+- Total LLM calls: {efficiency_metrics["total_llm_calls"]}
+- Average tokens per call: {efficiency_metrics["avg_tokens_per_call"]:.1f}
+- Primary reasoning pattern: {efficiency_metrics["reasoning_pattern"]}
+- {loop_info}
+{"- Average time between calls: {:.2f} seconds".format(efficiency_metrics.get("avg_time_between_calls", 0)) if "avg_time_between_calls" in efficiency_metrics else ""}
+
+Sample of agent reasoning flow (chronological sequence):
+{call_samples}
+
+Agent's final output:
+{final_output.raw[:500]}... (truncated)
+
+Evaluate the reasoning efficiency of this agent based on these interaction patterns.
+Identify any inefficient reasoning patterns and provide specific suggestions for optimization.
+"""}
+        ]
+
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data = extract_json_from_llm_response(response)
+
+            scores = evaluation_data.get("scores", {})
+            focus = scores.get("focus", 5.0)
+            progression = scores.get("progression", 5.0)
+            decision_quality = scores.get("decision_quality", 5.0)
+            conciseness = scores.get("conciseness", 5.0)
+            loop_avoidance = scores.get("loop_avoidance", 5.0)
+
+            overall_score = evaluation_data.get("overall_score", evaluation_data.get("score", 5.0))
+            feedback = evaluation_data.get("feedback", "No detailed feedback provided.")
+            optimization_suggestions = evaluation_data.get("optimization_suggestions", "No specific suggestions provided.")
+
+            detailed_feedback = "Reasoning Efficiency Evaluation:\n"
+            detailed_feedback += f"• Focus: {focus}/10 - Staying on topic without tangents\n"
+            detailed_feedback += f"• Progression: {progression}/10 - Building on previous thinking\n"
+            detailed_feedback += f"• Decision Quality: {decision_quality}/10 - Making appropriate decisions\n"
+            detailed_feedback += f"• Conciseness: {conciseness}/10 - Communicating efficiently\n"
+            detailed_feedback += f"• Loop Avoidance: {loop_avoidance}/10 - Avoiding repetitive patterns\n\n"
+
+            detailed_feedback += f"Feedback:\n{feedback}\n\n"
+            detailed_feedback += f"Optimization Suggestions:\n{optimization_suggestions}"
+
+            return EvaluationScore(
+                score=float(overall_score),
+                feedback=detailed_feedback,
+                raw_response=response
+            )
+        except Exception as e:
+            logging.warning(f"Failed to parse reasoning efficiency evaluation: {e}")
+            return EvaluationScore(
+                score=None,
+                feedback=f"Failed to parse reasoning efficiency evaluation. Raw response: {response[:200]}...",
+                raw_response=response
+            )
+
+    def _detect_loops(self, llm_calls: List[Dict]) -> Tuple[bool, List[Dict]]:
+        loop_details = []
+
+        messages = []
+        for call in llm_calls:
+            content = call.get("response", "")
+            if isinstance(content, str):
+                messages.append(content)
+            elif isinstance(content, list) and len(content) > 0:
+                # Handle message list format
+                for msg in content:
+                    if isinstance(msg, dict) and "content" in msg:
+                        messages.append(msg["content"])
+
+        # Simple n-gram based similarity detection
+        # For a more robust implementation, consider using embedding-based similarity
+        for i in range(len(messages) - 2):
+            for j in range(i + 1, len(messages) - 1):
+                # Check for repeated patterns (simplistic approach)
+                # A more sophisticated approach would use semantic similarity
+                similarity = self._calculate_text_similarity(messages[i], messages[j])
+                if similarity > 0.7:  # Arbitrary threshold
+                    loop_details.append({
+                        "first_occurrence": i,
+                        "second_occurrence": j,
+                        "similarity": similarity,
+                        "snippet": messages[i][:100] + "..."
+                    })
+
+        return len(loop_details) > 0, loop_details
+
+    def _calculate_text_similarity(self, text1: str, text2: str) -> float:
+        text1 = re.sub(r'\s+', ' ', text1.lower()).strip()
+        text2 = re.sub(r'\s+', ' ', text2.lower()).strip()
+
+        # Simple Jaccard similarity on word sets
+        words1 = set(text1.split())
+        words2 = set(text2.split())
+
+        intersection = len(words1.intersection(words2))
+        union = len(words1.union(words2))
+
+        return intersection / union if union > 0 else 0.0
+
+    def _analyze_reasoning_patterns(self, llm_calls: List[Dict]) -> Dict[str, Any]:
+        call_lengths = []
+        response_times = []
+
+        for call in llm_calls:
+            content = call.get("response", "")
+            if isinstance(content, str):
+                call_lengths.append(len(content))
+            elif isinstance(content, list) and len(content) > 0:
+                # Handle message list format
+                total_length = 0
+                for msg in content:
+                    if isinstance(msg, dict) and "content" in msg:
+                        total_length += len(msg["content"])
+                call_lengths.append(total_length)
+
+            start_time = call.get("start_time")
+            end_time = call.get("end_time")
+            if start_time and end_time:
+                try:
+                    response_times.append(end_time - start_time)
+                except Exception:
+                    pass
+
+        avg_length = np.mean(call_lengths) if call_lengths else 0
+        std_length = np.std(call_lengths) if call_lengths else 0
+        length_trend = self._calculate_trend(call_lengths)
+
+        primary_pattern = ReasoningPatternType.EFFICIENT
+        details = "Agent demonstrates efficient reasoning patterns."
+
+        loop_score = self._calculate_loop_likelihood(call_lengths, response_times)
+        if loop_score > 0.7:
+            primary_pattern = ReasoningPatternType.LOOP
+            details = "Agent appears to be stuck in repetitive thinking patterns."
+        elif avg_length > 1000 and std_length / avg_length < 0.3:
+            primary_pattern = ReasoningPatternType.VERBOSE
+            details = "Agent is consistently verbose across interactions."
+        elif len(llm_calls) > 10 and length_trend > 0.5:
+            primary_pattern = ReasoningPatternType.INDECISIVE
+            details = "Agent shows signs of indecisiveness with increasing message lengths."
+        elif std_length / avg_length > 0.8:
+            primary_pattern = ReasoningPatternType.SCATTERED
+            details = "Agent shows inconsistent reasoning flow with highly variable responses."
+
+        return {
+            "primary_pattern": primary_pattern,
+            "details": details,
+            "metrics": {
+                "avg_length": avg_length,
+                "std_length": std_length,
+                "length_trend": length_trend,
+                "loop_score": loop_score
+            }
+        }
+
+    def _calculate_trend(self, values: Sequence[float | int]) -> float:
+        if not values or len(values) < 2:
+            return 0.0
+
+        try:
+            x = np.arange(len(values))
+            y = np.array(values)
+
+            # Simple linear regression
+            slope = np.polyfit(x, y, 1)[0]
+
+            # Normalize slope to -1 to 1 range
+            max_possible_slope = max(values) - min(values)
+            if max_possible_slope > 0:
+                normalized_slope = slope / max_possible_slope
+                return max(min(normalized_slope, 1.0), -1.0)
+            return 0.0
+        except Exception:
+            return 0.0
+
+    def _calculate_loop_likelihood(self, call_lengths: Sequence[float], response_times: Sequence[float]) -> float:
+        if not call_lengths or len(call_lengths) < 3:
+            return 0.0
+
+        indicators = []
+
+        if len(call_lengths) >= 4:
+            repeated_lengths = 0
+            for i in range(len(call_lengths) - 2):
+                ratio = call_lengths[i] / call_lengths[i + 2] if call_lengths[i + 2] > 0 else 0
+                if 0.85 <= ratio <= 1.15:
+                    repeated_lengths += 1
+
+            length_repetition_score = repeated_lengths / (len(call_lengths) - 2)
+            indicators.append(length_repetition_score)
+
+        if response_times and len(response_times) >= 3:
+            try:
+                std_time = np.std(response_times)
+                mean_time = np.mean(response_times)
+                if mean_time > 0:
+                    time_consistency = 1.0 - (std_time / mean_time)
+                    indicators.append(max(0, time_consistency - 0.3) * 1.5)
+            except Exception:
+                pass
+
+        return np.mean(indicators) if indicators else 0.0
+
+    def _get_call_samples(self, llm_calls: List[Dict]) -> str:
+        samples = []
+
+        if len(llm_calls) <= 6:
+            sample_indices = list(range(len(llm_calls)))
+        else:
+            sample_indices = [0, 1, len(llm_calls) // 2 - 1, len(llm_calls) // 2,
+                             len(llm_calls) - 2, len(llm_calls) - 1]
+
+        for idx in sample_indices:
+            call = llm_calls[idx]
+            content = call.get("response", "")
+
+            if isinstance(content, str):
+                sample = content
+            elif isinstance(content, list) and len(content) > 0:
+                sample_parts = []
+                for msg in content:
+                    if isinstance(msg, dict) and "content" in msg:
+                        sample_parts.append(msg["content"])
+                sample = "\n".join(sample_parts)
+            else:
+                sample = str(content)
+
+            truncated = sample[:200] + "..." if len(sample) > 200 else sample
+            samples.append(f"Call {idx + 1}:\n{truncated}\n")
+
+        return "\n".join(samples)

src/crewai/evaluation/metrics/semantic_quality_metrics.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict
+
+from crewai.agent import Agent
+from crewai.task import Task
+
+from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory
+from crewai.evaluation.json_parser import extract_json_from_llm_response
+
+class SemanticQualityEvaluator(BaseEvaluator):
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.SEMANTIC_QUALITY
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: Any,
+    ) -> EvaluationScore:
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing the semantic quality of an AI agent's output.
+
+Score the semantic quality on a scale from 0-10 where:
+- 0: Completely incoherent, confusing, or logically flawed output
+- 5: Moderately clear and logical output with some issues
+- 10: Exceptionally clear, coherent, and logically sound output
+
+Consider:
+1. Is the output well-structured and organized?
+2. Is the reasoning logical and well-supported?
+3. Is the language clear, precise, and appropriate for the task?
+4. Are claims supported by evidence when appropriate?
+5. Is the output free from contradictions and logical fallacies?
+
+Return your evaluation as JSON with fields 'score' (number) and 'feedback' (string).
+"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Task description: {task.description}
+
+Agent's final output:
+{final_output}
+
+Evaluate the semantic quality and reasoning of this output.
+"""}
+        ]
+
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data: dict[str, Any] = extract_json_from_llm_response(response)
+            assert evaluation_data is not None
+            return EvaluationScore(
+                score=float(evaluation_data["score"]) if evaluation_data.get("score") is not None else None,
+                feedback=evaluation_data.get("feedback", response),
+                raw_response=response
+            )
+        except Exception:
+            return EvaluationScore(
+                score=None,
+                feedback=f"Failed to parse evaluation. Raw response: {response}",
+                raw_response=response
+            )

src/crewai/evaluation/metrics/tools_metrics.py

@@ -0,0 +1,400 @@
+import json
+from typing import Dict, Any
+
+from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory
+from crewai.evaluation.json_parser import extract_json_from_llm_response
+from crewai.agent import Agent
+from crewai.task import Task
+
+
+class ToolSelectionEvaluator(BaseEvaluator):
+
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.TOOL_SELECTION
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: str,
+    ) -> EvaluationScore:
+        tool_uses = execution_trace.get("tool_uses", [])
+        tool_count = len(tool_uses)
+        unique_tool_types = set([tool.get("tool", "Unknown tool") for tool in tool_uses])
+
+        if tool_count == 0:
+            if not agent.tools:
+                return EvaluationScore(
+                    score=None,
+                    feedback="Agent had no tools available to use."
+                )
+            else:
+                return EvaluationScore(
+                    score=None,
+                    feedback="Agent had tools available but didn't use any."
+                )
+
+        available_tools_info = ""
+        if agent.tools:
+            for tool in agent.tools:
+                available_tools_info += f"- {tool.name}: {tool.description}\n"
+        else:
+            available_tools_info = "No tools available"
+
+        tool_types_summary = "Tools selected by the agent:\n"
+        for tool_type in sorted(unique_tool_types):
+            tool_types_summary += f"- {tool_type}\n"
+
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing if an AI agent selected the most appropriate tools for a given task.
+
+You must evaluate based on these 2 criteria:
+1. Relevance (0-10): Were the tools chosen directly aligned with the task's goals?
+2. Coverage (0-10): Did the agent select ALL appropriate tools from the AVAILABLE tools?
+
+IMPORTANT:
+- ONLY consider tools that are listed as available to the agent
+- DO NOT suggest tools that aren't in the 'Available tools' list
+- DO NOT evaluate the quality or accuracy of tool outputs/results
+- DO NOT evaluate how many times each tool was used
+- DO NOT evaluate how the agent used the parameters
+- DO NOT evaluate whether the agent interpreted the task correctly
+
+Focus ONLY on whether the correct CATEGORIES of tools were selected from what was available.
+
+Return your evaluation as JSON with these fields:
+- scores: {"relevance": number, "coverage": number}
+- overall_score: number (average of all scores, 0-10)
+- feedback: string (focused ONLY on tool selection decisions from available tools)
+- improvement_suggestions: string (ONLY suggest better selection from the AVAILABLE tools list, NOT new tools)
+"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Task description: {task.description}
+
+Available tools for this agent:
+{available_tools_info}
+
+{tool_types_summary}
+
+Based ONLY on the task description and comparing the AVAILABLE tools with those that were selected (listed above), evaluate if the agent selected the appropriate tool types for this task.
+
+IMPORTANT:
+- ONLY evaluate selection from tools listed as available
+- DO NOT suggest new tools that aren't in the available tools list
+- DO NOT evaluate tool usage or results
+"""}
+        ]
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data = extract_json_from_llm_response(response)
+            assert evaluation_data is not None
+
+            scores = evaluation_data.get("scores", {})
+            relevance = scores.get("relevance", 5.0)
+            coverage = scores.get("coverage", 5.0)
+            overall_score = float(evaluation_data.get("overall_score", 5.0))
+
+            feedback = "Tool Selection Evaluation:\n"
+            feedback += f"• Relevance: {relevance}/10 - Selection of appropriate tool types for the task\n"
+            feedback += f"• Coverage: {coverage}/10 - Selection of all necessary tool types\n"
+            if "improvement_suggestions" in evaluation_data:
+                feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"
+            else:
+                feedback += evaluation_data.get("feedback", "No detailed feedback available.")
+
+            return EvaluationScore(
+                score=overall_score,
+                feedback=feedback,
+                raw_response=response
+            )
+        except Exception as e:
+            return EvaluationScore(
+                score=None,
+                feedback=f"Error evaluating tool selection: {e}",
+                raw_response=response
+            )
+
+
+class ParameterExtractionEvaluator(BaseEvaluator):
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.PARAMETER_EXTRACTION
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: str,
+    ) -> EvaluationScore:
+        tool_uses = execution_trace.get("tool_uses", [])
+        tool_count = len(tool_uses)
+
+        if tool_count == 0:
+            return EvaluationScore(
+                score=None,
+                feedback="No tool usage detected. Cannot evaluate parameter extraction."
+            )
+
+        validation_errors = []
+        for tool_use in tool_uses:
+            if not tool_use.get("success", True) and tool_use.get("error_type") == "validation_error":
+                validation_errors.append({
+                    "tool": tool_use.get("tool", "Unknown tool"),
+                    "error": tool_use.get("result"),
+                    "args": tool_use.get("args", {})
+                })
+
+        validation_error_rate = len(validation_errors) / tool_count if tool_count > 0 else 0
+
+        param_samples = []
+        for i, tool_use in enumerate(tool_uses[:5]):
+            tool_name = tool_use.get("tool", "Unknown tool")
+            tool_args = tool_use.get("args", {})
+            success = tool_use.get("success", True) and not tool_use.get("error", False)
+            error_type = tool_use.get("error_type", "") if not success else ""
+
+            is_validation_error = error_type == "validation_error"
+
+            sample = f"Tool use #{i+1} - {tool_name}:\n"
+            sample += f"- Parameters: {json.dumps(tool_args, indent=2)}\n"
+            sample += f"- Success: {'No' if not success else 'Yes'}"
+
+            if is_validation_error:
+                sample += " (PARAMETER VALIDATION ERROR)\n"
+                sample += f"- Error: {tool_use.get('result', 'Unknown error')}"
+            elif not success:
+                sample += f" (Other error: {error_type})\n"
+
+            param_samples.append(sample)
+
+        validation_errors_info = ""
+        if validation_errors:
+            validation_errors_info = f"\nParameter validation errors detected: {len(validation_errors)} ({validation_error_rate:.1%} of tool uses)\n"
+            for i, err in enumerate(validation_errors[:3]):
+                tool_name = err.get("tool", "Unknown tool")
+                error_msg = err.get("error", "Unknown error")
+                args = err.get("args", {})
+                validation_errors_info += f"\nValidation Error #{i+1}:\n- Tool: {tool_name}\n- Args: {json.dumps(args, indent=2)}\n- Error: {error_msg}"
+
+            if len(validation_errors) > 3:
+                validation_errors_info += f"\n...and {len(validation_errors) - 3} more validation errors."
+        param_samples_text = "\n\n".join(param_samples)
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing how well an AI agent extracts and formats PARAMETER VALUES for tool calls.
+
+Your job is to evaluate ONLY whether the agent used the correct parameter VALUES, not whether the right tools were selected or how the tools were invoked.
+
+Evaluate parameter extraction based on these criteria:
+1. Accuracy (0-10): Are parameter values correctly identified from the context/task?
+2. Formatting (0-10): Are values formatted correctly for each tool's requirements?
+3. Completeness (0-10): Are all required parameter values provided, with no missing information?
+
+IMPORTANT: DO NOT evaluate:
+- Whether the right tool was chosen (that's the ToolSelectionEvaluator's job)
+- How the tools were structurally invoked (that's the ToolInvocationEvaluator's job)
+- The quality of results from tools
+
+Focus ONLY on the PARAMETER VALUES - whether they were correctly extracted from the context, properly formatted, and complete.
+
+Validation errors are important signals that parameter values weren't properly extracted or formatted.
+
+Return your evaluation as JSON with these fields:
+- scores: {"accuracy": number, "formatting": number, "completeness": number}
+- overall_score: number (average of all scores, 0-10)
+- feedback: string (focused ONLY on parameter value extraction quality)
+- improvement_suggestions: string (concrete suggestions for better parameter VALUE extraction)
+"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Task description: {task.description}
+
+Parameter extraction examples:
+{param_samples_text}
+{validation_errors_info}
+
+Evaluate the quality of the agent's parameter extraction for this task.
+"""}
+        ]
+
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data = extract_json_from_llm_response(response)
+            assert evaluation_data is not None
+
+            scores = evaluation_data.get("scores", {})
+            accuracy = scores.get("accuracy", 5.0)
+            formatting = scores.get("formatting", 5.0)
+            completeness = scores.get("completeness", 5.0)
+
+            overall_score = float(evaluation_data.get("overall_score", 5.0))
+
+            feedback = "Parameter Extraction Evaluation:\n"
+            feedback += f"• Accuracy: {accuracy}/10 - Correctly identifying required parameters\n"
+            feedback += f"• Formatting: {formatting}/10 - Properly formatting parameters for tools\n"
+            feedback += f"• Completeness: {completeness}/10 - Including all necessary information\n\n"
+
+            if "improvement_suggestions" in evaluation_data:
+                feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"
+            else:
+                feedback += evaluation_data.get("feedback", "No detailed feedback available.")
+
+            return EvaluationScore(
+                score=overall_score,
+                feedback=feedback,
+                raw_response=response
+            )
+        except Exception as e:
+            return EvaluationScore(
+                score=None,
+                feedback=f"Error evaluating parameter extraction: {e}",
+                raw_response=response
+            )
+
+
+class ToolInvocationEvaluator(BaseEvaluator):
+    @property
+    def metric_category(self) -> MetricCategory:
+        return MetricCategory.TOOL_INVOCATION
+
+    def evaluate(
+        self,
+        agent: Agent,
+        task: Task,
+        execution_trace: Dict[str, Any],
+        final_output: str,
+    ) -> EvaluationScore:
+        tool_uses = execution_trace.get("tool_uses", [])
+        tool_errors = []
+        tool_count = len(tool_uses)
+
+        if tool_count == 0:
+            return EvaluationScore(
+                score=None,
+                feedback="No tool usage detected. Cannot evaluate tool invocation."
+            )
+
+        for tool_use in tool_uses:
+            if not tool_use.get("success", True) or tool_use.get("error", False):
+                error_info = {
+                    "tool": tool_use.get("tool", "Unknown tool"),
+                    "error": tool_use.get("result"),
+                    "error_type": tool_use.get("error_type", "unknown_error")
+                }
+                tool_errors.append(error_info)
+
+        error_rate = len(tool_errors) / tool_count if tool_count > 0 else 0
+
+        error_types = {}
+        for error in tool_errors:
+            error_type = error.get("error_type", "unknown_error")
+            if error_type not in error_types:
+                error_types[error_type] = 0
+            error_types[error_type] += 1
+
+        invocation_samples = []
+        for i, tool_use in enumerate(tool_uses[:5]):
+            tool_name = tool_use.get("tool", "Unknown tool")
+            tool_args = tool_use.get("args", {})
+            success = tool_use.get("success", True) and not tool_use.get("error", False)
+            error_type = tool_use.get("error_type", "") if not success else ""
+            error_msg = tool_use.get("result", "No error") if not success else "No error"
+
+            sample = f"Tool invocation #{i+1}:\n"
+            sample += f"- Tool: {tool_name}\n"
+            sample += f"- Parameters: {json.dumps(tool_args, indent=2)}\n"
+            sample += f"- Success: {'No' if not success else 'Yes'}\n"
+            if not success:
+                sample += f"- Error type: {error_type}\n"
+                sample += f"- Error: {error_msg}"
+            invocation_samples.append(sample)
+
+        error_type_summary = ""
+        if error_types:
+            error_type_summary = "Error type breakdown:\n"
+            for error_type, count in error_types.items():
+                error_type_summary += f"- {error_type}: {count} occurrences ({(count/tool_count):.1%})\n"
+
+        invocation_samples_text = "\n\n".join(invocation_samples)
+        prompt = [
+            {"role": "system", "content": """You are an expert evaluator assessing how correctly an AI agent's tool invocations are STRUCTURED.
+
+Your job is to evaluate ONLY the structural and syntactical aspects of how the agent called tools, NOT which tools were selected or what parameter values were used.
+
+Evaluate the agent's tool invocation based on these criteria:
+1. Structure (0-10): Does the tool call follow the expected syntax and format?
+2. Error Handling (0-10): Does the agent handle tool errors appropriately?
+3. Invocation Patterns (0-10): Are tool calls properly sequenced, batched, or managed?
+
+Error types that indicate invocation issues:
+- execution_error: The tool was called correctly but failed during execution
+- usage_error: General errors in how the tool was used structurally
+
+IMPORTANT: DO NOT evaluate:
+- Whether the right tool was chosen (that's the ToolSelectionEvaluator's job)
+- Whether the parameter values are correct (that's the ParameterExtractionEvaluator's job)
+- The quality of results from tools
+
+Focus ONLY on HOW tools were invoked - the structure, format, and handling of the invocation process.
+
+Return your evaluation as JSON with these fields:
+- scores: {"structure": number, "error_handling": number, "invocation_patterns": number}
+- overall_score: number (average of all scores, 0-10)
+- feedback: string (focused ONLY on structural aspects of tool invocation)
+- improvement_suggestions: string (concrete suggestions for better structuring of tool calls)
+"""},
+            {"role": "user", "content": f"""
+Agent role: {agent.role}
+Task description: {task.description}
+
+Tool invocation examples:
+{invocation_samples_text}
+
+Tool error rate: {error_rate:.2%} ({len(tool_errors)} errors out of {tool_count} invocations)
+{error_type_summary}
+
+Evaluate the quality of the agent's tool invocation structure during this task.
+"""}
+        ]
+
+        assert self.llm is not None
+        response = self.llm.call(prompt)
+
+        try:
+            evaluation_data = extract_json_from_llm_response(response)
+            assert evaluation_data is not None
+            scores = evaluation_data.get("scores", {})
+            structure = scores.get("structure", 5.0)
+            error_handling = scores.get("error_handling", 5.0)
+            invocation_patterns = scores.get("invocation_patterns", 5.0)
+
+            overall_score = float(evaluation_data.get("overall_score", 5.0))
+
+            feedback = "Tool Invocation Evaluation:\n"
+            feedback += f"• Structure: {structure}/10 - Following proper syntax and format\n"
+            feedback += f"• Error Handling: {error_handling}/10 - Appropriately handling tool errors\n"
+            feedback += f"• Invocation Patterns: {invocation_patterns}/10 - Proper sequencing and management of calls\n\n"
+
+            if "improvement_suggestions" in evaluation_data:
+                feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"
+            else:
+                feedback += evaluation_data.get("feedback", "No detailed feedback available.")
+
+            return EvaluationScore(
+                score=overall_score,
+                feedback=feedback,
+                raw_response=response
+            )
+        except Exception as e:
+            return EvaluationScore(
+                score=None,
+                feedback=f"Error evaluating tool invocation: {e}",
+                raw_response=response
+            )

src/crewai/lite_agent.py

@@ -537,6 +537,7 @@ class LiteAgent(FlowTrackable, BaseModel):
                     crewai_event_bus.emit(
                         self,
                         event=LLMCallCompletedEvent(
+                            messages=self._messages,
                             response=answer,
                             call_type=LLMCallType.LLM_CALL,
                             from_agent=self,

src/crewai/llm.py

@@ -508,7 +508,6 @@ class LLM(BaseLLM):
                             # Enable tool calls using streaming
                             if "tool_calls" in delta:
                                 tool_calls = delta["tool_calls"]
-
                                 if tool_calls:
                                     result = self._handle_streaming_tool_calls(
                                         tool_calls=tool_calls,
@@ -517,6 +516,7 @@ class LLM(BaseLLM):
                                         from_task=from_task,
                                         from_agent=from_agent,
                                     )
+
                                     if result is not None:
                                         chunk_content = result
 
@@ -631,7 +631,7 @@ class LLM(BaseLLM):
                 # Log token usage if available in streaming mode
                 self._handle_streaming_callbacks(callbacks, usage_info, last_chunk)
                 # Emit completion event and return response
-                self._handle_emit_call_events(full_response, LLMCallType.LLM_CALL, from_task, from_agent)
+                self._handle_emit_call_events(response=full_response, call_type=LLMCallType.LLM_CALL, from_task=from_task, from_agent=from_agent, messages=params["messages"])
                 return full_response
 
             # --- 9) Handle tool calls if present
@@ -643,7 +643,7 @@ class LLM(BaseLLM):
             self._handle_streaming_callbacks(callbacks, usage_info, last_chunk)
 
             # --- 11) Emit completion event and return response
-            self._handle_emit_call_events(full_response, LLMCallType.LLM_CALL, from_task, from_agent)
+            self._handle_emit_call_events(response=full_response, call_type=LLMCallType.LLM_CALL, from_task=from_task, from_agent=from_agent, messages=params["messages"])
             return full_response
 
         except ContextWindowExceededError as e:
@@ -655,7 +655,7 @@ class LLM(BaseLLM):
             logging.error(f"Error in streaming response: {str(e)}")
             if full_response.strip():
                 logging.warning(f"Returning partial response despite error: {str(e)}")
-                self._handle_emit_call_events(full_response, LLMCallType.LLM_CALL, from_task, from_agent)
+                self._handle_emit_call_events(response=full_response, call_type=LLMCallType.LLM_CALL, from_task=from_task, from_agent=from_agent, messages=params["messages"])
                 return full_response
 
             # Emit failed event and re-raise the exception
@@ -809,7 +809,7 @@ class LLM(BaseLLM):
 
         # --- 5) If no tool calls or no available functions, return the text response directly
         if not tool_calls or not available_functions:
-            self._handle_emit_call_events(text_response, LLMCallType.LLM_CALL, from_task, from_agent)
+            self._handle_emit_call_events(response=text_response, call_type=LLMCallType.LLM_CALL, from_task=from_task, from_agent=from_agent, messages=params["messages"])
             return text_response
 
         # --- 6) Handle tool calls if present
@@ -818,7 +818,7 @@ class LLM(BaseLLM):
             return tool_result
 
         # --- 7) If tool call handling didn't return a result, emit completion event and return text response
-        self._handle_emit_call_events(text_response, LLMCallType.LLM_CALL, from_task, from_agent)
+        self._handle_emit_call_events(response=text_response, call_type=LLMCallType.LLM_CALL, from_task=from_task, from_agent=from_agent, messages=params["messages"])
         return text_response
 
     def _handle_tool_call(
@@ -861,6 +861,7 @@ class LLM(BaseLLM):
                         tool_args=function_args,
                     ),
                 )
+
                 result = fn(**function_args)
                 crewai_event_bus.emit(
                     self,
@@ -874,7 +875,7 @@ class LLM(BaseLLM):
                 )
 
                 # --- 3.3) Emit success event
-                self._handle_emit_call_events(result, LLMCallType.TOOL_CALL)
+                self._handle_emit_call_events(response=result, call_type=LLMCallType.TOOL_CALL)
                 return result
             except Exception as e:
                 # --- 3.4) Handle execution errors
@@ -991,17 +992,20 @@ class LLM(BaseLLM):
                 logging.error(f"LiteLLM call failed: {str(e)}")
                 raise
 
-    def _handle_emit_call_events(self, response: Any, call_type: LLMCallType, from_task: Optional[Any] = None, from_agent: Optional[Any] = None):
+    def _handle_emit_call_events(self, response: Any, call_type: LLMCallType, from_task: Optional[Any] = None, from_agent: Optional[Any] = None, messages: str | list[dict[str, Any]] | None = None):
         """Handle the events for the LLM call.
 
         Args:
             response (str): The response from the LLM call.
             call_type (str): The type of call, either "tool_call" or "llm_call".
+            from_task: Optional task object
+            from_agent: Optional agent object
+            messages: Optional messages object
         """
         assert hasattr(crewai_event_bus, "emit")
         crewai_event_bus.emit(
             self,
-            event=LLMCallCompletedEvent(response=response, call_type=call_type, from_task=from_task, from_agent=from_agent),
+            event=LLMCallCompletedEvent(messages=messages, response=response, call_type=call_type, from_task=from_task, from_agent=from_agent),
         )
 
     def _format_messages_for_provider(

src/crewai/utilities/evaluators/crew_evaluator_handler.py

@@ -155,6 +155,7 @@ class CrewEvaluator:
         )
 
         console = Console()
+        console.print("\n")
         console.print(table)
 
     def evaluate(self, task_output: TaskOutput):

src/crewai/utilities/events/llm_events.py

@@ -48,8 +48,8 @@ class LLMCallStartedEvent(LLMEventBase):
     """
 
     type: str = "llm_call_started"
-    messages: Union[str, List[Dict[str, Any]]]
-    tools: Optional[List[dict]] = None
+    messages: Optional[Union[str, List[Dict[str, Any]]]] = None
+    tools: Optional[List[dict[str, Any]]] = None
     callbacks: Optional[List[Any]] = None
     available_functions: Optional[Dict[str, Any]] = None
 
@@ -58,10 +58,10 @@ class LLMCallCompletedEvent(LLMEventBase):
     """Event emitted when a LLM call completes"""
 
     type: str = "llm_call_completed"
+    messages: str | list[dict[str, Any]] | None = None
     response: Any
     call_type: LLMCallType
 
-
 class LLMCallFailedEvent(LLMEventBase):
     """Event emitted when a LLM call fails"""
 

tests/cassettes/TestAgentEvaluator.test_evaluate_current_iteration.yaml

@@ -0,0 +1,430 @@
+interactions:
+- request:
+    body: null
+    headers: {}
+    method: GET
+    uri: https://pypi.org/pypi/agentops/json
+  response:
+    body:
+      string: '{"info":{"author":null,"author_email":"Alex Reibman <areibman@gmail.com>,
+        Shawn Qiu <siyangqiu@gmail.com>, Braelyn Boynton <bboynton97@gmail.com>, Howard
+        Gil <howardbgil@gmail.com>, Constantin Teodorescu <teocns@gmail.com>, Pratyush
+        Shukla <ps4534@nyu.edu>, Travis Dent <tcdent@gmail.com>, Dwij Patel <dwijpatel1704@gmail.com>,
+        Fenil Faldu <fenilfaldu143@gmail.com>","bugtrack_url":null,"classifiers":["License
+        :: OSI Approved :: MIT License","Operating System :: OS Independent","Programming
+        Language :: Python :: 3","Programming Language :: Python :: 3.10","Programming
+        Language :: Python :: 3.11","Programming Language :: Python :: 3.12","Programming
+        Language :: Python :: 3.13","Programming Language :: Python :: 3.9"],"description":"","description_content_type":null,"docs_url":null,"download_url":null,"downloads":{"last_day":-1,"last_month":-1,"last_week":-1},"dynamic":null,"home_page":null,"keywords":null,"license":null,"license_expression":null,"license_files":["LICENSE"],"maintainer":null,"maintainer_email":null,"name":"agentops","package_url":"https://pypi.org/project/agentops/","platform":null,"project_url":"https://pypi.org/project/agentops/","project_urls":{"Homepage":"https://github.com/AgentOps-AI/agentops","Issues":"https://github.com/AgentOps-AI/agentops/issues"},"provides_extra":null,"release_url":"https://pypi.org/project/agentops/0.4.17/","requires_dist":["httpx<0.29.0,>=0.24.0","opentelemetry-api==1.29.0;
+        python_version < \"3.10\"","opentelemetry-api>1.29.0; python_version >= \"3.10\"","opentelemetry-exporter-otlp-proto-http==1.29.0;
+        python_version < \"3.10\"","opentelemetry-exporter-otlp-proto-http>1.29.0;
+        python_version >= \"3.10\"","opentelemetry-instrumentation==0.50b0; python_version
+        < \"3.10\"","opentelemetry-instrumentation>=0.50b0; python_version >= \"3.10\"","opentelemetry-sdk==1.29.0;
+        python_version < \"3.10\"","opentelemetry-sdk>1.29.0; python_version >= \"3.10\"","opentelemetry-semantic-conventions==0.50b0;
+        python_version < \"3.10\"","opentelemetry-semantic-conventions>=0.50b0; python_version
+        >= \"3.10\"","ordered-set<5.0.0,>=4.0.0","packaging<25.0,>=21.0","psutil<7.0.1,>=5.9.8","pyyaml<7.0,>=5.3","requests<3.0.0,>=2.0.0","termcolor<2.5.0,>=2.3.0","wrapt<2.0.0,>=1.0.0"],"requires_python":">=3.9","summary":"Observability
+        and DevTool Platform for AI Agents","version":"0.4.17","yanked":false,"yanked_reason":null},"last_serial":29926354,"releases":{"0.0.1":[{"comment_text":"","digests":{"blake2b_256":"9b4641d084346e88671acc02e3a0049d3e0925fe99edd88c8b82700dc3c04d01","md5":"2b491f3b3dd01edd4ee37c361087bb46","sha256":"f2cb9d59a0413e7977a44a23dbd6a9d89cda5309b63ed08f5c346c7488acf645"},"downloads":-1,"filename":"agentops-0.0.1-py3-none-any.whl","has_sig":false,"md5_digest":"2b491f3b3dd01edd4ee37c361087bb46","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":10328,"upload_time":"2023-08-21T18:33:47","upload_time_iso_8601":"2023-08-21T18:33:47.827866Z","url":"https://files.pythonhosted.org/packages/9b/46/41d084346e88671acc02e3a0049d3e0925fe99edd88c8b82700dc3c04d01/agentops-0.0.1-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"b280bf609d98778499bd42df723100a8e910d9b9827cbd00b804cf0b13bb3c87","md5":"ff218fc16d45cf72f73d50ee9a0afe82","sha256":"5c3d4311b9dde0c71cb475ec99d2963a71604c78d468b333f55e81364f4fe79e"},"downloads":-1,"filename":"agentops-0.0.1.tar.gz","has_sig":false,"md5_digest":"ff218fc16d45cf72f73d50ee9a0afe82","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":11452,"upload_time":"2023-08-21T18:33:49","upload_time_iso_8601":"2023-08-21T18:33:49.613830Z","url":"https://files.pythonhosted.org/packages/b2/80/bf609d98778499bd42df723100a8e910d9b9827cbd00b804cf0b13bb3c87/agentops-0.0.1.tar.gz","yanked":false,"yanked_reason":null}],"0.0.10":[{"comment_text":"","digests":{"blake2b_256":"92933862af53105332cb524db237138d3284b5d6abcc7df5fd4406e382372d94","md5":"8bdea319b5579775eb88efac72e70cd6","sha256":"e8a333567458c1df35538d626bc596f3ba7b8fa2aac5015bc378f3f7f8850669"},"downloads":-1,"filename":"agentops-0.0.10-py3-none-any.whl","has_sig":false,"md5_digest":"8bdea319b5579775eb88efac72e70cd6","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":14752,"upload_time":"2023-12-16T01:40:40","upload_time_iso_8601":"2023-12-16T01:40:40.867657Z","url":"https://files.pythonhosted.org/packages/92/93/3862af53105332cb524db237138d3284b5d6abcc7df5fd4406e382372d94/agentops-0.0.10-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"c63136b1f2e508b67f92ddb5f51f2acf5abdf2bf4b32d5b355d8018b368dc854","md5":"87bdcd4d7469d22ce922234d4f0b2b98","sha256":"5fbc567bece7b218fc35ce70d208e88e89bb399a9dbf84ab7ad59a2aa559648c"},"downloads":-1,"filename":"agentops-0.0.10.tar.gz","has_sig":false,"md5_digest":"87bdcd4d7469d22ce922234d4f0b2b98","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":15099,"upload_time":"2023-12-16T01:40:42","upload_time_iso_8601":"2023-12-16T01:40:42.281826Z","url":"https://files.pythonhosted.org/packages/c6/31/36b1f2e508b67f92ddb5f51f2acf5abdf2bf4b32d5b355d8018b368dc854/agentops-0.0.10.tar.gz","yanked":false,"yanked_reason":null}],"0.0.11":[{"comment_text":"","digests":{"blake2b_256":"7125ed114f918332cda824092f620b1002fd76ab6b538dd83711b31c93907139","md5":"83ba7e621f01412144aa38306fc1e04c","sha256":"cb80823e065d17dc26bdc8fe951ea7e04b23677ef2b4da939669c6fe1b2502bf"},"downloads":-1,"filename":"agentops-0.0.11-py3-none-any.whl","has_sig":false,"md5_digest":"83ba7e621f01412144aa38306fc1e04c","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":16627,"upload_time":"2023-12-21T19:50:28","upload_time_iso_8601":"2023-12-21T19:50:28.595886Z","url":"https://files.pythonhosted.org/packages/71/25/ed114f918332cda824092f620b1002fd76ab6b538dd83711b31c93907139/agentops-0.0.11-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"9e037750b04398cda2548bbf3d84ce554c4009592095c060c4904e773f3a43da","md5":"5bbb120cc9a5f5ff6fb5dd45691ba279","sha256":"cbf0f39768d47e32be448a3ff3ded665fce64ff8a90c0e10692fd7a3ab4790ee"},"downloads":-1,"filename":"agentops-0.0.11.tar.gz","has_sig":false,"md5_digest":"5bbb120cc9a5f5ff6fb5dd45691ba279","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":16794,"upload_time":"2023-12-21T19:50:29","upload_time_iso_8601":"2023-12-21T19:50:29.881561Z","url":"https://files.pythonhosted.org/packages/9e/03/7750b04398cda2548bbf3d84ce554c4009592095c060c4904e773f3a43da/agentops-0.0.11.tar.gz","yanked":false,"yanked_reason":null}],"0.0.12":[{"comment_text":"","digests":{"blake2b_256":"adf5cc3e93b2328532ea80b8b36450b8b48a8199ebbe1f75ebb490e57a926b88","md5":"694ba49ca8841532039bdf8dc0250b85","sha256":"9a2c773efbe3353f60d1b86da12333951dad288ba54839615a53b57e5965bea8"},"downloads":-1,"filename":"agentops-0.0.12-py3-none-any.whl","has_sig":false,"md5_digest":"694ba49ca8841532039bdf8dc0250b85","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":18602,"upload_time":"2024-01-03T03:47:07","upload_time_iso_8601":"2024-01-03T03:47:07.184203Z","url":"https://files.pythonhosted.org/packages/ad/f5/cLine truncated
+        breaking bug"},{"comment_text":"","digests":{"blake2b_256":"b2447ce75e71fcc9605a609b41adc52d517eba4356d15f7ca77d46f683ca07f1","md5":"9a9bb22af4b30c454d46b9a01e8701a0","sha256":"70d22e9a71ea13af6e6ad9c1cffe63c98f9dbccf91bda199825609379b2babaf"},"downloads":-1,"filename":"agentops-0.1.7.tar.gz","has_sig":false,"md5_digest":"9a9bb22af4b30c454d46b9a01e8701a0","packagetype":"sdist","python_version":"source","requires_python":">=3.10","size":28122,"upload_time":"2024-05-03T19:21:39","upload_time_iso_8601":"2024-05-03T19:21:39.415523Z","url":"https://files.pythonhosted.org/packages/b2/44/7ce75e71fcc9605a609b41adc52d517eba4356d15f7ca77d46f683ca07f1/agentops-0.1.7.tar.gz","yanked":true,"yanked_reason":"Introduced
+        breaking bug"}],"0.1.8":[{"comment_text":"","digests":{"blake2b_256":"38c63d0d19eeae4c3c9e3ff5957b10c3c16a4a9fd2be6673fbfc965f8bb4fd08","md5":"e12d3d92f51f5b2fed11a01742e5b5b5","sha256":"d49d113028a891d50900bb4fae253218cc49519f7fe39f9ea15f8f2b29d6d7ef"},"downloads":-1,"filename":"agentops-0.1.8-py3-none-any.whl","has_sig":false,"md5_digest":"e12d3d92f51f5b2fed11a01742e5b5b5","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.10","size":27977,"upload_time":"2024-05-04T03:01:53","upload_time_iso_8601":"2024-05-04T03:01:53.905081Z","url":"https://files.pythonhosted.org/packages/38/c6/3d0d19eeae4c3c9e3ff5957b10c3c16a4a9fd2be6673fbfc965f8bb4fd08/agentops-0.1.8-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"9269e51fa1714f169f692e4fad0a42ebeb77c7a27c48f62b751c869ad6441c69","md5":"07dbdb45f9ec086b1bc314d6a8264423","sha256":"5762137a84e2309e1b6ca9a0fd72c8b72c90f6f73ba49549980722221960cac8"},"downloads":-1,"filename":"agentops-0.1.8.tar.gz","has_sig":false,"md5_digest":"07dbdb45f9ec086b1bc314d6a8264423","packagetype":"sdist","python_version":"source","requires_python":">=3.10","size":28189,"upload_time":"2024-05-04T03:01:55","upload_time_iso_8601":"2024-05-04T03:01:55.328668Z","url":"https://files.pythonhosted.org/packages/92/69/e51fa1714f169f692e4fad0a42ebeb77c7a27c48f62b751c869ad6441c69/agentops-0.1.8.tar.gz","yanked":false,"yanked_reason":null}],"0.1.9":[{"comment_text":"","digests":{"blake2b_256":"eb5a920e71729bd1f06b002ee146b38b0d1862357a1f484628e6b20a7d3dcca1","md5":"6ae4929d91c4bb8025edc86b5322630c","sha256":"af7983ba4929b04a34714dd97d7e82c11384ebbe9d7d8bc7b673e1263c4c79a1"},"downloads":-1,"filename":"agentops-0.1.9-py3-none-any.whl","has_sig":false,"md5_digest":"6ae4929d91c4bb8025edc86b5322630c","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":28458,"upload_time":"2024-05-07T07:07:30","upload_time_iso_8601":"2024-05-07T07:07:30.798380Z","url":"https://files.pythonhosted.org/packages/eb/5a/920e71729bd1f06b002ee146b38b0d1862357a1f484628e6b20a7d3dcca1/agentops-0.1.9-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"df2b8fc76d629d8a83b0796612a27b966426550114c930eee5d730654fcd9fe9","md5":"43090632f87cd398ed77b57daa8c28d6","sha256":"7f428bfda2db57a994029b1c9f72b63ca7660616635c9c671b2b729d112a833e"},"downloads":-1,"filename":"agentops-0.1.9.tar.gz","has_sig":false,"md5_digest":"43090632f87cd398ed77b57daa8c28d6","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":28596,"upload_time":"2024-05-07T07:07:35","upload_time_iso_8601":"2024-05-07T07:07:35.242350Z","url":"https://files.pythonhosted.org/packages/df/2b/8fc76d629d8a83b0796612a27b966426550114c930eee5d730654fcd9fe9/agentops-0.1.9.tar.gz","yanked":false,"yanked_reason":null}],"0.2.0":[{"comment_text":"","digests":{"blake2b_256":"483560ec38a81a7e9588d32730ed4f581621169216f968771d5f611388f68a9b","md5":"bdda5480977cccd55628e117e8c8da04","sha256":"bee84bf046c9b4346c5f0f50e2087a992e8d2eae80b3fe9f01c456b49c299bcc"},"downloads":-1,"filename":"agentops-0.2.0-py3-none-any.whl","has_sig":false,"md5_digest":"bdda5480977cccd55628e117e8c8da04","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":35921,"upload_time":"2024-05-28T22:04:14","upload_time_iso_8601":"2024-05-28T22:04:14.813154Z","url":"https://files.pythonhosted.org/packages/48/35/60ec38a81a7e9588d32730ed4f581621169216f968771d5f611388f68a9b/agentops-0.2.0-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"8d7591c79141d31da4e56d6c6a00737b50dcc2f1ce8a711c1293d2a1d70478fc","md5":"71e3c3b9fe0286c9b58d81ba1c12a42d","sha256":"ca340136abff6a3727729c3eda87f0768e5ba2b672ce03320cb52ad138b05598"},"downloads":-1,"filename":"agentops-0.2.0.tar.gz","has_sig":false,"md5_digest":"71e3c3b9fe0286c9b58d81ba1c12a42d","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":35498,"upload_time":"2024-05-28T22:04:16","upload_time_iso_8601":"2024-05-28T22:04:16.598374Z","url":"https://files.pythonhosted.org/packages/8d/75/91c79141d31da4e56d6c6a00737b50dcc2f1ce8a711c1293d2a1d70478fc/agentops-0.2.0.tar.gz","yanked":false,"yanked_reason":null}],"0.2.1":[{"comment_text":"","digests":{"blake2b_256":"fa3b84032b7dca3d7315b329db6681bbfe0872c2a46d62ca992a05f2d6a078e1","md5":"ce3fc46711fa8225a3d6a9566f95f875","sha256":"7dde95db92c8306c0a17e193bfb5ee20e71e16630ccc629db685e148b3aca3f6"},"downloads":-1,"filename":"agentops-0.2.1-py3-none-any.whl","has_sig":false,"md5_digest":"ce3fc46711fa8225a3d6a9566f95f875","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":36375,"upload_time":"2024-06-03T18:40:02","upload_time_iso_8601":"2024-06-03T18:40:02.820700Z","url":"https://files.pythonhosted.org/packages/fa/3b/84032b7dca3d7315b329db6681bbfe0872c2a46d62ca992a05f2d6a078e1/agentops-0.2.1-py3-none-any.whl","yanked":false,"yanked_reasoLine truncated
+        breaking change"},{"comment_text":"","digests":{"blake2b_256":"8e4b920629e08c956cdc74a31ab466d005eb13d86c2d58fa2d2bd261cf36c37b","md5":"527c82f21f01f13b879a1fca90ddb209","sha256":"d263de21eb40e15eb17adc31821fc0dee4ff4ca4501a9feb7ed376d473063208"},"downloads":-1,"filename":"agentops-0.2.4.tar.gz","has_sig":false,"md5_digest":"527c82f21f01f13b879a1fca90ddb209","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":37656,"upload_time":"2024-06-24T19:32:01","upload_time_iso_8601":"2024-06-24T19:32:01.155014Z","url":"https://files.pythonhosted.org/packages/8e/4b/920629e08c956cdc74a31ab466d005eb13d86c2d58fa2d2bd261cf36c37b/agentops-0.2.4.tar.gz","yanked":true,"yanked_reason":"Potential
+        breaking change"}],"0.2.5":[{"comment_text":"","digests":{"blake2b_256":"47c73ab9d7d971b664a9bdff6e6464afb6c1de8eb0f845d8de93eb036d5dcc60","md5":"bed576cc1591da4783777920fb223761","sha256":"ff87b82d1efaf50b10624e00c6e9334f4c16ffe08ec7f9889b4417c231c31471"},"downloads":-1,"filename":"agentops-0.2.5-py3-none-any.whl","has_sig":false,"md5_digest":"bed576cc1591da4783777920fb223761","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":37529,"upload_time":"2024-06-26T22:57:15","upload_time_iso_8601":"2024-06-26T22:57:15.646328Z","url":"https://files.pythonhosted.org/packages/47/c7/3ab9d7d971b664a9bdff6e6464afb6c1de8eb0f845d8de93eb036d5dcc60/agentops-0.2.5-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"31c48f2af30ae75dbdb4697506f80f76ce786f79014deb8c6679fa62962fdd6f","md5":"42def99798edfaf201fa6f62846e77c5","sha256":"6bad7aca37af6174307769550a53ec00824049a57e97b8868a9a213b2272adb4"},"downloads":-1,"filename":"agentops-0.2.5.tar.gz","has_sig":false,"md5_digest":"42def99798edfaf201fa6f62846e77c5","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":37703,"upload_time":"2024-06-26T22:57:17","upload_time_iso_8601":"2024-06-26T22:57:17.337904Z","url":"https://files.pythonhosted.org/packages/31/c4/8f2af30ae75dbdb4697506f80f76ce786f79014deb8c6679fa62962fdd6f/agentops-0.2.5.tar.gz","yanked":false,"yanked_reason":null}],"0.2.6":[{"comment_text":"","digests":{"blake2b_256":"5af2f90538b00d887c04a5570e8a3af4aef27a600a67c058a0ee6befafd60748","md5":"8ef3ed13ed582346b71648ca9df30f7c","sha256":"59e88000a9f108931fd68056f22def7a7f4b3015906de5791e777c23ba7dee52"},"downloads":-1,"filename":"agentops-0.2.6-py3-none-any.whl","has_sig":false,"md5_digest":"8ef3ed13ed582346b71648ca9df30f7c","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":37534,"upload_time":"2024-06-28T21:41:56","upload_time_iso_8601":"2024-06-28T21:41:56.933334Z","url":"https://files.pythonhosted.org/packages/5a/f2/f90538b00d887c04a5570e8a3af4aef27a600a67c058a0ee6befafd60748/agentops-0.2.6-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"bcf412c388dccc301ad54a501843ba5b5dd359575dcef9ac24c18a619a32214d","md5":"89a6b04f12801682b53ee0133593ce74","sha256":"7906a08c9154355484deb173b82631f9acddec3775b2d5e8ca946abdee27183b"},"downloads":-1,"filename":"agentops-0.2.6.tar.gz","has_sig":false,"md5_digest":"89a6b04f12801682b53ee0133593ce74","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":37874,"upload_time":"2024-06-28T21:41:59","upload_time_iso_8601":"2024-06-28T21:41:59.143953Z","url":"https://files.pythonhosted.org/packages/bc/f4/12c388dccc301ad54a501843ba5b5dd359575dcef9ac24c18a619a32214d/agentops-0.2.6.tar.gz","yanked":false,"yanked_reason":null}],"0.3.0":[{"comment_text":"","digests":{"blake2b_256":"b8e996f12ac457f46c370c6f70f344e975d534f2c92853703ee29802f0127024","md5":"d9c6995a843b49ac7eb6f500fa1f3c2a","sha256":"22aeb3355e66b32a2b2a9f676048b81979b2488feddb088f9266034b3ed50539"},"downloads":-1,"filename":"agentops-0.3.0-py3-none-any.whl","has_sig":false,"md5_digest":"d9c6995a843b49ac7eb6f500fa1f3c2a","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":39430,"upload_time":"2024-07-17T18:38:24","upload_time_iso_8601":"2024-07-17T18:38:24.763919Z","url":"https://files.pythonhosted.org/packages/b8/e9/96f12ac457f46c370c6f70f344e975d534f2c92853703ee29802f0127024/agentops-0.3.0-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"7e2d6fda9613562c0394d7ef3dd8f0cb9fc4ebaa8d413862fce33940c73564d6","md5":"8fa67ca01ca726e3bfcd66898313f33f","sha256":"6c0c08a57410fa5e826a7bafa1deeba9f7b3524709427d9e1abbd0964caaf76b"},"downloads":-1,"filename":"agentops-0.3.0.tar.gz","has_sig":false,"md5_digest":"8fa67ca01ca726e3bfcd66898313f33f","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":41734,"upload_time":"2024-07-17T18:38:26","upload_time_iso_8601":"2024-07-17T18:38:26.447237Z","url":"https://files.pythonhosted.org/packages/7e/2d/6fda9613562c0394d7ef3dd8f0cb9fc4ebaa8d413862fce33940c73564d6/agentops-0.3.0.tar.gz","yanked":false,"yanked_reason":null}],"0.3.10":[{"comment_text":"","digests":{"blake2b_256":"eb5e3ac36b33d3e95747d64effd509f66a9b3b76b47216b16f492e27d8d90b0c","md5":"6fade0b81fc65b2c79a869b5f240590b","sha256":"b304d366691281e08c1f02307aabdd551ae4f68b0de82bbbb4cf6f651af2dd16"},"downloads":-1,"filename":"agentops-0.3.10-py3-none-any.whl","has_sig":false,"md5_digest":"6fade0b81fc65b2c79a869b5f240590b","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":41201,"upload_time":"2024-08-19T20:51:49","upload_time_iso_8601":"2024-08-19T20:51:49.487947Z","url":"https://files.pythonhosted.org/packages/eb/5e/3ac36b33d3e95747d64effd509f66a9b3b76b47216b16f492e27d8d90b0c/agentops-0.3.10-py3-none-any.whl","yanked":false,"yanked_rLine truncated
+        dependency, please install 0.3.18"},{"comment_text":"","digests":{"blake2b_256":"b319bb0e9895cb6da29f764f8d7b95b10ac8fde400bc17028f9bd486e9574dbe","md5":"f6424c41464d438007e9628748a0bea6","sha256":"ca0d4ba35ae699169ae20f74f72ca6a5780a8768ba2a2c32589fc5292ed81674"},"downloads":-1,"filename":"agentops-0.3.19.tar.gz","has_sig":false,"md5_digest":"f6424c41464d438007e9628748a0bea6","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":48360,"upload_time":"2024-12-04T00:54:01","upload_time_iso_8601":"2024-12-04T00:54:01.418776Z","url":"https://files.pythonhosted.org/packages/b3/19/bb0e9895cb6da29f764f8d7b95b10ac8fde400bc17028f9bd486e9574dbe/agentops-0.3.19.tar.gz","yanked":true,"yanked_reason":"Broken
+        dependency, please install 0.3.18"}],"0.3.2":[{"comment_text":"","digests":{"blake2b_256":"9d2c23b745a61d48df788b8020e5ea37e94f9da59b322a17accafe18d8cb4006","md5":"62d576d9518a627fe4232709c0721eff","sha256":"b35988e04378624204572bb3d7a454094f879ea573f05b57d4e75ab0bfbb82af"},"downloads":-1,"filename":"agentops-0.3.2-py3-none-any.whl","has_sig":false,"md5_digest":"62d576d9518a627fe4232709c0721eff","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":39527,"upload_time":"2024-07-21T03:09:56","upload_time_iso_8601":"2024-07-21T03:09:56.844372Z","url":"https://files.pythonhosted.org/packages/9d/2c/23b745a61d48df788b8020e5ea37e94f9da59b322a17accafe18d8cb4006/agentops-0.3.2-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"d2a1cc21406646c065e83435fe30fa205b99b2204d8074eca31926a5f8ef4381","md5":"30b247bcae25b181485a89213518241c","sha256":"55559ac4a43634831dfa8937c2597c28e332809dc7c6bb3bc3c8b233442e224c"},"downloads":-1,"filename":"agentops-0.3.2.tar.gz","has_sig":false,"md5_digest":"30b247bcae25b181485a89213518241c","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":41894,"upload_time":"2024-07-21T03:09:58","upload_time_iso_8601":"2024-07-21T03:09:58.409826Z","url":"https://files.pythonhosted.org/packages/d2/a1/cc21406646c065e83435fe30fa205b99b2204d8074eca31926a5f8ef4381/agentops-0.3.2.tar.gz","yanked":false,"yanked_reason":null}],"0.3.20":[{"comment_text":"","digests":{"blake2b_256":"a854ae9147a490dd9bd03ab7bfc5af47f40ff675840a9aa143896b385a8f8d3a","md5":"a13af8737ddff8a0c7c0f05cee70085f","sha256":"b5396e11b0bfef46b85604e8e36ab17668057711edd56f1edb0a067b8676fdcc"},"downloads":-1,"filename":"agentops-0.3.20-py3-none-any.whl","has_sig":false,"md5_digest":"a13af8737ddff8a0c7c0f05cee70085f","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":38674,"upload_time":"2024-12-07T00:06:31","upload_time_iso_8601":"2024-12-07T00:06:31.901162Z","url":"https://files.pythonhosted.org/packages/a8/54/ae9147a490dd9bd03ab7bfc5af47f40ff675840a9aa143896b385a8f8d3a/agentops-0.3.20-py3-none-any.whl","yanked":true,"yanked_reason":"Wrong
+        release"},{"comment_text":"","digests":{"blake2b_256":"c1eb19d04c801854ba75e235eb87c51a6a9c5b1a89e8579cb745c83f8bf84e08","md5":"11754497191d8340eda7a831720d9b74","sha256":"c71406294804a82795310a4afc492064a8884b1ba47e12607230975bc1291ce3"},"downloads":-1,"filename":"agentops-0.3.20.tar.gz","has_sig":false,"md5_digest":"11754497191d8340eda7a831720d9b74","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":48332,"upload_time":"2024-12-07T00:06:33","upload_time_iso_8601":"2024-12-07T00:06:33.568362Z","url":"https://files.pythonhosted.org/packages/c1/eb/19d04c801854ba75e235eb87c51a6a9c5b1a89e8579cb745c83f8bf84e08/agentops-0.3.20.tar.gz","yanked":true,"yanked_reason":"Wrong
+        release"}],"0.3.20rc1":[{"comment_text":"","digests":{"blake2b_256":"073de7eba58e2a60c0136eee2760b20f99607001d372de26505feee891e0976b","md5":"73c6ac515ee9d555e27a7ba7e26e3a46","sha256":"079ea8138938e27a3e1319a235a6f4cf98c0d6846731d854aa83b8422d570bda"},"downloads":-1,"filename":"agentops-0.3.20rc1-py3-none-any.whl","has_sig":false,"md5_digest":"73c6ac515ee9d555e27a7ba7e26e3a46","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":38718,"upload_time":"2024-12-07T00:10:18","upload_time_iso_8601":"2024-12-07T00:10:18.796963Z","url":"https://files.pythonhosted.org/packages/07/3d/e7eba58e2a60c0136eee2760b20f99607001d372de26505feee891e0976b/agentops-0.3.20rc1-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"02ff111d618c21aad946caedb666030f1f374a0d558228b9061ea2b46acb6bcd","md5":"17062e985b931dc85b4855922d7842ce","sha256":"ef48447e07a3eded246b2f7e10bba74422a34563ffdc667ac16b2d3383475a3f"},"downloads":-1,"filename":"agentops-0.3.20rc1.tar.gz","has_sig":false,"md5_digest":"17062e985b931dc85b4855922d7842ce","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":48329,"upload_time":"2024-12-07T00:10:20","upload_time_iso_8601":"2024-12-07T00:10:20.510407Z","url":"https://files.pythonhosted.org/packages/02/ff/111d618c21aad946caedb666030f1f374a0d558228b9061ea2b46acb6bcd/agentops-0.3.20rc1.tar.gz","yanked":false,"yanked_reason":null}],"0.3.20rc10":[{"comment_text":"","digests":{"blake2b_256":"a7274706d8d9c8f4abecc1dda2b9b02cd02ffe895220bd39f58322a46ccc7254","md5":"2c66a93c691c6b8cac2f2dc8fab9efae","sha256":"3c10d77f2fe88b61d97ad007820c1ba968c62f692986ea2b2cbfd8b22ec9e5bc"},"downloads":-1,"filename":"agentops-0.3.20rc10-py3-none-any.whl","has_sig":false,"md5_digest":"2c66a93c691c6b8cac2f2dc8fab9efae","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":57423,"upload_time":"2024-12-10T03:41:04","upload_time_iso_8601":"2024-12-10T03:41:04.579814Z","url":"https://files.pythonhosted.org/packages/a7/27/4706d8d9c8f4abecc1dda2b9b02cd02ffe895220bd39f58322a46ccc7254/agentops-0.3.20rc10-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"efe9e304f465945f57e4c6d35cd35fff53dc2a2e36b9b32793fa57017467b0c2","md5":"9882d32866b94d925ba36ac376c30bea","sha256":"f0c72c20e7fe41054c22c6257420314863549dd91428a892ac9b47b81cdfcc8c"},"downloads":-1,"filename":"agentops-0.3.20rc10.tar.gz","has_sig":false,"md5_digest":"9882d32866b94d925ba36ac376c30bea","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":57564,"upload_time":"2024-12-10T03:41:06","upload_time_iso_8601":"2024-12-10T03:41:06.899043Z","url":"https://files.pythonhosted.org/packages/ef/e9/e304f465945f57e4c6d35cd35fff53dc2a2e36b9b32793fa57017467b0c2/agentops-0.3.20rc10.tar.gz","yanked":false,"yanked_reason":null}],"0.3.20rc11":[{"comment_text":"","digests":{"blake2b_256":"8dbf598ec2532b713a228f4041c9b2c10358cd43e6aecf6128d0988a0b5f103e","md5":"d9ab67a850aefcb5bf9467b48f74675d","sha256":"3e5d4c19de6c58ae684693f47a2f03db35eaf4cd6d8aafc1e804a134462c2b55"},"downloads":-1,"filename":"agentops-0.3.20rc11-py3-none-any.whl","has_sig":false,"md5_digest":"d9ab67a850aefcb5bf9467b48f74675d","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":60280,"upload_time":"2024-12-10T22:45:05","upload_time_iso_8601":"2024-12-10T22:45:05.280119Z","url":"https://files.pythonhosted.org/packages/8d/bf/598ec2532b713a228f4041c9b2c10358cd43e6aecf6128d0988a0b5f103e/agentops-0.3.20rc11-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"210642e51fff6a4537fb811a15bc22d00343145285c6246dc069433d61436e1b","md5":"ca5279f4cb6ad82e06ef542a2d08d06e","sha256":"9211489c6a01bc9cda4061826f8b80d0989cfcd7fbabe1dd2ed5a5cb76b3d6f0"},"downloads":-1,"filename":"agentops-0.3.20rc11.tar.gz","has_sig":false,"md5_digest":"ca5279f4cb6ad82e06ef542a2d08d06e","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":59718,"upload_time":"2024-12-10T22:45:09","upload_time_iso_8601":"2024-12-10T22:45:09.616947Z","url":"https://files.pythonhosted.org/packages/21/06/42e51fff6a4537fb811a15bc22d00343145285c6246dc069433d61436e1b/agentops-0.3.20rc11.tar.gz","yanked":false,"yanked_reason":null}],"0.3.20rc12":[{"comment_text":"","digests":{"blake2b_256":"dc281db6f49f10ac849683de1d7f5b5ef492be2a996325302167b8388f375d51","md5":"8b2611d2510f0d4fac7ab824d7658ff7","sha256":"9237652d28db89315c49c0705829b291c17280e07d41272f909e2609acec650b"},"downloads":-1,"filename":"agentops-0.3.20rc12-py3-none-any.whl","has_sig":false,"md5_digest":"8b2611d2510f0d4fac7ab824d7658ff7","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":60282,"upload_time":"2024-12-10T23:10:54","upload_time_iso_8601":"2024-12-10T23:10:54.516317Z","url":"https://files.pythonhosted.org/packages/dc/28/1db6f49f10ac849683de1d7f5b5ef492be2a996325302167bLine truncated
+        dependency"},{"comment_text":"","digests":{"blake2b_256":"e067e61aa4c2e329da10b5e95d325091e599d8a00a28843a54bdcefa7a2eef8d","md5":"bcf45b6c4c56884ed2409f835571af62","sha256":"705d772b6994f8bab0cd163b24602009353f7906c72d9db008af11683f6e9341"},"downloads":-1,"filename":"agentops-0.3.22.tar.gz","has_sig":false,"md5_digest":"bcf45b6c4c56884ed2409f835571af62","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":52845,"upload_time":"2025-01-11T03:21:41","upload_time_iso_8601":"2025-01-11T03:21:41.762282Z","url":"https://files.pythonhosted.org/packages/e0/67/e61aa4c2e329da10b5e95d325091e599d8a00a28843a54bdcefa7a2eef8d/agentops-0.3.22.tar.gz","yanked":true,"yanked_reason":"Broken
+        dependency"}],"0.3.23":[{"comment_text":null,"digests":{"blake2b_256":"e67de1434765cf0a3d62372b74f47919aa17c0b01909823f7d3ee705edf821a9","md5":"1f0f02509b8ba713db72e57a072f01a6","sha256":"ecfff77d8f9006361ef2a2e8593271e97eb54b7b504abfb8abd6504006baca56"},"downloads":-1,"filename":"agentops-0.3.23-py3-none-any.whl","has_sig":false,"md5_digest":"1f0f02509b8ba713db72e57a072f01a6","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":70098,"upload_time":"2025-01-12T02:11:56","upload_time_iso_8601":"2025-01-12T02:11:56.319763Z","url":"https://files.pythonhosted.org/packages/e6/7d/e1434765cf0a3d62372b74f47919aa17c0b01909823f7d3ee705edf821a9/agentops-0.3.23-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"5c7fa4fd91f8fd819e1ecfdc608d1c7ade83de0f9dddd868e2c2c139a2fdae25","md5":"b7922399f81fb26517eb69fc7fef97c9","sha256":"4e4de49caeaf567b8746082f84a8cdd65afe2c698720f6f40251bbc4fdffe4c9"},"downloads":-1,"filename":"agentops-0.3.23.tar.gz","has_sig":false,"md5_digest":"b7922399f81fb26517eb69fc7fef97c9","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":64225,"upload_time":"2025-01-12T02:11:59","upload_time_iso_8601":"2025-01-12T02:11:59.360077Z","url":"https://files.pythonhosted.org/packages/5c/7f/a4fd91f8fd819e1ecfdc608d1c7ade83de0f9dddd868e2c2c139a2fdae25/agentops-0.3.23.tar.gz","yanked":false,"yanked_reason":null}],"0.3.24":[{"comment_text":null,"digests":{"blake2b_256":"254ea7d131802bac2ece5302ebf78dcef1ba1ba2f8b3a51fbe44c7f52bae6a53","md5":"39c39d8a7f1285add0fec21830a89a4a","sha256":"c5dfc8098b0dd49ddd819aa55280d07f8bfbf2f8fa088fc51ff5849b65062b10"},"downloads":-1,"filename":"agentops-0.3.24-py3-none-any.whl","has_sig":false,"md5_digest":"39c39d8a7f1285add0fec21830a89a4a","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":71957,"upload_time":"2025-01-18T19:08:02","upload_time_iso_8601":"2025-01-18T19:08:02.053316Z","url":"https://files.pythonhosted.org/packages/25/4e/a7d131802bac2ece5302ebf78dcef1ba1ba2f8b3a51fbe44c7f52bae6a53/agentops-0.3.24-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"71fee96e22c4bf762f34cd5ba435880470dad4576ab357ee61742fe053752322","md5":"3e1b7e0a31197936e099a7509128f794","sha256":"c97a3af959b728bcfbfb1ac2494cef82d8804defc9dac858648b39a9ecdcd2e4"},"downloads":-1,"filename":"agentops-0.3.24.tar.gz","has_sig":false,"md5_digest":"3e1b7e0a31197936e099a7509128f794","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":233974,"upload_time":"2025-01-18T19:08:04","upload_time_iso_8601":"2025-01-18T19:08:04.121618Z","url":"https://files.pythonhosted.org/packages/71/fe/e96e22c4bf762f34cd5ba435880470dad4576ab357ee61742fe053752322/agentops-0.3.24.tar.gz","yanked":false,"yanked_reason":null}],"0.3.25":[{"comment_text":null,"digests":{"blake2b_256":"e6e39cff4ed65c5deac34f427ed60cd7af3604ec7ed8a999c351f6411e190d3b","md5":"328dedc417be02fc28f8a4c7ed7b52e9","sha256":"4faebf73a62aa0bcac8578428277ca5b9af5e828f49f2cb03a9695b8426e6b9d"},"downloads":-1,"filename":"agentops-0.3.25-py3-none-any.whl","has_sig":false,"md5_digest":"328dedc417be02fc28f8a4c7ed7b52e9","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":71971,"upload_time":"2025-01-22T10:43:16","upload_time_iso_8601":"2025-01-22T10:43:16.070593Z","url":"https://files.pythonhosted.org/packages/e6/e3/9cff4ed65c5deac34f427ed60cd7af3604ec7ed8a999c351f6411e190d3b/agentops-0.3.25-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"2fdfeb00eaabebb51feae0724a5928f25df4d71d1c8392204f4f849351fd748c","md5":"a40bc7037baf6dbba92d63331f561a28","sha256":"868d855b6531d1fa2d1047db2cb03ddb1121062fd51c44b564dc626f15cc1e40"},"downloads":-1,"filename":"agentops-0.3.25.tar.gz","has_sig":false,"md5_digest":"a40bc7037baf6dbba92d63331f561a28","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":234024,"upload_time":"2025-01-22T10:43:17","upload_time_iso_8601":"2025-01-22T10:43:17.986230Z","url":"https://files.pythonhosted.org/packages/2f/df/eb00eaabebb51feae0724a5928f25df4d71d1c8392204f4f849351fd748c/agentops-0.3.25.tar.gz","yanked":false,"yanked_reason":null}],"0.3.26":[{"comment_text":null,"digests":{"blake2b_256":"f521671c458951850bd3a445aa09eafd2793aae1104fa68351a5c3976cdf762b","md5":"c3f8fa92ff5a94a37516e774c7f58b9a","sha256":"20948f52e3ffb4ba1d52301c3a82e59490182c4dad22774ad831dce0181eb5c2"},"downloads":-1,"filename":"agentops-0.3.26-py3-none-any.whl","has_sig":false,"md5_digest":"c3f8fa92ff5a94a37516e774c7f58b9a","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":72090,"upload_time":"2025-01-24T23:44:06","upload_time_iso_8601":"2025-01-24T23:44:06.828461Z","url":"https://files.pythonhosted.org/packages/f5/21/671c458951850bd3a445aa09eafd2793aae1104fa68351a5c39Line truncated
+        FileNotFoundError impacting OpenAI and LiteLLM integrations"},{"comment_text":"","digests":{"blake2b_256":"235508ce5915f1ceb86ea6f7a6e8c8dc025b34981408a1b638316b5140fad525","md5":"53ef2f5230de09260f4ead09633dde62","sha256":"ae98540355ce9b892a630e61a7224a9175657cad1b7e799269238748ca7bc0ea"},"downloads":-1,"filename":"agentops-0.3.5.tar.gz","has_sig":false,"md5_digest":"53ef2f5230de09260f4ead09633dde62","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":42699,"upload_time":"2024-08-01T19:32:21","upload_time_iso_8601":"2024-08-01T19:32:21.259555Z","url":"https://files.pythonhosted.org/packages/23/55/08ce5915f1ceb86ea6f7a6e8c8dc025b34981408a1b638316b5140fad525/agentops-0.3.5.tar.gz","yanked":true,"yanked_reason":"Introduces
+        FileNotFoundError impacting OpenAI and LiteLLM integrations"}],"0.3.6":[{"comment_text":"","digests":{"blake2b_256":"be89412afc864df3715d377cff9fe15deadaccdc0902b0a242f742f286e6d84b","md5":"149922f5cd986a8641b6e88c991af0cc","sha256":"413f812eb015fb31175a507784afe08123adfa9e227870e315899b059f42b443"},"downloads":-1,"filename":"agentops-0.3.6-py3-none-any.whl","has_sig":false,"md5_digest":"149922f5cd986a8641b6e88c991af0cc","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":39431,"upload_time":"2024-08-02T06:48:19","upload_time_iso_8601":"2024-08-02T06:48:19.594149Z","url":"https://files.pythonhosted.org/packages/be/89/412afc864df3715d377cff9fe15deadaccdc0902b0a242f742f286e6d84b/agentops-0.3.6-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"c3bf85f1439c3951ef69c81dbd7ef6df8a11df957e8d1180d835d71c11fa5131","md5":"b68d3124e365867f891bec4fb211a398","sha256":"0941f2486f3a561712ba6f77d560b49e2df55be141f243da0f9dc97ed43e6968"},"downloads":-1,"filename":"agentops-0.3.6.tar.gz","has_sig":false,"md5_digest":"b68d3124e365867f891bec4fb211a398","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":42933,"upload_time":"2024-08-02T06:48:21","upload_time_iso_8601":"2024-08-02T06:48:21.508300Z","url":"https://files.pythonhosted.org/packages/c3/bf/85f1439c3951ef69c81dbd7ef6df8a11df957e8d1180d835d71c11fa5131/agentops-0.3.6.tar.gz","yanked":false,"yanked_reason":null}],"0.3.7":[{"comment_text":"","digests":{"blake2b_256":"a34d05ba61e4fbd976dabe736d74fb2bb14d064ca758f05f084c0dadb6ac5cb1","md5":"551df1e89278270e0f5522d41f5c28ae","sha256":"7eeec5bef41e9ba397b3d880bcec8cd0818209ab31665c85e8b97615011a23d9"},"downloads":-1,"filename":"agentops-0.3.7-py3-none-any.whl","has_sig":false,"md5_digest":"551df1e89278270e0f5522d41f5c28ae","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":39816,"upload_time":"2024-08-08T23:21:45","upload_time_iso_8601":"2024-08-08T23:21:45.035395Z","url":"https://files.pythonhosted.org/packages/a3/4d/05ba61e4fbd976dabe736d74fb2bb14d064ca758f05f084c0dadb6ac5cb1/agentops-0.3.7-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"9f31034c3e062287f4fe9f57f2448e9508617a26bbb8a16b11c77cda9b28e1c0","md5":"1c48a797903a25988bae9b72559307ec","sha256":"048ee3caa5edf01b98c994e4e3ff90c09d83f820a43a70f07db96032c3386750"},"downloads":-1,"filename":"agentops-0.3.7.tar.gz","has_sig":false,"md5_digest":"1c48a797903a25988bae9b72559307ec","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":43495,"upload_time":"2024-08-08T23:21:46","upload_time_iso_8601":"2024-08-08T23:21:46.798531Z","url":"https://files.pythonhosted.org/packages/9f/31/034c3e062287f4fe9f57f2448e9508617a26bbb8a16b11c77cda9b28e1c0/agentops-0.3.7.tar.gz","yanked":false,"yanked_reason":null}],"0.3.9":[{"comment_text":"","digests":{"blake2b_256":"660ce931f892e0cedd40d861c3deff4134e1af1d226d6dc9762b32514d6dbc9f","md5":"82792de7bccabed058a24d3bd47443db","sha256":"582c9ddb30a9bb951b4d3ee2fd0428ba77d4a4367950b9cc6043f45b10bf12d8"},"downloads":-1,"filename":"agentops-0.3.9-py3-none-any.whl","has_sig":false,"md5_digest":"82792de7bccabed058a24d3bd47443db","packagetype":"bdist_wheel","python_version":"py3","requires_python":">=3.7","size":40235,"upload_time":"2024-08-15T21:21:33","upload_time_iso_8601":"2024-08-15T21:21:33.468748Z","url":"https://files.pythonhosted.org/packages/66/0c/e931f892e0cedd40d861c3deff4134e1af1d226d6dc9762b32514d6dbc9f/agentops-0.3.9-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":"","digests":{"blake2b_256":"e17b68cef3aaf44d423046b7779e9325e4feef5257e6d784a55c9dadf84bd61a","md5":"470f3b2663b71eb2f1597903bf8922e7","sha256":"7c999edbc64196924acdb06da09ec664a09d9fec8e73ba4e0f89e5f3dafc79e5"},"downloads":-1,"filename":"agentops-0.3.9.tar.gz","has_sig":false,"md5_digest":"470f3b2663b71eb2f1597903bf8922e7","packagetype":"sdist","python_version":"source","requires_python":">=3.7","size":43796,"upload_time":"2024-08-15T21:21:34","upload_time_iso_8601":"2024-08-15T21:21:34.591272Z","url":"https://files.pythonhosted.org/packages/e1/7b/68cef3aaf44d423046b7779e9325e4feef5257e6d784a55c9dadf84bd61a/agentops-0.3.9.tar.gz","yanked":false,"yanked_reason":null}],"0.4.0":[{"comment_text":null,"digests":{"blake2b_256":"060e66184fab1fc3bdd955ac20ea7bdef78f5b9aecc4080ea3e054c2a2436991","md5":"250de44e3599992c75625cef67682ecd","sha256":"b4821b8ec69c05a4d13b34eaad4762bb06a4f14e1241d57c16fdd28de5c8c929"},"downloads":-1,"filename":"agentops-0.4.0-py3-none-any.whl","has_sig":false,"md5_digest":"250de44e3599992c75625cef67682ecd","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":171419,"upload_time":"2025-03-13T11:24:15","upload_time_iso_8601":"2025-03-13T11:24:15.042606Z","url":"https://files.pythonhosted.org/packages/06/0e/66184fab1fc3bdd955ac20ea7bdef78f5b9aecc4080ea3e054c2a2436991/agentopLine truncated
+        dependencies"},{"comment_text":null,"digests":{"blake2b_256":"ff7f8a57d060489c780db3e15c4d9ff8c670e5db583549c74dd2d32ae6ec10c0","md5":"ea0932849a7311750c6ac0e567c90182","sha256":"45f5367cecd8a0b648055b6ec76e8a6a2801425e80dede8f86b39e9c6cfe1d98"},"downloads":-1,"filename":"agentops-0.4.0.tar.gz","has_sig":false,"md5_digest":"ea0932849a7311750c6ac0e567c90182","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":248757,"upload_time":"2025-03-13T11:24:16","upload_time_iso_8601":"2025-03-13T11:24:16.866033Z","url":"https://files.pythonhosted.org/packages/ff/7f/8a57d060489c780db3e15c4d9ff8c670e5db583549c74dd2d32ae6ec10c0/agentops-0.4.0.tar.gz","yanked":true,"yanked_reason":"broken
+        dependencies"}],"0.4.1":[{"comment_text":null,"digests":{"blake2b_256":"736e7ab03c56260ec59bfaeeb08efb76f55ec6153861ad2a9cf20b38b222e4e7","md5":"3fcebe0141ca19b2fbcb53e918003ce9","sha256":"69c944e22628bc0f52c534007d2453da2a1988a7fd1f993720c4a15b0f70465a"},"downloads":-1,"filename":"agentops-0.4.1-py3-none-any.whl","has_sig":false,"md5_digest":"3fcebe0141ca19b2fbcb53e918003ce9","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":171402,"upload_time":"2025-03-13T16:29:26","upload_time_iso_8601":"2025-03-13T16:29:26.477091Z","url":"https://files.pythonhosted.org/packages/73/6e/7ab03c56260ec59bfaeeb08efb76f55ec6153861ad2a9cf20b38b222e4e7/agentops-0.4.1-py3-none-any.whl","yanked":true,"yanked_reason":"Broken
+        dependencies"},{"comment_text":null,"digests":{"blake2b_256":"ca303217cd3480ad099ffa92848ccbc8672e5232c22918c95a4b99e49c0ef31e","md5":"ec421fa88b575b827fc0d3fd02f45515","sha256":"fec044f0346dca6aba17e458e669ac1f52f1b618a4a15b43342615096c5e7d56"},"downloads":-1,"filename":"agentops-0.4.1.tar.gz","has_sig":false,"md5_digest":"ec421fa88b575b827fc0d3fd02f45515","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":248747,"upload_time":"2025-03-13T16:29:27","upload_time_iso_8601":"2025-03-13T16:29:27.905694Z","url":"https://files.pythonhosted.org/packages/ca/30/3217cd3480ad099ffa92848ccbc8672e5232c22918c95a4b99e49c0ef31e/agentops-0.4.1.tar.gz","yanked":true,"yanked_reason":"Broken
+        dependencies"}],"0.4.10":[{"comment_text":null,"digests":{"blake2b_256":"301e0fe4fb617a5a69a8692b571d726f03e713a37d94d6a43c595a08fc33cff3","md5":"5ac7ec12e80bae6946dc10e46ef768f7","sha256":"917ad7ad51af0ca00cace2a3ae1d1d36e0d65dc813e030fcd377ff98535002bd"},"downloads":-1,"filename":"agentops-0.4.10-py3-none-any.whl","has_sig":false,"md5_digest":"5ac7ec12e80bae6946dc10e46ef768f7","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":198777,"upload_time":"2025-05-08T20:37:29","upload_time_iso_8601":"2025-05-08T20:37:29.322288Z","url":"https://files.pythonhosted.org/packages/30/1e/0fe4fb617a5a69a8692b571d726f03e713a37d94d6a43c595a08fc33cff3/agentops-0.4.10-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"a0ef0a56be3981bd464ad5a22fa3a859421f4b5560cbbb082f3ef9aca9cdb1a7","md5":"1954d07bfa38ba5c5ce0e516b7dbfdc9","sha256":"b66a48b4ec50c9cb34abc6ff1df873f0dcddbbb528d8a8c0527cb97b24c91b36"},"downloads":-1,"filename":"agentops-0.4.10.tar.gz","has_sig":false,"md5_digest":"1954d07bfa38ba5c5ce0e516b7dbfdc9","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":284727,"upload_time":"2025-05-08T20:37:30","upload_time_iso_8601":"2025-05-08T20:37:30.744217Z","url":"https://files.pythonhosted.org/packages/a0/ef/0a56be3981bd464ad5a22fa3a859421f4b5560cbbb082f3ef9aca9cdb1a7/agentops-0.4.10.tar.gz","yanked":false,"yanked_reason":null}],"0.4.11":[{"comment_text":null,"digests":{"blake2b_256":"35cde66dea05d2d8070f886e8f4ce86905cf1cce2f89622e041f26e39f717c9e","md5":"20424d54ba76517d586d4bcc92dda3bf","sha256":"b08c84fd69f36fcd5d6f2b14d16ff88b977a9a417d92448c9709f3c7990d6438"},"downloads":-1,"filename":"agentops-0.4.11-py3-none-any.whl","has_sig":false,"md5_digest":"20424d54ba76517d586d4bcc92dda3bf","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":198789,"upload_time":"2025-05-12T20:38:29","upload_time_iso_8601":"2025-05-12T20:38:29.202046Z","url":"https://files.pythonhosted.org/packages/35/cd/e66dea05d2d8070f886e8f4ce86905cf1cce2f89622e041f26e39f717c9e/agentops-0.4.11-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"349df76fc1760cb21788967db3dd22ff2e6521c42b8ecee152e6ac4278e7cade","md5":"b7affd8b15834e4f9cb63066d7d160d1","sha256":"6eb80ee4a0653f9bdc9fc7641bf60cb7546cd34ff1c04dfbc4fca77dbb07edda"},"downloads":-1,"filename":"agentops-0.4.11.tar.gz","has_sig":false,"md5_digest":"b7affd8b15834e4f9cb63066d7d160d1","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":284735,"upload_time":"2025-05-12T20:38:30","upload_time_iso_8601":"2025-05-12T20:38:30.393540Z","url":"https://files.pythonhosted.org/packages/34/9d/f76fc1760cb21788967db3dd22ff2e6521c42b8ecee152e6ac4278e7cade/agentops-0.4.11.tar.gz","yanked":false,"yanked_reason":null}],"0.4.12":[{"comment_text":null,"digests":{"blake2b_256":"eb86772ed94e4e55433e8014933dab08aa6dfbcd8072f7fd74ffcad335ba0e73","md5":"831a3d54bccce09cc6c2a352776d02e6","sha256":"7c2685ae9c9de1a1071f6a29d395444191744d5ee58e33c020a69e2388dc2f7c"},"downloads":-1,"filename":"agentops-0.4.12-py3-none-any.whl","has_sig":false,"md5_digest":"831a3d54bccce09cc6c2a352776d02e6","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":198319,"upload_time":"2025-05-15T19:59:27","upload_time_iso_8601":"2025-05-15T19:59:27.609093Z","url":"https://files.pythonhosted.org/packages/eb/86/772ed94e4e55433e8014933dab08aa6dfbcd8072f7fd74ffcad335ba0e73/agentops-0.4.12-py3-none-any.whl","yanked":false,"yanked_reason":null},{"comment_text":null,"digests":{"blake2b_256":"0cf664cea8e916a305d2dc2f3f3840a1d4cae40e1927892e1fcc11f83ec7ebee","md5":"7e97e0612a6e8544b37a2fa2e1633166","sha256":"530f15d428a4c78db918fa766366c8f11105c4d1d3b1a56de027747d805a573f"},"downloads":-1,"filename":"agentops-0.4.12.tar.gz","has_sig":false,"md5_digest":"7e97e0612a6e8544b37a2fa2e1633166","packagetype":"sdist","python_version":"source","requires_python":"<3.14,>=3.9","size":284309,"upload_time":"2025-05-15T19:59:28","upload_time_iso_8601":"2025-05-15T19:59:28.955745Z","url":"https://files.pythonhosted.org/packages/0c/f6/64cea8e916a305d2dc2f3f3840a1d4cae40e1927892e1fcc11f83ec7ebee/agentops-0.4.12.tar.gz","yanked":false,"yanked_reason":null}],"0.4.13":[{"comment_text":null,"digests":{"blake2b_256":"637f1514550d55e8ba0e2aef4f652678413e9979f4f6c019d8032cfd9fade10e","md5":"ddea9230651973616b50a1f089657999","sha256":"256cfcd4eb257d0a3c9538bd461ffe1dceb15cd0627b405b45d99661d8925247"},"downloads":-1,"filename":"agentops-0.4.13-py3-none-any.whl","has_sig":false,"md5_digest":"ddea9230651973616b50a1f089657999","packagetype":"bdist_wheel","python_version":"py3","requires_python":"<3.14,>=3.9","size":214973,"upload_time":"2025-05-27T22:32:40","upload_time_iso_8601":"2025-05-27T22:32:40.986531Z","url":"https://files.pythonhosted.org/packages/63/7f/1514550d55e8ba0e2aef4f652678413e9979f4f6c019Line truncated
+
+        '
+    headers:
+      Accept-Ranges:
+      - bytes
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '148467'
+      Date:
+      - Wed, 09 Jul 2025 19:06:37 GMT
+      Permissions-Policy:
+      - publickey-credentials-create=(self),publickey-credentials-get=(self),accelerometer=(),ambient-light-sensor=(),autoplay=(),battery=(),camera=(),display-capture=(),document-domain=(),encrypted-media=(),execution-while-not-rendered=(),execution-while-out-of-viewport=(),fullscreen=(),gamepad=(),geolocation=(),gyroscope=(),hid=(),identity-credentials-get=(),idle-detection=(),local-fonts=(),magnetometer=(),microphone=(),midi=(),otp-credentials=(),payment=(),picture-in-picture=(),screen-wake-lock=(),serial=(),speaker-selection=(),storage-access=(),usb=(),web-share=(),xr-spatial-tracking=()
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubDomains; preload
+      Vary:
+      - Accept-Encoding
+      X-Cache:
+      - MISS, HIT, HIT
+      X-Cache-Hits:
+      - 0, 931, 1
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Permitted-Cross-Domain-Policies:
+      - none
+      X-Served-By:
+      - cache-iad-kjyo7100056-IAD, cache-iad-kjyo7100044-IAD, cache-gru-sbgr1930062-GRU
+      X-Timer:
+      - S1752087997.099673,VS0,VE2
+      X-XSS-Protection:
+      - 1; mode=block
+      access-control-allow-headers:
+      - Content-Type, If-Match, If-Modified-Since, If-None-Match, If-Unmodified-Since
+      access-control-allow-methods:
+      - GET
+      access-control-allow-origin:
+      - '*'
+      access-control-expose-headers:
+      - X-PyPI-Last-Serial
+      access-control-max-age:
+      - '86400'
+      cache-control:
+      - max-age=900, public
+      content-security-policy:
+      - base-uri 'self'; connect-src 'self' https://api.github.com/repos/ https://api.github.com/search/issues
+        https://gitlab.com/api/ https://analytics.python.org fastly-insights.com *.fastly-insights.com
+        *.ethicalads.io https://api.pwnedpasswords.com https://cdn.jsdelivr.net/npm/mathjax@3.2.2/es5/sre/mathmaps/
+        https://2p66nmmycsj3.statuspage.io; default-src 'none'; font-src 'self' fonts.gstatic.com;
+        form-action 'self' https://checkout.stripe.com https://billing.stripe.com;
+        frame-ancestors 'none'; frame-src 'none'; img-src 'self' https://pypi-camo.freetls.fastly.net/
+        *.fastly-insights.com *.ethicalads.io ethicalads.blob.core.windows.net; script-src
+        'self' https://analytics.python.org *.fastly-insights.com *.ethicalads.io
+        'sha256-U3hKDidudIaxBDEzwGJApJgPEf2mWk6cfMWghrAa6i0=' https://cdn.jsdelivr.net/npm/mathjax@3.2.2/
+        'sha256-1CldwzdEg2k1wTmf7s5RWVd7NMXI/7nxxjJM2C4DqII=' 'sha256-0POaN8stWYQxhzjKS+/eOfbbJ/u4YHO5ZagJvLpMypo=';
+        style-src 'self' fonts.googleapis.com *.ethicalads.io 'sha256-2YHqZokjiizkHi1Zt+6ar0XJ0OeEy/egBnlm+MDMtrM='
+        'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' 'sha256-JLEjeN9e5dGsz5475WyRaoA4eQOdNPxDIeUhclnJDCE='
+        'sha256-mQyxHEuwZJqpxCw3SLmc4YOySNKXunyu2Oiz1r3/wAE=' 'sha256-OCf+kv5Asiwp++8PIevKBYSgnNLNUZvxAp4a7wMLuKA='
+        'sha256-h5LOiLhk6wiJrGsG5ItM0KimwzWQH/yAcmoJDJL//bY='; worker-src *.fastly-insights.com
+      content-type:
+      - application/json
+      etag:
+      - '"MJKXiK9CyQEJ7WiUzX702g"'
+      referrer-policy:
+      - origin-when-cross-origin
+      x-pypi-last-serial:
+      - '29926354'
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are Test Agent. An agent
+      created for testing purposes\nYour personal goal is: Complete test tasks successfully\nTo
+      give my best complete final answer to the task respond using the exact following
+      format:\n\nThought: I now can give a great answer\nFinal Answer: Your final
+      answer must be the great and the most complete as possible, it must be outcome
+      described.\n\nI MUST use these formats, my job depends on it!"}, {"role": "user",
+      "content": "\nCurrent Task: Test task description\n\nThis is the expected criteria
+      for your final answer: Expected test output\nyou MUST return the actual complete
+      content as the final answer, not a summary.\n\nBegin! This is VERY important
+      to you, use the tools available and give your best Final Answer, your job depends
+      on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '879'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.93.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.93.0
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.12
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-BrUPlyy6FJgrPxOZMBZbbIE86pw5y\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1752087997,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"I now can give a great answer  \\nFinal
+      Answer: The expected test output is a comprehensive document that outlines the
+      specific parameters and criteria that define success for the task at hand. It
+      should include detailed descriptions of the tasks, the goals that need to be
+      achieved, and any specific formatting or structural requirements necessary for
+      the output. Each component of the task must be analyzed and addressed, providing
+      context as well as examples where applicable. Additionally, any tools or methodologies
+      that are relevant to executing the tasks successfully should be outlined, including
+      any potential risks or challenges that may arise during the process. This document
+      serves as a guiding framework to ensure that all aspects of the task are thoroughly
+      considered and executed to meet the high standards expected.\",\n        \"refusal\":
+      null,\n        \"annotations\": []\n      },\n      \"logprobs\": null,\n      \"finish_reason\":
+      \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 161,\n    \"completion_tokens\":
+      142,\n    \"total_tokens\": 303,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\":
+      \"default\",\n  \"system_fingerprint\": \"fp_34a54ae93c\"\n}\n"
+    headers:
+      CF-RAY:
+      - 95ca197e89637df2-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 09 Jul 2025 19:06:39 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=.t6bOXGdf07C3xXSw4OEg_f3icel94zEZRH72nzbd2I-1752087999-1.0.1.1-6h4F1dxRC8TZNmW0mIEwZADPtL9aYoS7gEP4wcgn5fvf57s3W7WhUYNoYMRPoCTTSarVSBwBgtGgVzAd5EhbDkXGlv3NOqS3dE6CBS_gZ9E;
+        path=/; expires=Wed, 09-Jul-25 19:36:39 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=XwsgBfgvDGlKFQ4LiGYGIARIoSNTiwidqoo9UZcc.XY-1752087999227-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '1872'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '1875'
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999813'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_fdd2d2f329227f49b6ce3970bba95b31
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are an expert evaluator
+      assessing how well an AI agent''s output aligns with its assigned task goal.\n\nScore
+      the agent''s goal alignment on a scale from 0-10 where:\n- 0: Complete misalignment,
+      agent did not understand or attempt the task goal\n- 5: Partial alignment, agent
+      attempted the task but missed key requirements\n- 10: Perfect alignment, agent
+      fully satisfied all task requirements\n\nConsider:\n1. Did the agent correctly
+      interpret the task goal?\n2. Did the final output directly address the requirements?\n3.
+      Did the agent focus on relevant aspects of the task?\n4. Did the agent provide
+      all requested information or deliverables?\n\nReturn your evaluation as JSON
+      with fields ''score'' (number) and ''feedback'' (string).\n"}, {"role": "user",
+      "content": "\nAgent role: Test Agent\nAgent goal: Complete test tasks successfully\nTask
+      description: Test task description\nExpected output: Expected test output\n\nAgent''s
+      final output:\nThe expected test output is a comprehensive document that outlines
+      the specific parameters and criteria that define success for the task at hand.
+      It should include detailed descriptions of the tasks, the goals that need to
+      be achieved, and any specific formatting or structural requirements necessary
+      for the output. Each component of the task must be analyzed and addressed, providing
+      context as well as examples where applicable. Additionally, any tools or methodologies
+      that are relevant to executing the tasks successfully should be outlined, including
+      any potential risks or challenges that may arise during the process. This document
+      serves as a guiding framework to ensure that all aspects of the task are thoroughly
+      considered and executed to meet the high standards expected.\n\nEvaluate how
+      well the agent''s output aligns with the assigned task goal.\n"}], "model":
+      "gpt-4o-mini", "stop": []}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '1893'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=.t6bOXGdf07C3xXSw4OEg_f3icel94zEZRH72nzbd2I-1752087999-1.0.1.1-6h4F1dxRC8TZNmW0mIEwZADPtL9aYoS7gEP4wcgn5fvf57s3W7WhUYNoYMRPoCTTSarVSBwBgtGgVzAd5EhbDkXGlv3NOqS3dE6CBS_gZ9E;
+        _cfuvid=XwsgBfgvDGlKFQ4LiGYGIARIoSNTiwidqoo9UZcc.XY-1752087999227-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.93.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.93.0
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.12
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    content: "{\n  \"id\": \"chatcmpl-BrUPn4pG0PkiwTx9zAwzBaBYj6HG3\",\n  \"object\":
+      \"chat.completion\",\n  \"created\": 1752087999,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n
+      \ \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\":
+      \"assistant\",\n        \"content\": \"{\\n  \\\"score\\\": 5,\\n  \\\"feedback\\\":
+      \\\"The agent's output demonstrates an understanding of the need for a comprehensive
+      document outlining task parameters and success criteria. However, it does not
+      explicitly provide the expected test output or directly address the specific
+      test tasks as described in the task definition. The agent missed delivering
+      the precise expected output and did not include clear examples or structure
+      that align with the task at hand.\\\"\\n}\",\n        \"refusal\": null,\n        \"annotations\":
+      []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n
+      \   }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 344,\n    \"completion_tokens\":
+      84,\n    \"total_tokens\": 428,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\":
+      0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n
+      \     \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\":
+      0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\":
+      \"default\",\n  \"system_fingerprint\": \"fp_62a23a81ef\"\n}\n"
+    headers:
+      CF-RAY:
+      - 95ca198b5aef7df2-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 09 Jul 2025 19:06:40 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '1197'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '1200'
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999559'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_2a5ac0597056b7459275d4645963e215
+    http_version: HTTP/1.1
+    status_code: 200
+- request:
+    body: !!binary |
+      CuEMCiQKIgoMc2VydmljZS5uYW1lEhIKEGNyZXdBSS10ZWxlbWV0cnkSuAwKEgoQY3Jld2FpLnRl
+      bGVtZXRyeRKeCAoQNBzmDkHxynob0iVvgLhjnRIISZd+egh6hQQqDENyZXcgQ3JlYXRlZDABOVj2
+      1hXAqlAYQSg/4RXAqlAYShsKDmNyZXdhaV92ZXJzaW9uEgkKBzAuMTM0LjBKGwoOcHl0aG9uX3Zl
+      cnNpb24SCQoHMy4xMS4xMkouCghjcmV3X2tleRIiCiA3MWMxN2VmMjcwYzI2MTgzMTI4Y2NlNDgz
+      YzE4NWU3NUoxCgdjcmV3X2lkEiYKJGI3N2U2NWNiLTZhYjQtNGJjMi05ZDMzLTcxN2FiNTVhMzBk
+      MkocCgxjcmV3X3Byb2Nlc3MSDAoKc2VxdWVudGlhbEoRCgtjcmV3X21lbW9yeRICEABKGgoUY3Jl
+      d19udW1iZXJfb2ZfdGFza3MSAhgBShsKFWNyZXdfbnVtYmVyX29mX2FnZW50cxICGAFKOgoQY3Jl
+      d19maW5nZXJwcmludBImCiRjZDNmM2RhYi05NDBhLTQ2MTMtODgzOC1hMGM2YWQ2MDAzMDhKOwob
+      Y3Jld19maW5nZXJwcmludF9jcmVhdGVkX2F0EhwKGjIwMjUtMDctMDlUMTY6MDY6MzcuMDIyODI5
+      StECCgtjcmV3X2FnZW50cxLBAgq+Alt7ImtleSI6ICI1MzgxMzMzNTViYjlmZTZhYWZiN2U3YmYw
+      MzE2MTljNSIsICJpZCI6ICJlMjNmYWFmOC03M2M4LTQyNmQtOGJhYi1mN2Y2YjhhY2FkOWMiLCAi
+      cm9sZSI6ICJUZXN0IEFnZW50IiwgInZlcmJvc2U/IjogZmFsc2UsICJtYXhfaXRlciI6IDI1LCAi
+      bWF4X3JwbSI6IG51bGwsICJmdW5jdGlvbl9jYWxsaW5nX2xsbSI6ICIiLCAibGxtIjogImdwdC00
+      by1taW5pIiwgImRlbGVnYXRpb25fZW5hYmxlZD8iOiBmYWxzZSwgImFsbG93X2NvZGVfZXhlY3V0
+      aW9uPyI6IGZhbHNlLCAibWF4X3JldHJ5X2xpbWl0IjogMiwgInRvb2xzX25hbWVzIjogW119XUr/
+      AQoKY3Jld190YXNrcxLwAQrtAVt7ImtleSI6ICI0ZDNkNmE2NGM0ZjFhMzk2MTAzMGQ0NzViMTM0
+      ZTFjMyIsICJpZCI6ICJlMDczNGMwOC03MzE5LTQ0YmYtYTNiMi1kYjEwNzVkZjA5ODAiLCAiYXN5
+      bmNfZXhlY3V0aW9uPyI6IGZhbHNlLCAiaHVtYW5faW5wdXQ/IjogZmFsc2UsICJhZ2VudF9yb2xl
+      IjogIlRlc3QgQWdlbnQiLCAiYWdlbnRfa2V5IjogIjUzODEzMzM1NWJiOWZlNmFhZmI3ZTdiZjAz
+      MTYxOWM1IiwgInRvb2xzX25hbWVzIjogW119XXoCGAGFAQABAAASgAQKECJsWbMSQx8UwmQntIV8
+      1RoSCIiJiUMNOuNFKgxUYXNrIENyZWF0ZWQwATn4qvoVwKpQGEEonfsVwKpQGEouCghjcmV3X2tl
+      eRIiCiA3MWMxN2VmMjcwYzI2MTgzMTI4Y2NlNDgzYzE4NWU3NUoxCgdjcmV3X2lkEiYKJGI3N2U2
+      NWNiLTZhYjQtNGJjMi05ZDMzLTcxN2FiNTVhMzBkMkouCgh0YXNrX2tleRIiCiA0ZDNkNmE2NGM0
+      ZjFhMzk2MTAzMGQ0NzViMTM0ZTFjM0oxCgd0YXNrX2lkEiYKJGUwNzM0YzA4LTczMTktNDRiZi1h
+      M2IyLWRiMTA3NWRmMDk4MEo6ChBjcmV3X2ZpbmdlcnByaW50EiYKJGNkM2YzZGFiLTk0MGEtNDYx
+      My04ODM4LWEwYzZhZDYwMDMwOEo6ChB0YXNrX2ZpbmdlcnByaW50EiYKJDlmMWI3ZmVlLTEwNmYt
+      NDI1MS1iYWY5LTVhOGYzMWQ4NWFmZUo7Cht0YXNrX2ZpbmdlcnByaW50X2NyZWF0ZWRfYXQSHAoa
+      MjAyNS0wNy0wOVQxNjowNjozNy4wMjI3ODJKOwoRYWdlbnRfZmluZ2VycHJpbnQSJgokMTQ0MmQy
+      NTQtYmEyNS00NjkwLTg4ZTUtNGMyYzA2YWQwZmM4egIYAYUBAAEAAA==
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '1636'
+      Content-Type:
+      - application/x-protobuf
+      User-Agent:
+      - OTel-OTLP-Exporter-Python/1.34.1
+    method: POST
+    uri: https://telemetry.crewai.com:4319/v1/traces
+  response:
+    body:
+      string: "\n\0"
+    headers:
+      Content-Length:
+      - '2'
+      Content-Type:
+      - application/x-protobuf
+      Date:
+      - Wed, 09 Jul 2025 19:06:41 GMT
+    status:
+      code: 200
+      message: OK
+version: 1

tests/evaluation/metrics/base_evaluation_metrics_test.py

@@ -0,0 +1,28 @@
+import pytest
+from unittest.mock import MagicMock
+from crewai.agent import Agent
+from crewai.task import Task
+
+class BaseEvaluationMetricsTest:
+    @pytest.fixture
+    def mock_agent(self):
+        agent = MagicMock(spec=Agent)
+        agent.id = "test_agent_id"
+        agent.role = "Test Agent"
+        agent.goal = "Test goal"
+        agent.tools = []
+        return agent
+
+    @pytest.fixture
+    def mock_task(self):
+        task = MagicMock(spec=Task)
+        task.description = "Test task description"
+        task.expected_output = "Test expected output"
+        return task
+
+    @pytest.fixture
+    def execution_trace(self):
+        return {
+            "thinking": ["I need to analyze this data carefully"],
+            "actions": ["Gathered information", "Analyzed data"]
+        }

tests/evaluation/metrics/test_goal_metrics.py

@@ -0,0 +1,59 @@
+from unittest.mock import patch, MagicMock
+from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+
+from crewai.evaluation.base_evaluator import EvaluationScore
+from crewai.evaluation.metrics.goal_metrics import GoalAlignmentEvaluator
+from crewai.utilities.llm_utils import LLM
+
+
+class TestGoalAlignmentEvaluator(BaseEvaluationMetricsTest):
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluate_success(self, mock_create_llm, mock_agent, mock_task, execution_trace):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "score": 8.5,
+            "feedback": "The agent correctly understood the task and produced relevant output."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        evaluator = GoalAlignmentEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="This is the final output"
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score == 8.5
+        assert "correctly understood the task" in result.feedback
+
+        mock_llm.call.assert_called_once()
+        prompt = mock_llm.call.call_args[0][0]
+        assert len(prompt) >= 2
+        assert "system" in prompt[0]["role"]
+        assert "user" in prompt[1]["role"]
+        assert mock_agent.role in prompt[1]["content"]
+        assert mock_task.description in prompt[1]["content"]
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluate_error_handling(self, mock_create_llm, mock_agent, mock_task, execution_trace):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = "Invalid JSON response"
+        mock_create_llm.return_value = mock_llm
+
+        evaluator = GoalAlignmentEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="This is the final output"
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score is None
+        assert "Failed to parse" in result.feedback

tests/evaluation/metrics/test_reasoning_metrics.py

@@ -0,0 +1,166 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from typing import List, Dict, Any
+
+from crewai.tasks.task_output import TaskOutput
+from crewai.evaluation.metrics.reasoning_metrics import (
+    ReasoningEfficiencyEvaluator,
+)
+from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from crewai.utilities.llm_utils import LLM
+from crewai.evaluation.base_evaluator import EvaluationScore
+
+class TestReasoningEfficiencyEvaluator(BaseEvaluationMetricsTest):
+    @pytest.fixture
+    def mock_output(self):
+        output = MagicMock(spec=TaskOutput)
+        output.raw = "This is the test output"
+        return output
+
+    @pytest.fixture
+    def llm_calls(self) -> List[Dict[str, Any]]:
+        return [
+            {
+                "prompt": "How should I approach this task?",
+                "response": "I'll first research the topic, then compile findings.",
+                "timestamp": 1626987654
+            },
+            {
+                "prompt": "What resources should I use?",
+                "response": "I'll use relevant academic papers and reliable websites.",
+                "timestamp": 1626987754
+            },
+            {
+                "prompt": "How should I structure the output?",
+                "response": "I'll organize information clearly with headings and bullet points.",
+                "timestamp": 1626987854
+            }
+        ]
+
+    def test_insufficient_llm_calls(self, mock_agent, mock_task, mock_output):
+        execution_trace = {"llm_calls": []}
+
+        evaluator = ReasoningEfficiencyEvaluator()
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output=mock_output
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score is None
+        assert "Insufficient LLM calls" in result.feedback
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_successful_evaluation(self, mock_create_llm, mock_agent, mock_task, mock_output, llm_calls):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "scores": {
+                "focus": 8.0,
+                "progression": 7.0,
+                "decision_quality": 7.5,
+                "conciseness": 8.0,
+                "loop_avoidance": 9.0
+            },
+            "overall_score": 7.9,
+            "feedback": "The agent demonstrated good reasoning efficiency.",
+            "optimization_suggestions": "The agent could improve by being more concise."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace with sufficient LLM calls
+        execution_trace = {"llm_calls": llm_calls}
+
+        # Mock the _detect_loops method to return a simple result
+        evaluator = ReasoningEfficiencyEvaluator(llm=mock_llm)
+        evaluator._detect_loops = MagicMock(return_value=(False, []))
+
+        # Evaluate
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output=mock_output
+        )
+
+        # Assertions
+        assert isinstance(result, EvaluationScore)
+        assert result.score == 7.9
+        assert "The agent demonstrated good reasoning efficiency" in result.feedback
+        assert "Reasoning Efficiency Evaluation:" in result.feedback
+        assert "• Focus: 8.0/10" in result.feedback
+
+        # Verify LLM was called
+        mock_llm.call.assert_called_once()
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_parse_error_handling(self, mock_create_llm, mock_agent, mock_task, mock_output, llm_calls):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = "Invalid JSON response"
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace
+        execution_trace = {"llm_calls": llm_calls}
+
+        # Mock the _detect_loops method
+        evaluator = ReasoningEfficiencyEvaluator(llm=mock_llm)
+        evaluator._detect_loops = MagicMock(return_value=(False, []))
+
+        # Evaluate
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output=mock_output
+        )
+
+        # Assertions for error handling
+        assert isinstance(result, EvaluationScore)
+        assert result.score is None
+        assert "Failed to parse reasoning efficiency evaluation" in result.feedback
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_loop_detection(self, mock_create_llm, mock_agent, mock_task, mock_output):
+        # Setup LLM calls with a repeating pattern
+        repetitive_llm_calls = [
+            {"prompt": "How to solve?", "response": "I'll try method A", "timestamp": 1000},
+            {"prompt": "Let me try method A", "response": "It didn't work", "timestamp": 1100},
+            {"prompt": "How to solve?", "response": "I'll try method A again", "timestamp": 1200},
+            {"prompt": "Let me try method A", "response": "It didn't work", "timestamp": 1300},
+            {"prompt": "How to solve?", "response": "I'll try method A one more time", "timestamp": 1400}
+        ]
+
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "scores": {
+                "focus": 6.0,
+                "progression": 3.0,
+                "decision_quality": 4.0,
+                "conciseness": 6.0,
+                "loop_avoidance": 2.0
+            },
+            "overall_score": 4.2,
+            "feedback": "The agent is stuck in a reasoning loop.",
+            "optimization_suggestions": "The agent should try different approaches when one fails."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        execution_trace = {"llm_calls": repetitive_llm_calls}
+
+        evaluator = ReasoningEfficiencyEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output=mock_output
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score == 4.2
+        assert "• Loop Avoidance: 2.0/10" in result.feedback

tests/evaluation/metrics/test_semantic_quality_metrics.py

@@ -0,0 +1,82 @@
+from unittest.mock import patch, MagicMock
+
+from crewai.evaluation.base_evaluator import EvaluationScore
+from crewai.evaluation.metrics.semantic_quality_metrics import SemanticQualityEvaluator
+from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from crewai.utilities.llm_utils import LLM
+
+class TestSemanticQualityEvaluator(BaseEvaluationMetricsTest):
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluate_success(self, mock_create_llm, mock_agent, mock_task, execution_trace):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "score": 8.5,
+            "feedback": "The output is clear, coherent, and logically structured."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        evaluator = SemanticQualityEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="This is a well-structured analysis of the data."
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score == 8.5
+        assert "clear, coherent" in result.feedback
+
+        mock_llm.call.assert_called_once()
+        prompt = mock_llm.call.call_args[0][0]
+        assert len(prompt) >= 2
+        assert "system" in prompt[0]["role"]
+        assert "user" in prompt[1]["role"]
+        assert mock_agent.role in prompt[1]["content"]
+        assert mock_task.description in prompt[1]["content"]
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluate_with_empty_output(self, mock_create_llm, mock_agent, mock_task, execution_trace):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "score": 2.0,
+            "feedback": "The output is empty or minimal, lacking substance."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        evaluator = SemanticQualityEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output=""
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score == 2.0
+        assert "empty or minimal" in result.feedback
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluate_error_handling(self, mock_create_llm, mock_agent, mock_task, execution_trace):
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = "Invalid JSON response"
+        mock_create_llm.return_value = mock_llm
+
+        evaluator = SemanticQualityEvaluator(llm=mock_llm)
+
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="This is the output."
+        )
+
+        assert isinstance(result, EvaluationScore)
+        assert result.score is None
+        assert "Failed to parse" in result.feedback

tests/evaluation/metrics/test_tools_metrics.py

@@ -0,0 +1,230 @@
+from unittest.mock import patch, MagicMock
+
+from crewai.evaluation.metrics.tools_metrics import (
+    ToolSelectionEvaluator,
+    ParameterExtractionEvaluator,
+    ToolInvocationEvaluator
+)
+from crewai.utilities.llm_utils import LLM
+from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+
+class TestToolSelectionEvaluator(BaseEvaluationMetricsTest):
+    def test_no_tools_available(self, mock_task, mock_agent):
+        # Create agent with no tools
+        mock_agent.tools = []
+
+        execution_trace = {"tool_uses": []}
+
+        evaluator = ToolSelectionEvaluator()
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score is None
+        assert "no tools available" in result.feedback.lower()
+
+    def test_tools_available_but_none_used(self, mock_agent, mock_task):
+        mock_agent.tools = ["tool1", "tool2"]
+        execution_trace = {"tool_uses": []}
+
+        evaluator = ToolSelectionEvaluator()
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score is None
+        assert "had tools available but didn't use any" in result.feedback.lower()
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_successful_evaluation(self, mock_create_llm, mock_agent, mock_task):
+        # Setup mock LLM response
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "overall_score": 8.5,
+            "feedback": "The agent made good tool selections."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace with tool uses
+        execution_trace = {
+            "tool_uses": [
+                {"tool": "search_tool", "input": {"query": "test query"}, "output": "search results"},
+                {"tool": "calculator", "input": {"expression": "2+2"}, "output": "4"}
+            ]
+        }
+
+        evaluator = ToolSelectionEvaluator(llm=mock_llm)
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score == 8.5
+        assert "The agent made good tool selections" in result.feedback
+
+        # Verify LLM was called with correct prompt
+        mock_llm.call.assert_called_once()
+        prompt = mock_llm.call.call_args[0][0]
+        assert isinstance(prompt, list)
+        assert len(prompt) >= 2
+        assert "system" in prompt[0]["role"]
+        assert "user" in prompt[1]["role"]
+
+
+class TestParameterExtractionEvaluator(BaseEvaluationMetricsTest):
+    def test_no_tool_uses(self, mock_agent, mock_task):
+        execution_trace = {"tool_uses": []}
+
+        evaluator = ParameterExtractionEvaluator()
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score is None
+        assert "no tool usage" in result.feedback.lower()
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_successful_evaluation(self, mock_create_llm, mock_agent, mock_task):
+        mock_agent.tools = ["tool1", "tool2"]
+
+        # Setup mock LLM response
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "overall_score": 9.0,
+            "feedback": "The agent extracted parameters correctly."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace with tool uses
+        execution_trace = {
+            "tool_uses": [
+                {
+                    "tool": "search_tool",
+                    "input": {"query": "test query"},
+                    "output": "search results",
+                    "error": None
+                },
+                {
+                    "tool": "calculator",
+                    "input": {"expression": "2+2"},
+                    "output": "4",
+                    "error": None
+                }
+            ]
+        }
+
+        evaluator = ParameterExtractionEvaluator(llm=mock_llm)
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score == 9.0
+        assert "The agent extracted parameters correctly" in result.feedback
+
+
+class TestToolInvocationEvaluator(BaseEvaluationMetricsTest):
+    def test_no_tool_uses(self, mock_agent, mock_task):
+        execution_trace = {"tool_uses": []}
+
+        evaluator = ToolInvocationEvaluator()
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score is None
+        assert "no tool usage" in result.feedback.lower()
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_successful_evaluation(self, mock_create_llm, mock_agent, mock_task):
+        mock_agent.tools = ["tool1", "tool2"]
+        # Setup mock LLM response
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "overall_score": 8.0,
+            "feedback": "The agent invoked tools correctly."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace with tool uses
+        execution_trace = {
+            "tool_uses": [
+                {"tool": "search_tool", "input": {"query": "test query"}, "output": "search results"},
+                {"tool": "calculator", "input": {"expression": "2+2"}, "output": "4"}
+            ]
+        }
+
+        evaluator = ToolInvocationEvaluator(llm=mock_llm)
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score == 8.0
+        assert "The agent invoked tools correctly" in result.feedback
+
+    @patch("crewai.utilities.llm_utils.create_llm")
+    def test_evaluation_with_errors(self, mock_create_llm, mock_agent, mock_task):
+        mock_agent.tools = ["tool1", "tool2"]
+        # Setup mock LLM response
+        mock_llm = MagicMock(spec=LLM)
+        mock_llm.call.return_value = """
+        {
+            "overall_score": 5.5,
+            "feedback": "The agent had some errors in tool invocation."
+        }
+        """
+        mock_create_llm.return_value = mock_llm
+
+        # Setup execution trace with tool uses including errors
+        execution_trace = {
+            "tool_uses": [
+                {
+                    "tool": "search_tool",
+                    "input": {"query": "test query"},
+                    "output": "search results",
+                    "error": None
+                },
+                {
+                    "tool": "calculator",
+                    "input": {"expression": "2+"},
+                    "output": None,
+                    "error": "Invalid expression"
+                }
+            ]
+        }
+
+        evaluator = ToolInvocationEvaluator(llm=mock_llm)
+        result = evaluator.evaluate(
+            agent=mock_agent,
+            task=mock_task,
+            execution_trace=execution_trace,
+            final_output="Final output"
+        )
+
+        assert result.score == 5.5
+        assert "The agent had some errors in tool invocation" in result.feedback

tests/evaluation/test_agent_evaluator.py

@@ -0,0 +1,95 @@
+import pytest
+
+from crewai.agent import Agent
+from crewai.task import Task
+from crewai.crew import Crew
+from crewai.evaluation.agent_evaluator import AgentEvaluator
+from crewai.evaluation.base_evaluator import AgentEvaluationResult
+from crewai.evaluation import (
+    GoalAlignmentEvaluator,
+    SemanticQualityEvaluator,
+    ToolSelectionEvaluator,
+    ParameterExtractionEvaluator,
+    ToolInvocationEvaluator,
+    ReasoningEfficiencyEvaluator
+)
+
+from crewai.evaluation import create_default_evaluator
+class TestAgentEvaluator:
+    @pytest.fixture
+    def mock_crew(self):
+        agent = Agent(
+            role="Test Agent",
+            goal="Complete test tasks successfully",
+            backstory="An agent created for testing purposes",
+            allow_delegation=False,
+            verbose=False
+        )
+
+        task = Task(
+            description="Test task description",
+            agent=agent,
+            expected_output="Expected test output"
+        )
+
+        crew = Crew(
+            agents=[agent],
+            tasks=[task]
+        )
+        return crew
+
+    def test_set_iteration(self):
+        agent_evaluator = AgentEvaluator()
+
+        agent_evaluator.set_iteration(3)
+        assert agent_evaluator.iteration == 3
+
+    @pytest.mark.vcr(filter_headers=["authorization"])
+    def test_evaluate_current_iteration(self, mock_crew):
+        agent_evaluator = AgentEvaluator(crew=mock_crew, evaluators=[GoalAlignmentEvaluator()])
+
+        mock_crew.kickoff()
+
+        results = agent_evaluator.evaluate_current_iteration()
+
+        assert isinstance(results, dict)
+
+        agent, = mock_crew.agents
+        task, = mock_crew.tasks
+
+        assert len(mock_crew.agents) == 1
+        assert agent.role in results
+        assert len(results[agent.role]) == 1
+
+        result, = results[agent.role]
+        assert isinstance(result, AgentEvaluationResult)
+
+        assert result.agent_id == str(agent.id)
+        assert result.task_id == str(task.id)
+
+        goal_alignment, = result.metrics.values()
+        assert goal_alignment.score == 5.0
+
+        expected_feedback = "The agent's output demonstrates an understanding of the need for a comprehensive document"
+        assert expected_feedback in goal_alignment.feedback
+
+        assert goal_alignment.raw_response is not None
+        assert '"score": 5' in goal_alignment.raw_response
+
+    def test_create_default_evaluator(self, mock_crew):
+        agent_evaluator = create_default_evaluator(crew=mock_crew)
+        assert isinstance(agent_evaluator, AgentEvaluator)
+        assert agent_evaluator.crew == mock_crew
+
+        expected_types = [
+            GoalAlignmentEvaluator,
+            SemanticQualityEvaluator,
+            ToolSelectionEvaluator,
+            ParameterExtractionEvaluator,
+            ToolInvocationEvaluator,
+            ReasoningEfficiencyEvaluator
+        ]
+
+        assert len(agent_evaluator.evaluators) == len(expected_types)
+        for evaluator, expected_type in zip(agent_evaluator.evaluators, expected_types):
+            assert isinstance(evaluator, expected_type)

tests/llm_test.py

@@ -601,7 +601,7 @@ def test_handle_streaming_tool_calls(get_weather_tool_schema, mock_emit):
 def test_handle_streaming_tool_calls_with_error(get_weather_tool_schema, mock_emit):
     def get_weather_error(location):
         raise Exception("Error")
-        
+
     llm = LLM(model="openai/gpt-4o", stream=True)
     response = llm.call(
         messages=[
@@ -619,7 +619,7 @@ def test_handle_streaming_tool_calls_with_error(get_weather_tool_schema, mock_em
         expected_stream_chunk=9,
         expected_completed_llm_call=1,
         expected_tool_usage_started=1,
-        expected_tool_usage_error=1,    
+        expected_tool_usage_error=1,
         expected_final_chunk_result=expected_final_chunk_result,
     )