adding test results telemetry

This commit is contained in:
João Moura
2024-08-09 15:13:34 -07:00
parent 51ee483e9d
commit 5e83a36009
3 changed files with 64 additions and 0 deletions

View File

@@ -8,6 +8,7 @@ from rich.table import Table
from crewai.agent import Agent
from crewai.task import Task
from crewai.tasks.task_output import TaskOutput
from crewai.telemetry import Telemetry
class TaskEvaluationPydanticOutput(BaseModel):
@@ -34,6 +35,7 @@ class CrewEvaluator:
def __init__(self, crew, openai_model_name: str):
self.crew = crew
self.openai_model_name = openai_model_name
self._telemetry = Telemetry()
self._setup_for_evaluating()
def _setup_for_evaluating(self) -> None:
@@ -155,6 +157,12 @@ class CrewEvaluator:
evaluation_result = evaluation_task.execute_sync()
if isinstance(evaluation_result.pydantic, TaskEvaluationPydanticOutput):
self._test_result_span = self._telemetry.individual_test_result_span(
self,
evaluation_result.pydantic.quality,
current_task._execution_time,
self.openai_model_name,
)
self.tasks_scores[self.iteration].append(evaluation_result.pydantic.quality)
self.run_execution_times[self.iteration].append(
current_task._execution_time