feat: add regression tests and configure its workflow

* refactor: rely on task completion event to evaluate agents

* feat: remove Crew dependency to evaluate agent

* feat: drop execution_context in AgentEvaluator

* chore: drop experimental Agent Eval feature from stable crew.test

* feat: support eval LiteAgent

* resolve linter issues

.github/workflows/regression-tests.yml

@@ -0,0 +1,75 @@
+name: Regression Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to run tests on'
+        required: true
+        default: 'main'
+        type: string
+
+permissions:
+  contents: write
+
+env:
+  OPENAI_API_KEY: fake-api-key
+  PYTHONUNBUFFERED: 1
+
+jobs:
+  regression-tests:
+    name: Regression - ${{ github.event.inputs.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.branch }}
+          fetch-depth: 0
+
+      - name: Display execution info
+        run: |
+          echo "🚀 Running Regression Tests"
+          echo "📂 Branch: ${{ github.event.inputs.branch }}"
+          echo "📊 Current commit: $(git rev-parse --short HEAD)"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/pyproject.toml
+            **/uv.lock
+
+      - name: Set up Python 3.13
+        run: uv python install 3.13
+
+      - name: Install the project
+        run: uv sync --dev --all-extras
+
+      - name: Install SQLite with FTS5 support
+        run: |
+          # WORKAROUND: GitHub Actions' Ubuntu runner uses SQLite without FTS5 support compiled in.
+          # This is a temporary fix until the runner includes SQLite with FTS5 or Python's sqlite3
+          # module is compiled with FTS5 support by default.
+          # TODO: Remove this workaround once GitHub Actions runners include SQLite FTS5 support
+
+          # Install pysqlite3-binary which has FTS5 support
+          uv pip install pysqlite3-binary
+          # Create a sitecustomize.py to override sqlite3 with pysqlite3
+          mkdir -p .pytest_sqlite_override
+          echo "import sys; import pysqlite3; sys.modules['sqlite3'] = pysqlite3" > .pytest_sqlite_override/sitecustomize.py
+          # Test FTS5 availability
+          PYTHONPATH=.pytest_sqlite_override uv run python -c "import sqlite3; print(f'SQLite version: {sqlite3.sqlite_version}')"
+          PYTHONPATH=.pytest_sqlite_override uv run python -c "import sqlite3; conn = sqlite3.connect(':memory:'); conn.execute('CREATE VIRTUAL TABLE test USING fts5(content)'); print('FTS5 module available')"
+
+      - name: Run Regression Tests
+        run: |
+          PYTHONPATH=.pytest_sqlite_override uv run pytest \
+            --block-network \
+            --timeout=30 \
+            -vv \
+            --durations=10 \
+            -n auto \
+            --maxfail=5 \
+            tests/regression

docs/docs.json

@@ -9,12 +9,7 @@
   },
   "favicon": "/images/favicon.svg",
   "contextual": {
-    "options": [
-      "copy",
-      "view",
-      "chatgpt",
-      "claude"
-    ]
+    "options": ["copy", "view", "chatgpt", "claude"]
   },
   "navigation": {
     "languages": [
@@ -55,32 +50,22 @@
             "groups": [
               {
                 "group": "Get Started",
-                "pages": [
-                  "en/introduction",
-                  "en/installation",
-                  "en/quickstart"
-                ]
+                "pages": ["en/introduction", "en/installation", "en/quickstart"]
               },
               {
                 "group": "Guides",
                 "pages": [
                   {
                     "group": "Strategy",
-                    "pages": [
-                      "en/guides/concepts/evaluating-use-cases"
-                    ]
+                    "pages": ["en/guides/concepts/evaluating-use-cases"]
                   },
                   {
                     "group": "Agents",
-                    "pages": [
-                      "en/guides/agents/crafting-effective-agents"
-                    ]
+                    "pages": ["en/guides/agents/crafting-effective-agents"]
                   },
                   {
                     "group": "Crews",
-                    "pages": [
-                      "en/guides/crews/first-crew"
-                    ]
+                    "pages": ["en/guides/crews/first-crew"]
                   },
                   {
                     "group": "Flows",
@@ -94,7 +79,6 @@
                     "pages": [
                       "en/guides/advanced/customizing-prompts",
                       "en/guides/advanced/fingerprinting"
-
                     ]
                   }
                 ]
@@ -241,6 +225,7 @@
                   "en/observability/langtrace",
                   "en/observability/maxim",
                   "en/observability/mlflow",
+                  "en/observability/neatlogs",
                   "en/observability/openlit",
                   "en/observability/opik",
                   "en/observability/patronus-evaluation",
@@ -274,9 +259,7 @@
               },
               {
                 "group": "Telemetry",
-                "pages": [
-                  "en/telemetry"
-                ]
+                "pages": ["en/telemetry"]
               }
             ]
           },
@@ -285,9 +268,7 @@
             "groups": [
               {
                 "group": "Getting Started",
-                "pages": [
-                  "en/enterprise/introduction"
-                ]
+                "pages": ["en/enterprise/introduction"]
               },
               {
                 "group": "Features",
@@ -342,9 +323,7 @@
               },
               {
                 "group": "Resources",
-                "pages": [
-                  "en/enterprise/resources/frequently-asked-questions"
-                ]
+                "pages": ["en/enterprise/resources/frequently-asked-questions"]
               }
             ]
           },
@@ -353,9 +332,7 @@
             "groups": [
               {
                 "group": "Getting Started",
-                "pages": [
-                  "en/api-reference/introduction"
-                ]
+                "pages": ["en/api-reference/introduction"]
               },
               {
                 "group": "Endpoints",
@@ -365,16 +342,13 @@
           },
           {
             "tab": "Examples",
-                        "groups": [
+            "groups": [
               {
                 "group": "Examples",
-                "pages": [
-                  "en/examples/example"
-                ]
+                "pages": ["en/examples/example"]
               }
             ]
           }
-
         ]
       },
       {
@@ -425,21 +399,15 @@
                 "pages": [
                   {
                     "group": "Estratégia",
-                    "pages": [
-                      "pt-BR/guides/concepts/evaluating-use-cases"
-                    ]
+                    "pages": ["pt-BR/guides/concepts/evaluating-use-cases"]
                   },
                   {
                     "group": "Agentes",
-                    "pages": [
-                      "pt-BR/guides/agents/crafting-effective-agents"
-                    ]
+                    "pages": ["pt-BR/guides/agents/crafting-effective-agents"]
                   },
                   {
                     "group": "Crews",
-                    "pages": [
-                      "pt-BR/guides/crews/first-crew"
-                    ]
+                    "pages": ["pt-BR/guides/crews/first-crew"]
                   },
                   {
                     "group": "Flows",
@@ -632,9 +600,7 @@
               },
               {
                 "group": "Telemetria",
-                "pages": [
-                  "pt-BR/telemetry"
-                ]
+                "pages": ["pt-BR/telemetry"]
               }
             ]
           },
@@ -643,9 +609,7 @@
             "groups": [
               {
                 "group": "Começando",
-                "pages": [
-                  "pt-BR/enterprise/introduction"
-                ]
+                "pages": ["pt-BR/enterprise/introduction"]
               },
               {
                 "group": "Funcionalidades",
@@ -710,9 +674,7 @@
             "groups": [
               {
                 "group": "Começando",
-                "pages": [
-                  "pt-BR/api-reference/introduction"
-                ]
+                "pages": ["pt-BR/api-reference/introduction"]
               },
               {
                 "group": "Endpoints",
@@ -722,16 +684,13 @@
           },
           {
             "tab": "Exemplos",
-                        "groups": [
+            "groups": [
               {
                 "group": "Exemplos",
-                "pages": [
-                  "pt-BR/examples/example"
-                ]
+                "pages": ["pt-BR/examples/example"]
               }
             ]
           }
-
         ]
       }
     ]

docs/en/concepts/crews.mdx

@@ -32,6 +32,7 @@ A crew in crewAI represents a collaborative group of agents working together to
 | **Prompt File** _(optional)_          | `prompt_file`          | Path to the prompt JSON file to be used for the crew.                                                                                                                                                                                                     |
 | **Planning** *(optional)*             | `planning`             | Adds planning ability to the Crew. When activated before each Crew iteration, all Crew data is sent to an AgentPlanner that will plan the tasks and this plan will be added to each task description.                                                     |
 | **Planning LLM** *(optional)*         | `planning_llm`         | The language model used by the AgentPlanner in a planning process.                                                                                                                                                                                        |
+| **Knowledge Sources** _(optional)_    | `knowledge_sources`    | Knowledge sources available at the crew level, accessible to all the agents.                                                                                                                                                                                    |
 
 <Tip>
 **Crew Max RPM**: The `max_rpm` attribute sets the maximum number of requests per minute the crew can perform to avoid rate limits and will override individual agents' `max_rpm` settings if you set it.

docs/en/concepts/tasks.mdx

@@ -57,6 +57,7 @@ crew = Crew(
 | **Output JSON** _(optional)_     | `output_json`     | `Optional[Type[BaseModel]]`   | A Pydantic model to structure the JSON output.                                                                       |
 | **Output Pydantic** _(optional)_ | `output_pydantic` | `Optional[Type[BaseModel]]`   | A Pydantic model for task output.                                                                                    |
 | **Callback** _(optional)_        | `callback`        | `Optional[Any]`               | Function/object to be executed after task completion.                                                                |
+| **Guardrail** _(optional)_       | `guardrail`       | `Optional[Union[Callable, str]]` | Function or string description to validate task output before proceeding to next task.                            |
 
 ## Creating Tasks
 
@@ -86,6 +87,7 @@ research_task:
   expected_output: >
     A list with 10 bullet points of the most relevant information about {topic}
   agent: researcher
+  guardrail: ensure each bullet contains a minimum of 100 words
 
 reporting_task:
   description: >
@@ -332,9 +334,13 @@ Task guardrails provide a way to validate and transform task outputs before they
 are passed to the next task. This feature helps ensure data quality and provides
 feedback to agents when their output doesn't meet specific criteria.
 
-### Using Task Guardrails
+**Guardrails can be defined in two ways:**
+1. **Function-based guardrails**: Python functions that implement custom validation logic
+2. **String-based guardrails**: Natural language descriptions that are automatically converted to LLM-powered validation
 
-To add a guardrail to a task, provide a validation function through the `guardrail` parameter:
+### Function-Based Guardrails
+
+To add a function-based guardrail to a task, provide a validation function through the `guardrail` parameter:
 
 ```python Code
 from typing import Tuple, Union, Dict, Any
@@ -372,9 +378,82 @@ blog_task = Task(
    - On success: it returns a tuple of `(bool, Any)`. For example: `(True, validated_result)`
    - On Failure: it returns a tuple of `(bool, str)`. For example: `(False, "Error message explain the failure")`
 
-### LLMGuardrail
+### String-Based Guardrails
 
-The `LLMGuardrail` class offers a robust mechanism for validating task outputs.
+String-based guardrails allow you to describe validation criteria in natural language. When you provide a string instead of a function, CrewAI automatically converts it to an `LLMGuardrail` that uses an AI agent to validate the task output.
+
+#### Using String Guardrails in Python
+
+```python Code
+from crewai import Task
+
+# Simple string-based guardrail
+blog_task = Task(
+    description="Write a blog post about AI",
+    expected_output="A blog post under 200 words",
+    agent=blog_agent,
+    guardrail="Ensure the blog post is under 200 words and includes practical examples"
+)
+
+# More complex validation criteria
+research_task = Task(
+    description="Research AI trends for 2025",
+    expected_output="A comprehensive research report",
+    agent=research_agent,
+    guardrail="Ensure each finding includes a credible source and is backed by recent data from 2024-2025"
+)
+```
+
+#### Using String Guardrails in YAML
+
+```yaml
+research_task:
+  description: Research the latest AI developments
+  expected_output: A list of 10 bullet points about AI
+  agent: researcher
+  guardrail: ensure each bullet contains a minimum of 100 words
+
+validation_task:
+  description: Validate the research findings
+  expected_output: A validation report
+  agent: validator
+  guardrail: confirm all sources are from reputable publications and published within the last 2 years
+```
+
+#### How String Guardrails Work
+
+When you provide a string guardrail, CrewAI automatically:
+1. Creates an `LLMGuardrail` instance using the string as validation criteria
+2. Uses the task's agent LLM to power the validation
+3. Creates a temporary validation agent that checks the output against your criteria
+4. Returns detailed feedback if validation fails
+
+This approach is ideal when you want to use natural language to describe validation rules without writing custom validation functions.
+
+### LLMGuardrail Class
+
+The `LLMGuardrail` class is the underlying mechanism that powers string-based guardrails. You can also use it directly for more advanced control:
+
+```python Code
+from crewai import Task
+from crewai.tasks.llm_guardrail import LLMGuardrail
+from crewai.llm import LLM
+
+# Create a custom LLMGuardrail with specific LLM
+custom_guardrail = LLMGuardrail(
+    description="Ensure the response contains exactly 5 bullet points with proper citations",
+    llm=LLM(model="gpt-4o-mini")
+)
+
+task = Task(
+    description="Research AI safety measures",
+    expected_output="A detailed analysis with bullet points",
+    agent=research_agent,
+    guardrail=custom_guardrail
+)
+```
+
+**Note**: When you use a string guardrail, CrewAI automatically creates an `LLMGuardrail` instance using your task's agent LLM. Using `LLMGuardrail` directly gives you more control over the validation process and LLM selection.
 
 ### Error Handling Best Practices
 
@@ -798,166 +877,7 @@ While creating and executing tasks, certain validation mechanisms are in place t
 
 These validations help in maintaining the consistency and reliability of task executions within the crewAI framework.
 
-## Task Guardrails
 
-Task guardrails provide a powerful way to validate, transform, or filter task outputs before they are passed to the next task. Guardrails are optional functions that execute before the next task starts, allowing you to ensure that task outputs meet specific requirements or formats.
-
-### Basic Usage
-
-#### Define your own logic to validate
-
-```python Code
-from typing import Tuple, Union
-from crewai import Task
-
-def validate_json_output(result: str) -> Tuple[bool, Union[dict, str]]:
-    """Validate that the output is valid JSON."""
-    try:
-        json_data = json.loads(result)
-        return (True, json_data)
-    except json.JSONDecodeError:
-        return (False, "Output must be valid JSON")
-
-task = Task(
-    description="Generate JSON data",
-    expected_output="Valid JSON object",
-    guardrail=validate_json_output
-)
-```
-
-#### Leverage a no-code approach for validation
-
-```python Code
-from crewai import Task
-
-task = Task(
-    description="Generate JSON data",
-    expected_output="Valid JSON object",
-    guardrail="Ensure the response is a valid JSON object"
-)
-```
-
-#### Using YAML
-
-```yaml
-research_task:
-  ...
-  guardrail: make sure each bullet contains a minimum of 100 words
-  ...
-```
-
-```python Code
-@CrewBase
-class InternalCrew:
-    agents_config = "config/agents.yaml"
-    tasks_config = "config/tasks.yaml"
-
-    ...
-    @task
-    def research_task(self):
-        return Task(config=self.tasks_config["research_task"])  # type: ignore[index]
-    ...
-```
-
-
-#### Use custom models for code generation
-
-```python Code
-from crewai import Task
-from crewai.llm import LLM
-
-task = Task(
-    description="Generate JSON data",
-    expected_output="Valid JSON object",
-    guardrail=LLMGuardrail(
-        description="Ensure the response is a valid JSON object",
-        llm=LLM(model="gpt-4o-mini"),
-    )
-)
-```
-
-### How Guardrails Work
-
-1. **Optional Attribute**: Guardrails are an optional attribute at the task level, allowing you to add validation only where needed.
-2. **Execution Timing**: The guardrail function is executed before the next task starts, ensuring valid data flow between tasks.
-3. **Return Format**: Guardrails must return a tuple of `(success, data)`:
-   - If `success` is `True`, `data` is the validated/transformed result
-   - If `success` is `False`, `data` is the error message
-4. **Result Routing**:
-   - On success (`True`), the result is automatically passed to the next task
-   - On failure (`False`), the error is sent back to the agent to generate a new answer
-
-### Common Use Cases
-
-#### Data Format Validation
-```python Code
-def validate_email_format(result: str) -> Tuple[bool, Union[str, str]]:
-    """Ensure the output contains a valid email address."""
-    import re
-    email_pattern = r'^[\w\.-]+@[\w\.-]+\.\w+$'
-    if re.match(email_pattern, result.strip()):
-        return (True, result.strip())
-    return (False, "Output must be a valid email address")
-```
-
-#### Content Filtering
-```python Code
-def filter_sensitive_info(result: str) -> Tuple[bool, Union[str, str]]:
-    """Remove or validate sensitive information."""
-    sensitive_patterns = ['SSN:', 'password:', 'secret:']
-    for pattern in sensitive_patterns:
-        if pattern.lower() in result.lower():
-            return (False, f"Output contains sensitive information ({pattern})")
-    return (True, result)
-```
-
-#### Data Transformation
-```python Code
-def normalize_phone_number(result: str) -> Tuple[bool, Union[str, str]]:
-    """Ensure phone numbers are in a consistent format."""
-    import re
-    digits = re.sub(r'\D', '', result)
-    if len(digits) == 10:
-        formatted = f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
-        return (True, formatted)
-    return (False, "Output must be a 10-digit phone number")
-```
-
-### Advanced Features
-
-#### Chaining Multiple Validations
-```python Code
-def chain_validations(*validators):
-    """Chain multiple validators together."""
-    def combined_validator(result):
-        for validator in validators:
-            success, data = validator(result)
-            if not success:
-                return (False, data)
-            result = data
-        return (True, result)
-    return combined_validator
-
-# Usage
-task = Task(
-    description="Get user contact info",
-    expected_output="Email and phone",
-    guardrail=chain_validations(
-        validate_email_format,
-        filter_sensitive_info
-    )
-)
-```
-
-#### Custom Retry Logic
-```python Code
-task = Task(
-    description="Generate data",
-    expected_output="Valid data",
-    guardrail=validate_data,
-    max_retries=5  # Override default retry limit
-)
-```
 
 ## Creating Directories when Saving Files
 

docs/en/observability/neatlogs.mdx

@@ -0,0 +1,140 @@
+---
+title: Neatlogs Integration
+description: Understand, debug, and share your CrewAI agent runs
+icon: magnifying-glass-chart
+---
+
+# Introduction
+
+Neatlogs helps you **see what your agent did**, **why**, and **share it**.
+
+It captures every step: thoughts, tool calls, responses, evaluations. No raw logs. Just clear, structured traces. Great for debugging and collaboration.
+
+---
+
+## Why use Neatlogs?
+
+CrewAI agents use multiple tools and reasoning steps. When something goes wrong, you need context — not just errors.
+
+Neatlogs lets you:
+
+- Follow the full decision path
+- Add feedback directly on steps
+- Chat with the trace using AI assistant
+- Share runs publicly for feedback
+- Turn insights into tasks
+
+All in one place.
+
+Manage your traces effortlessly
+
+![Traces](/images/neatlogs-1.png)
+![Trace Response](/images/neatlogs-2.png)
+
+The best UX to view a CrewAI trace. Post comments anywhere you want. Use AI to debug.
+
+![Trace Details](/images/neatlogs-3.png)
+![Ai Chat Bot With A Trace](/images/neatlogs-4.png)
+![Comments Drawer](/images/neatlogs-5.png)
+
+---
+
+## Core Features
+
+- **Trace Viewer**: Track thoughts, tools, and decisions in sequence
+- **Inline Comments**: Tag teammates on any trace step
+- **Feedback & Evaluation**: Mark outputs as correct or incorrect
+- **Error Highlighting**: Automatic flagging of API/tool failures
+- **Task Conversion**: Convert comments into assigned tasks
+- **Ask the Trace (AI)**: Chat with your trace using Neatlogs AI bot
+- **Public Sharing**: Publish trace links to your community
+
+---
+
+## Quick Setup with CrewAI
+
+<Steps>
+  <Step title="Sign Up & Get API Key">
+    Visit [neatlogs.com](https://neatlogs.com/?utm_source=crewAI-docs), create a project, copy the API key.
+  </Step>
+  <Step title="Install SDK">
+    ```bash
+    pip install neatlogs
+    ```
+    (Latest version 0.8.0, Python 3.8+; MIT license) :contentReference[oaicite:1]{index=1}
+  </Step>
+  <Step title="Initialize Neatlogs">
+    Before starting Crew agents, add:
+
+    ```python
+    import neatlogs
+    neatlogs.init("YOUR_PROJECT_API_KEY")
+    ```
+
+    Agents run as usual. Neatlogs captures everything automatically.
+
+  </Step>
+</Steps>
+
+---
+
+## Under the Hood
+
+According to GitHub, Neatlogs:
+
+- Captures thoughts, tool calls, responses, errors, and token stats :contentReference[oaicite:2]{index=2}
+- Supports AI-powered task generation and robust evaluation workflows :contentReference[oaicite:3]{index=3}
+
+All with just two lines of code.
+
+---
+
+## Watch It Work
+
+### 🔍 Full Demo (4 min)
+
+<iframe
+  width="100%"
+  height="315"
+  src="https://www.youtube.com/embed/8KDme9T2I7Q?si=b8oHteaBwFNs_Duk"
+  title="YouTube video player"
+  frameBorder="0"
+  allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+  allowFullScreen
+></iframe>
+
+### ⚙️ CrewAI Integration (30 s)
+
+<iframe
+  className="w-full aspect-video rounded-xl"
+  src="https://www.loom.com/embed/9c78b552af43452bb3e4783cb8d91230?sid=e9d7d370-a91a-49b0-809e-2f375d9e801d"
+  title="Loom video player"
+  frameBorder="0"
+  allowFullScreen
+></iframe>
+
+---
+
+## Links & Support
+
+- 📘 [Neatlogs Docs](https://docs.neatlogs.com/)
+- 🔐 [Dashboard & API Key](https://app.neatlogs.com/)
+- 🐦 [Follow on Twitter](https://twitter.com/neatlogs)
+- 📧 Contact: hello@neatlogs.com
+- 🛠 [GitHub SDK](https://github.com/NeatLogs/neatlogs) :contentReference[oaicite:4]{index=4}
+
+---
+
+## TL;DR
+
+With just:
+
+```bash
+pip install neatlogs
+
+import neatlogs
+neatlogs.init("YOUR_API_KEY")
+
+You can now capture, understand, share, and act on your CrewAI agent runs in seconds.
+No setup overhead. Full trace transparency. Full team collaboration.
+```

docs/pt-BR/concepts/cli.mdx

@@ -76,6 +76,7 @@ Exemplo:
 crewai train -n 10 -f my_training_data.pkl
 ```
 
+```python
 # Exemplo de uso programático do comando train
 n_iterations = 2
 inputs = {"topic": "Treinamento CrewAI"}
@@ -89,6 +90,7 @@ try:
     )
 except Exception as e:
     raise Exception(f"Ocorreu um erro ao treinar a crew: {e}")
+```
 
 ### 4. Replay
 

docs/pt-BR/concepts/tasks.mdx

@@ -57,6 +57,7 @@ crew = Crew(
 | **Saída JSON** _(opcional)_      | `output_json`     | `Optional[Type[BaseModel]]`  | Um modelo Pydantic para estruturar a saída em JSON.                                                                |
 | **Output Pydantic** _(opcional)_ | `output_pydantic` | `Optional[Type[BaseModel]]`  | Um modelo Pydantic para a saída da tarefa.                                                                         |
 | **Callback** _(opcional)_        | `callback`        | `Optional[Any]`              | Função/objeto a ser executado após a conclusão da tarefa.                                                          |
+| **Guardrail** _(opcional)_       | `guardrail`       | `Optional[Union[Callable, str]]` | Função ou descrição em string para validar a saída da tarefa antes de prosseguir para a próxima tarefa.        |
 
 ## Criando Tarefas
 
@@ -86,6 +87,7 @@ research_task:
   expected_output: >
     Uma lista com 10 tópicos em bullet points das informações mais relevantes sobre {topic}
   agent: researcher
+  guardrail: garanta que cada bullet point contenha no mínimo 100 palavras
 
 reporting_task:
   description: >
@@ -330,9 +332,13 @@ analysis_task = Task(
 
 Guardrails (trilhas de proteção) de tarefas fornecem uma maneira de validar e transformar as saídas das tarefas antes que elas sejam passadas para a próxima tarefa. Esse recurso assegura a qualidade dos dados e oferece feedback aos agentes quando sua saída não atende a critérios específicos.
 
-### Usando Guardrails em Tarefas
+**Guardrails podem ser definidos de duas maneiras:**
+1. **Guardrails baseados em função**: Funções Python que implementam lógica de validação customizada
+2. **Guardrails baseados em string**: Descrições em linguagem natural que são automaticamente convertidas em validação baseada em LLM
 
-Para adicionar um guardrail a uma tarefa, forneça uma função de validação por meio do parâmetro `guardrail`:
+### Guardrails Baseados em Função
+
+Para adicionar um guardrail baseado em função a uma tarefa, forneça uma função de validação por meio do parâmetro `guardrail`:
 
 ```python Code
 from typing import Tuple, Union, Dict, Any
@@ -370,9 +376,82 @@ blog_task = Task(
    - Em caso de sucesso: retorna uma tupla `(True, resultado_validado)`
    - Em caso de falha: retorna uma tupla `(False, "mensagem de erro explicando a falha")`
 
-### LLMGuardrail
+### Guardrails Baseados em String
 
-A classe `LLMGuardrail` oferece um mecanismo robusto para validação das saídas das tarefas.
+Guardrails baseados em string permitem que você descreva critérios de validação em linguagem natural. Quando você fornece uma string em vez de uma função, o CrewAI automaticamente a converte em um `LLMGuardrail` que usa um agente de IA para validar a saída da tarefa.
+
+#### Usando Guardrails de String em Python
+
+```python Code
+from crewai import Task
+
+# Guardrail simples baseado em string
+blog_task = Task(
+    description="Escreva um post de blog sobre IA",
+    expected_output="Um post de blog com menos de 200 palavras",
+    agent=blog_agent,
+    guardrail="Garanta que o post do blog tenha menos de 200 palavras e inclua exemplos práticos"
+)
+
+# Critérios de validação mais complexos
+research_task = Task(
+    description="Pesquise tendências de IA para 2025",
+    expected_output="Um relatório abrangente de pesquisa",
+    agent=research_agent,
+    guardrail="Garanta que cada descoberta inclua uma fonte confiável e seja respaldada por dados recentes de 2024-2025"
+)
+```
+
+#### Usando Guardrails de String em YAML
+
+```yaml
+research_task:
+  description: Pesquise os últimos desenvolvimentos em IA
+  expected_output: Uma lista de 10 bullet points sobre IA
+  agent: researcher
+  guardrail: garanta que cada bullet point contenha no mínimo 100 palavras
+
+validation_task:
+  description: Valide os achados da pesquisa
+  expected_output: Um relatório de validação
+  agent: validator
+  guardrail: confirme que todas as fontes são de publicações respeitáveis e publicadas nos últimos 2 anos
+```
+
+#### Como Funcionam os Guardrails de String
+
+Quando você fornece um guardrail de string, o CrewAI automaticamente:
+1. Cria uma instância `LLMGuardrail` usando a string como critério de validação
+2. Usa o LLM do agente da tarefa para alimentar a validação
+3. Cria um agente temporário de validação que verifica a saída contra seus critérios
+4. Retorna feedback detalhado se a validação falhar
+
+Esta abordagem é ideal quando você quer usar linguagem natural para descrever regras de validação sem escrever funções de validação customizadas.
+
+### Classe LLMGuardrail
+
+A classe `LLMGuardrail` é o mecanismo subjacente que alimenta os guardrails baseados em string. Você também pode usá-la diretamente para maior controle avançado:
+
+```python Code
+from crewai import Task
+from crewai.tasks.llm_guardrail import LLMGuardrail
+from crewai.llm import LLM
+
+# Crie um LLMGuardrail customizado com LLM específico
+custom_guardrail = LLMGuardrail(
+    description="Garanta que a resposta contenha exatamente 5 bullet points com citações adequadas",
+    llm=LLM(model="gpt-4o-mini")
+)
+
+task = Task(
+    description="Pesquise medidas de segurança em IA",
+    expected_output="Uma análise detalhada com bullet points",
+    agent=research_agent,
+    guardrail=custom_guardrail
+)
+```
+
+**Nota**: Quando você usa um guardrail de string, o CrewAI automaticamente cria uma instância `LLMGuardrail` usando o LLM do agente da sua tarefa. Usar `LLMGuardrail` diretamente lhe dá mais controle sobre o processo de validação e seleção de LLM.
 
 ### Melhores Práticas de Tratamento de Erros
 

pyproject.toml

@@ -137,3 +137,6 @@ exclude = [
 "docs/**",
 "docs/",
 ]
+
+[tool.pytest.ini_options]
+norecursedirs = ["tests/regression"]

src/crewai/agent.py

@@ -210,7 +210,6 @@ class Agent(BaseAgent):
                         sources=self.knowledge_sources,
                         embedder=self.embedder,
                         collection_name=self.role,
-                        storage=self.knowledge_storage or None,
                     )
                     self.knowledge.add_sources()
         except (TypeError, ValueError) as e:
@@ -341,7 +340,8 @@ class Agent(BaseAgent):
             self.knowledge_config.model_dump() if self.knowledge_config else {}
         )
 
-        if self.knowledge:
+
+        if self.knowledge or (self.crew and self.crew.knowledge):
             crewai_event_bus.emit(
                 self,
                 event=KnowledgeRetrievalStartedEvent(
@@ -353,25 +353,28 @@ class Agent(BaseAgent):
                     task_prompt
                 )
                 if self.knowledge_search_query:
-                    agent_knowledge_snippets = self.knowledge.query(
-                        [self.knowledge_search_query], **knowledge_config
-                    )
-                    if agent_knowledge_snippets:
-                        self.agent_knowledge_context = extract_knowledge_context(
-                            agent_knowledge_snippets
-                        )
-                        if self.agent_knowledge_context:
-                            task_prompt += self.agent_knowledge_context
-                    if self.crew:
-                        knowledge_snippets = self.crew.query_knowledge(
+                    # Quering agent specific knowledge
+                    if self.knowledge:
+                        agent_knowledge_snippets = self.knowledge.query(
                             [self.knowledge_search_query], **knowledge_config
                         )
-                        if knowledge_snippets:
-                            self.crew_knowledge_context = extract_knowledge_context(
-                                knowledge_snippets
+                        if agent_knowledge_snippets:
+                            self.agent_knowledge_context = extract_knowledge_context(
+                                agent_knowledge_snippets
                             )
-                            if self.crew_knowledge_context:
-                                task_prompt += self.crew_knowledge_context
+                            if self.agent_knowledge_context:
+                                task_prompt += self.agent_knowledge_context
+
+                    # Quering crew specific knowledge
+                    knowledge_snippets = self.crew.query_knowledge(
+                        [self.knowledge_search_query], **knowledge_config
+                    )
+                    if knowledge_snippets:
+                        self.crew_knowledge_context = extract_knowledge_context(
+                            knowledge_snippets
+                        )
+                        if self.crew_knowledge_context:
+                            task_prompt += self.crew_knowledge_context
 
                     crewai_event_bus.emit(
                         self,

src/crewai/crew.py

@@ -1313,7 +1313,6 @@ class Crew(FlowTrackable, BaseModel):
         n_iterations: int,
         eval_llm: Union[str, InstanceOf[BaseLLM]],
         inputs: Optional[Dict[str, Any]] = None,
-        include_agent_eval: Optional[bool] = False
     ) -> None:
         """Test and evaluate the Crew with the given inputs for n iterations concurrently using concurrent.futures."""
         try:
@@ -1333,28 +1332,13 @@ class Crew(FlowTrackable, BaseModel):
             )
             test_crew = self.copy()
 
-            # TODO: Refator to use a single Evaluator Manage class
             evaluator = CrewEvaluator(test_crew, llm_instance)
 
-            if include_agent_eval:
-                from crewai.experimental.evaluation import create_default_evaluator
-                agent_evaluator = create_default_evaluator(crew=test_crew)
-
             for i in range(1, n_iterations + 1):
                 evaluator.set_iteration(i)
-
-                if include_agent_eval:
-                    agent_evaluator.set_iteration(i)
-
                 test_crew.kickoff(inputs=inputs)
 
-                # TODO: Refactor to use ListenerEvents instead of trigger each iteration manually
-                if include_agent_eval:
-                    agent_evaluator.evaluate_current_iteration()
-
             evaluator.print_crew_evaluation_result()
-            if include_agent_eval:
-                agent_evaluator.get_agent_evaluation(include_evaluation_feedback=True)
 
             crewai_event_bus.emit(
                 self,

src/crewai/experimental/evaluation/agent_evaluator.py

@@ -3,107 +3,140 @@ from crewai.agent import Agent
 from crewai.task import Task
 from crewai.experimental.evaluation.evaluation_display import EvaluationDisplayFormatter
 
-from typing import Any, Dict
-from collections import defaultdict
+from typing import Any
 from crewai.experimental.evaluation import BaseEvaluator, create_evaluation_callbacks
 from collections.abc import Sequence
-from crewai.crew import Crew
 from crewai.utilities.events.crewai_event_bus import crewai_event_bus
 from crewai.utilities.events.utils.console_formatter import ConsoleFormatter
-from crewai.experimental.evaluation.evaluation_display import AgentAggregatedEvaluationResult
+from crewai.utilities.events.task_events import TaskCompletedEvent
+from crewai.utilities.events.agent_events import LiteAgentExecutionCompletedEvent
+from crewai.experimental.evaluation.base_evaluator import AgentAggregatedEvaluationResult
+import threading
+
+class ExecutionState:
+    def __init__(self):
+        self.traces = {}
+        self.current_agent_id = None
+        self.current_task_id = None
+        self.iteration = 1
+        self.iterations_results = {}
+        self.agent_evaluators = {}
 
 class AgentEvaluator:
     def __init__(
         self,
+        agents: list[Agent],
         evaluators: Sequence[BaseEvaluator] | None = None,
-        crew: Crew | None = None,
     ):
-        self.crew: Crew | None = crew
+        self.agents: list[Agent] = agents
         self.evaluators: Sequence[BaseEvaluator] | None = evaluators
 
-        self.agent_evaluators: dict[str, Sequence[BaseEvaluator] | None] = {}
-        if crew is not None:
-            assert crew and crew.agents is not None
-            for agent in crew.agents:
-                self.agent_evaluators[str(agent.id)] = self.evaluators
-
         self.callback = create_evaluation_callbacks()
         self.console_formatter = ConsoleFormatter()
         self.display_formatter = EvaluationDisplayFormatter()
 
-        self.iteration = 1
-        self.iterations_results: dict[int, dict[str, list[AgentEvaluationResult]]] = {}
+        self._thread_local: threading.local = threading.local()
+
+        for agent in self.agents:
+            self._execution_state.agent_evaluators[str(agent.id)] = self.evaluators
+
+        self._subscribe_to_events()
+
+    @property
+    def _execution_state(self) -> ExecutionState:
+        if not hasattr(self._thread_local, 'execution_state'):
+            self._thread_local.execution_state = ExecutionState()
+        return self._thread_local.execution_state
+
+    def _subscribe_to_events(self) -> None:
+        crewai_event_bus.register_handler(TaskCompletedEvent, self._handle_task_completed)
+        crewai_event_bus.register_handler(LiteAgentExecutionCompletedEvent, self._handle_lite_agent_completed)
+
+    def _handle_task_completed(self, source: Any, event: TaskCompletedEvent) -> None:
+        assert event.task is not None
+        agent = event.task.agent
+        if agent and str(getattr(agent, 'id', 'unknown')) in self._execution_state.agent_evaluators:
+            state = ExecutionState()
+            state.current_agent_id = str(agent.id)
+            state.current_task_id = str(event.task.id)
+
+            trace = self.callback.get_trace(state.current_agent_id, state.current_task_id)
+
+            if not trace:
+                return
+
+            result = self.evaluate(
+                agent=agent,
+                task=event.task,
+                execution_trace=trace,
+                final_output=event.output,
+                state=state
+            )
+
+            current_iteration = self._execution_state.iteration
+            if current_iteration not in self._execution_state.iterations_results:
+                self._execution_state.iterations_results[current_iteration] = {}
+
+            if agent.role not in self._execution_state.iterations_results[current_iteration]:
+                self._execution_state.iterations_results[current_iteration][agent.role] = []
+
+            self._execution_state.iterations_results[current_iteration][agent.role].append(result)
+
+    def _handle_lite_agent_completed(self, source: object, event: LiteAgentExecutionCompletedEvent) -> None:
+        agent_info = event.agent_info
+        agent_id = str(agent_info["id"])
+
+        if agent_id in self._execution_state.agent_evaluators:
+            state = ExecutionState()
+            state.current_agent_id = agent_id
+            state.current_task_id = "lite_task"
+
+            target_agent = None
+            for agent in self.agents:
+                if str(agent.id) == agent_id:
+                    target_agent = agent
+                    break
+
+            if not target_agent:
+                return
+
+            trace = self.callback.get_trace(state.current_agent_id, state.current_task_id)
+
+            if not trace:
+                return
+
+            result = self.evaluate(
+                agent=target_agent,
+                execution_trace=trace,
+                final_output=event.output,
+                state=state
+            )
+
+            current_iteration = self._execution_state.iteration
+            if current_iteration not in self._execution_state.iterations_results:
+                self._execution_state.iterations_results[current_iteration] = {}
+
+            agent_role = target_agent.role
+            if agent_role not in self._execution_state.iterations_results[current_iteration]:
+                self._execution_state.iterations_results[current_iteration][agent_role] = []
+
+            self._execution_state.iterations_results[current_iteration][agent_role].append(result)
 
     def set_iteration(self, iteration: int) -> None:
-        self.iteration = iteration
+        self._execution_state.iteration = iteration
 
-    def reset_iterations_results(self):
-        self.iterations_results = {}
+    def reset_iterations_results(self) -> None:
+        self._execution_state.iterations_results = {}
 
-    def evaluate_current_iteration(self) -> dict[str, list[AgentEvaluationResult]]:
-        if not self.crew:
-            raise ValueError("Cannot evaluate: no crew was provided to the evaluator.")
+    def get_evaluation_results(self) -> dict[str, list[AgentEvaluationResult]]:
+        if self._execution_state.iterations_results and self._execution_state.iteration in self._execution_state.iterations_results:
+            return self._execution_state.iterations_results[self._execution_state.iteration]
+        return {}
 
-        if not self.callback:
-            raise ValueError("Cannot evaluate: no callback was set. Use set_callback() method first.")
+    def display_results_with_iterations(self) -> None:
+        self.display_formatter.display_summary_results(self._execution_state.iterations_results)
 
-        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-        evaluation_results: defaultdict[str, list[AgentEvaluationResult]] = defaultdict(list)
-
-        total_evals = 0
-        for agent in self.crew.agents:
-            for task in self.crew.tasks:
-                if task.agent and task.agent.id == agent.id and self.agent_evaluators.get(str(agent.id)):
-                    total_evals += 1
-
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[bold blue]{task.description}[/bold blue]"),
-            BarColumn(),
-            TextColumn("{task.percentage:.0f}% completed"),
-            console=self.console_formatter.console
-        ) as progress:
-            eval_task = progress.add_task(f"Evaluating agents (iteration {self.iteration})...", total=total_evals)
-
-            for agent in self.crew.agents:
-                evaluator = self.agent_evaluators.get(str(agent.id))
-                if not evaluator:
-                    continue
-
-                for task in self.crew.tasks:
-
-                    if task.agent and str(task.agent.id) != str(agent.id):
-                        continue
-
-                    trace = self.callback.get_trace(str(agent.id), str(task.id))
-                    if not trace:
-                        self.console_formatter.print(f"[yellow]Warning: No trace found for agent {agent.role} on task {task.description[:30]}...[/yellow]")
-                        progress.update(eval_task, advance=1)
-                        continue
-
-                    with crewai_event_bus.scoped_handlers():
-                        result = self.evaluate(
-                            agent=agent,
-                            task=task,
-                            execution_trace=trace,
-                            final_output=task.output
-                        )
-                        evaluation_results[agent.role].append(result)
-                        progress.update(eval_task, advance=1)
-
-        self.iterations_results[self.iteration] = evaluation_results
-        return evaluation_results
-
-    def get_evaluation_results(self):
-        if self.iteration in self.iterations_results:
-            return self.iterations_results[self.iteration]
-
-        return self.evaluate_current_iteration()
-
-    def display_results_with_iterations(self):
-        self.display_formatter.display_summary_results(self.iterations_results)
-
-    def get_agent_evaluation(self, strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE, include_evaluation_feedback: bool = False) -> Dict[str, AgentAggregatedEvaluationResult]:
+    def get_agent_evaluation(self, strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE, include_evaluation_feedback: bool = True) -> dict[str, AgentAggregatedEvaluationResult]:
         agent_results = {}
         with crewai_event_bus.scoped_handlers():
             task_results = self.get_evaluation_results()
@@ -123,7 +156,7 @@ class AgentEvaluator:
                 agent_results[agent_role] = aggregated_result
 
 
-            if self.iteration == max(self.iterations_results.keys()):
+            if self._execution_state.iterations_results and self._execution_state.iteration == max(self._execution_state.iterations_results.keys(), default=0):
                 self.display_results_with_iterations()
 
             if include_evaluation_feedback:
@@ -131,20 +164,22 @@ class AgentEvaluator:
 
         return agent_results
 
-    def display_evaluation_with_feedback(self):
-        self.display_formatter.display_evaluation_with_feedback(self.iterations_results)
+    def display_evaluation_with_feedback(self) -> None:
+        self.display_formatter.display_evaluation_with_feedback(self._execution_state.iterations_results)
 
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
-        execution_trace: Dict[str, Any],
-        final_output: Any
+        execution_trace: dict[str, Any],
+        final_output: Any,
+        state: ExecutionState,
+        task: Task | None = None,
     ) -> AgentEvaluationResult:
         result = AgentEvaluationResult(
-            agent_id=str(agent.id),
-            task_id=str(task.id)
+            agent_id=state.current_agent_id or str(agent.id),
+            task_id=state.current_task_id or (str(task.id) if task else "unknown_task")
         )
+
         assert self.evaluators is not None
         for evaluator in self.evaluators:
             try:
@@ -160,7 +195,7 @@ class AgentEvaluator:
 
         return result
 
-def create_default_evaluator(crew, llm=None):
+def create_default_evaluator(agents: list[Agent], llm: None = None):
     from crewai.experimental.evaluation import (
         GoalAlignmentEvaluator,
         SemanticQualityEvaluator,
@@ -179,4 +214,4 @@ def create_default_evaluator(crew, llm=None):
         ReasoningEfficiencyEvaluator(llm=llm),
     ]
 
-    return AgentEvaluator(evaluators=evaluators, crew=crew)
+    return AgentEvaluator(evaluators=evaluators, agents=agents)

src/crewai/experimental/evaluation/base_evaluator.py

@@ -57,9 +57,9 @@ class BaseEvaluator(abc.ABC):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: Any,
+        task: Task | None = None,
     ) -> EvaluationScore:
         pass
 

src/crewai/experimental/evaluation/evaluation_display.py

@@ -17,7 +17,6 @@ class EvaluationDisplayFormatter:
             self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")
             return
 
-        # Get all agent roles across all iterations
         all_agent_roles: set[str] = set()
         for iter_results in iterations_results.values():
             all_agent_roles.update(iter_results.keys())
@@ -25,7 +24,6 @@ class EvaluationDisplayFormatter:
         for agent_role in sorted(all_agent_roles):
             self.console_formatter.print(f"\n[bold cyan]Agent: {agent_role}[/bold cyan]")
 
-            # Process each iteration
             for iter_num, results in sorted(iterations_results.items()):
                 if agent_role not in results or not results[agent_role]:
                     continue
@@ -33,23 +31,19 @@ class EvaluationDisplayFormatter:
                 agent_results = results[agent_role]
                 agent_id = agent_results[0].agent_id
 
-                # Aggregate results for this agent in this iteration
                 aggregated_result = self._aggregate_agent_results(
                     agent_id=agent_id,
                     agent_role=agent_role,
                     results=agent_results,
                 )
 
-                # Display iteration header
                 self.console_formatter.print(f"\n[bold]Iteration {iter_num}[/bold]")
 
-                # Create table for this iteration
                 table = Table(box=ROUNDED)
                 table.add_column("Metric", style="cyan")
                 table.add_column("Score (1-10)", justify="center")
                 table.add_column("Feedback", style="green")
 
-                # Add metrics to table
                 if aggregated_result.metrics:
                     for metric, evaluation_score in aggregated_result.metrics.items():
                         score = evaluation_score.score
@@ -91,7 +85,6 @@ class EvaluationDisplayFormatter:
                         "Overall agent evaluation score"
                     )
 
-                # Print the table for this iteration
                 self.console_formatter.print(table)
 
     def display_summary_results(self, iterations_results: Dict[int, Dict[str, List[AgentAggregatedEvaluationResult]]]):
@@ -248,7 +241,6 @@ class EvaluationDisplayFormatter:
             feedback_summary = None
             if feedbacks:
                 if len(feedbacks) > 1:
-                    # Use the summarization method for multiple feedbacks
                     feedback_summary = self._summarize_feedbacks(
                         agent_role=agent_role,
                         metric=category.title(),
@@ -307,7 +299,7 @@ class EvaluationDisplayFormatter:
                 strategy_guidance = "Focus on the highest-scoring aspects and strengths demonstrated."
             elif strategy == AggregationStrategy.WORST_PERFORMANCE:
                 strategy_guidance = "Focus on areas that need improvement and common issues across tasks."
-            else:  # Default/average strategies
+            else:
                 strategy_guidance = "Provide a balanced analysis of strengths and weaknesses across all tasks."
 
             prompt = [

src/crewai/experimental/evaluation/evaluation_listener.py

@@ -9,7 +9,9 @@ from crewai.utilities.events.base_event_listener import BaseEventListener
 from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus
 from crewai.utilities.events.agent_events import (
     AgentExecutionStartedEvent,
-    AgentExecutionCompletedEvent
+    AgentExecutionCompletedEvent,
+    LiteAgentExecutionStartedEvent,
+    LiteAgentExecutionCompletedEvent
 )
 from crewai.utilities.events.tool_usage_events import (
     ToolUsageFinishedEvent,
@@ -52,10 +54,18 @@ class EvaluationTraceCallback(BaseEventListener):
         def on_agent_started(source, event: AgentExecutionStartedEvent):
             self.on_agent_start(event.agent, event.task)
 
+        @event_bus.on(LiteAgentExecutionStartedEvent)
+        def on_lite_agent_started(source, event: LiteAgentExecutionStartedEvent):
+            self.on_lite_agent_start(event.agent_info)
+
         @event_bus.on(AgentExecutionCompletedEvent)
         def on_agent_completed(source, event: AgentExecutionCompletedEvent):
             self.on_agent_finish(event.agent, event.task, event.output)
 
+        @event_bus.on(LiteAgentExecutionCompletedEvent)
+        def on_lite_agent_completed(source, event: LiteAgentExecutionCompletedEvent):
+            self.on_lite_agent_finish(event.output)
+
         @event_bus.on(ToolUsageFinishedEvent)
         def on_tool_completed(source, event: ToolUsageFinishedEvent):
             self.on_tool_use(event.tool_name, event.tool_args, event.output, success=True)
@@ -88,19 +98,38 @@ class EvaluationTraceCallback(BaseEventListener):
         def on_llm_call_completed(source, event: LLMCallCompletedEvent):
             self.on_llm_call_end(event.messages, event.response)
 
+    def on_lite_agent_start(self, agent_info: dict[str, Any]):
+        self.current_agent_id = agent_info['id']
+        self.current_task_id = "lite_task"
+
+        trace_key = f"{self.current_agent_id}_{self.current_task_id}"
+        self._init_trace(
+            trace_key=trace_key,
+            agent_id=self.current_agent_id,
+            task_id=self.current_task_id,
+            tool_uses=[],
+            llm_calls=[],
+            start_time=datetime.now(),
+            final_output=None
+        )
+
+    def _init_trace(self, trace_key: str, **kwargs: Any):
+        self.traces[trace_key] = kwargs
+
     def on_agent_start(self, agent: Agent, task: Task):
         self.current_agent_id = agent.id
         self.current_task_id = task.id
 
         trace_key = f"{agent.id}_{task.id}"
-        self.traces[trace_key] = {
-            "agent_id": agent.id,
-            "task_id": task.id,
-            "tool_uses": [],
-            "llm_calls": [],
-            "start_time": datetime.now(),
-            "final_output": None
-        }
+        self._init_trace(
+            trace_key=trace_key,
+            agent_id=agent.id,
+            task_id=task.id,
+            tool_uses=[],
+            llm_calls=[],
+            start_time=datetime.now(),
+            final_output=None
+        )
 
     def on_agent_finish(self, agent: Agent, task: Task, output: Any):
         trace_key = f"{agent.id}_{task.id}"
@@ -108,9 +137,20 @@ class EvaluationTraceCallback(BaseEventListener):
             self.traces[trace_key]["final_output"] = output
             self.traces[trace_key]["end_time"] = datetime.now()
 
+        self._reset_current()
+
+    def _reset_current(self):
         self.current_agent_id = None
         self.current_task_id = None
 
+    def on_lite_agent_finish(self, output: Any):
+        trace_key = f"{self.current_agent_id}_lite_task"
+        if trace_key in self.traces:
+            self.traces[trace_key]["final_output"] = output
+            self.traces[trace_key]["end_time"] = datetime.now()
+
+        self._reset_current()
+
     def on_tool_use(self, tool_name: str, tool_args: dict[str, Any] | str, result: Any,
                    success: bool = True, error_type: str | None = None):
         if not self.current_agent_id or not self.current_task_id:

src/crewai/experimental/evaluation/experiment/runner.py

@@ -2,7 +2,7 @@ from collections import defaultdict
 from hashlib import md5
 from typing import Any
 
-from crewai import Crew
+from crewai import Crew, Agent
 from crewai.experimental.evaluation import AgentEvaluator, create_default_evaluator
 from crewai.experimental.evaluation.experiment.result_display import ExperimentResultsDisplay
 from crewai.experimental.evaluation.experiment.result import ExperimentResults, ExperimentResult
@@ -14,14 +14,18 @@ class ExperimentRunner:
         self.evaluator: AgentEvaluator | None = None
         self.display = ExperimentResultsDisplay()
 
-    def run(self, crew: Crew, print_summary: bool = False) -> ExperimentResults:
-        self.evaluator = create_default_evaluator(crew=crew)
+    def run(self, crew: Crew | None = None, agents: list[Agent] | None = None, print_summary: bool = False) -> ExperimentResults:
+        if crew and not agents:
+            agents = crew.agents
+
+        assert agents is not None
+        self.evaluator = create_default_evaluator(agents=agents)
 
         results = []
 
         for test_case in self.dataset:
             self.evaluator.reset_iterations_results()
-            result = self._run_test_case(test_case, crew)
+            result = self._run_test_case(test_case=test_case, crew=crew, agents=agents)
             results.append(result)
 
         experiment_results = ExperimentResults(results)
@@ -31,7 +35,7 @@ class ExperimentRunner:
 
         return experiment_results
 
-    def _run_test_case(self, test_case: dict[str, Any], crew: Crew) -> ExperimentResult:
+    def _run_test_case(self, test_case: dict[str, Any], agents: list[Agent], crew: Crew | None = None) -> ExperimentResult:
         inputs = test_case["inputs"]
         expected_score = test_case["expected_score"]
         identifier = test_case.get("identifier") or md5(str(test_case).encode(), usedforsecurity=False).hexdigest()
@@ -39,7 +43,11 @@ class ExperimentRunner:
         try:
             self.display.console.print(f"[dim]Running crew with input: {str(inputs)[:50]}...[/dim]")
             self.display.console.print("\n")
-            crew.kickoff(inputs=inputs)
+            if crew:
+                crew.kickoff(inputs=inputs)
+            else:
+                for agent in agents:
+                    agent.kickoff(**inputs)
 
             assert self.evaluator is not None
             agent_evaluations = self.evaluator.get_agent_evaluation()

src/crewai/experimental/evaluation/metrics/goal_metrics.py

@@ -14,10 +14,14 @@ class GoalAlignmentEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: Any,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}\nExpected output: {task.expected_output}\n"
+
         prompt = [
             {"role": "system", "content": """You are an expert evaluator assessing how well an AI agent's output aligns with its assigned task goal.
 
@@ -37,8 +41,7 @@ Return your evaluation as JSON with fields 'score' (number) and 'feedback' (stri
             {"role": "user", "content": f"""
 Agent role: {agent.role}
 Agent goal: {agent.goal}
-Task description: {task.description}
-Expected output: {task.expected_output}
+{task_context}
 
 Agent's final output:
 {final_output}

src/crewai/experimental/evaluation/metrics/reasoning_metrics.py

@@ -36,10 +36,14 @@ class ReasoningEfficiencyEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
-        final_output: TaskOutput,
+        final_output: TaskOutput | str,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}\nExpected output: {task.expected_output}\n"
+
         llm_calls = execution_trace.get("llm_calls", [])
 
         if not llm_calls or len(llm_calls) < 2:
@@ -83,6 +87,8 @@ class ReasoningEfficiencyEvaluator(BaseEvaluator):
 
         call_samples = self._get_call_samples(llm_calls)
 
+        final_output = final_output.raw if isinstance(final_output, TaskOutput) else final_output
+
         prompt = [
             {"role": "system", "content": """You are an expert evaluator assessing the reasoning efficiency of an AI agent's thought process.
 
@@ -117,7 +123,7 @@ Return your evaluation as JSON with the following structure:
 }"""},
             {"role": "user", "content": f"""
 Agent role: {agent.role}
-Task description: {task.description}
+{task_context}
 
 Reasoning efficiency metrics:
 - Total LLM calls: {efficiency_metrics["total_llm_calls"]}
@@ -130,7 +136,7 @@ Sample of agent reasoning flow (chronological sequence):
 {call_samples}
 
 Agent's final output:
-{final_output.raw[:500]}... (truncated)
+{final_output[:500]}... (truncated)
 
 Evaluate the reasoning efficiency of this agent based on these interaction patterns.
 Identify any inefficient reasoning patterns and provide specific suggestions for optimization.

src/crewai/experimental/evaluation/metrics/semantic_quality_metrics.py

@@ -14,10 +14,13 @@ class SemanticQualityEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: Any,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}"
         prompt = [
             {"role": "system", "content": """You are an expert evaluator assessing the semantic quality of an AI agent's output.
 
@@ -37,7 +40,7 @@ Return your evaluation as JSON with fields 'score' (number) and 'feedback' (stri
 """},
             {"role": "user", "content": f"""
 Agent role: {agent.role}
-Task description: {task.description}
+{task_context}
 
 Agent's final output:
 {final_output}

src/crewai/experimental/evaluation/metrics/tools_metrics.py

@@ -16,10 +16,14 @@ class ToolSelectionEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: str,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}"
+
         tool_uses = execution_trace.get("tool_uses", [])
         tool_count = len(tool_uses)
         unique_tool_types = set([tool.get("tool", "Unknown tool") for tool in tool_uses])
@@ -72,7 +76,7 @@ Return your evaluation as JSON with these fields:
 """},
             {"role": "user", "content": f"""
 Agent role: {agent.role}
-Task description: {task.description}
+{task_context}
 
 Available tools for this agent:
 {available_tools_info}
@@ -128,10 +132,13 @@ class ParameterExtractionEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: str,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}"
         tool_uses = execution_trace.get("tool_uses", [])
         tool_count = len(tool_uses)
 
@@ -212,7 +219,7 @@ Return your evaluation as JSON with these fields:
 """},
             {"role": "user", "content": f"""
 Agent role: {agent.role}
-Task description: {task.description}
+{task_context}
 
 Parameter extraction examples:
 {param_samples_text}
@@ -267,10 +274,13 @@ class ToolInvocationEvaluator(BaseEvaluator):
     def evaluate(
         self,
         agent: Agent,
-        task: Task,
         execution_trace: Dict[str, Any],
         final_output: str,
+        task: Task | None = None,
     ) -> EvaluationScore:
+        task_context = ""
+        if task is not None:
+            task_context = f"Task description: {task.description}"
         tool_uses = execution_trace.get("tool_uses", [])
         tool_errors = []
         tool_count = len(tool_uses)
@@ -352,7 +362,7 @@ Return your evaluation as JSON with these fields:
 """},
             {"role": "user", "content": f"""
 Agent role: {agent.role}
-Task description: {task.description}
+{task_context}
 
 Tool invocation examples:
 {invocation_samples_text}

src/crewai/experimental/evaluation/testing.py

@@ -0,0 +1,71 @@
+import inspect
+from pathlib import Path
+
+from typing_extensions import Any
+import warnings
+from crewai.experimental.evaluation.experiment import ExperimentResults, ExperimentRunner
+from crewai import Crew, Agent
+
+def assert_experiment_successfully(experiment_results: ExperimentResults, baseline_filepath: str | None = None) -> None:
+    failed_tests = [result for result in experiment_results.results if not result.passed]
+
+    if failed_tests:
+        detailed_failures: list[str] = []
+
+        for result in failed_tests:
+            expected = result.expected_score
+            actual = result.score
+            detailed_failures.append(f"- {result.identifier}: expected {expected}, got {actual}")
+
+        failure_details = "\n".join(detailed_failures)
+        raise AssertionError(f"The following test cases failed:\n{failure_details}")
+
+    baseline_filepath = baseline_filepath or _get_baseline_filepath_fallback()
+    comparison = experiment_results.compare_with_baseline(baseline_filepath=baseline_filepath)
+    assert_experiment_no_regression(comparison)
+
+def assert_experiment_no_regression(comparison_result: dict[str, list[str]]) -> None:
+    regressed = comparison_result.get("regressed", [])
+    if regressed:
+        raise AssertionError(f"Regression detected! The following tests that previously passed now fail: {regressed}")
+
+    missing_tests = comparison_result.get("missing_tests", [])
+    if missing_tests:
+        warnings.warn(
+            f"Warning: {len(missing_tests)} tests from the baseline are missing in the current run: {missing_tests}",
+            UserWarning
+        )
+
+def run_experiment(dataset: list[dict[str, Any]], crew: Crew | None = None, agents: list[Agent] | None = None, verbose: bool = False) -> ExperimentResults:
+    runner = ExperimentRunner(dataset=dataset)
+
+    return runner.run(agents=agents, crew=crew, print_summary=verbose)
+
+def _get_baseline_filepath_fallback() -> str:
+    filename = "experiment_fallback.json"
+    calling_file = None
+
+    try:
+        current_frame = inspect.currentframe()
+        if current_frame is not None:
+            test_func_name = current_frame.f_back.f_back.f_code.co_name # type: ignore[union-attr]
+            filename = f"{test_func_name}.json"
+            calling_file = current_frame.f_back.f_back.f_code.co_filename # type: ignore[union-attr]
+    except Exception:
+        return filename
+
+    if not calling_file:
+        return filename
+
+    calling_path = Path(calling_file)
+    try:
+        baseline_dir_parts = calling_path.parts[:-1]
+        baseline_dir = Path(*baseline_dir_parts) / "results"
+        baseline_dir.mkdir(parents=True, exist_ok=True)
+        baseline_filepath = baseline_dir / filename
+        return str(baseline_filepath)
+
+    except (ValueError, IndexError):
+        pass
+
+    return filename

src/crewai/lite_agent.py

@@ -28,7 +28,7 @@ from pydantic import (
     InstanceOf,
     PrivateAttr,
     model_validator,
-    field_validator,
+    field_validator
 )
 
 from crewai.agents.agent_builder.base_agent import BaseAgent
@@ -40,7 +40,7 @@ from crewai.agents.parser import (
     OutputParserException,
 )
 from crewai.flow.flow_trackable import FlowTrackable
-from crewai.llm import LLM
+from crewai.llm import LLM, BaseLLM
 from crewai.tools.base_tool import BaseTool
 from crewai.tools.structured_tool import CrewStructuredTool
 from crewai.utilities import I18N
@@ -135,7 +135,7 @@ class LiteAgent(FlowTrackable, BaseModel):
     role: str = Field(description="Role of the agent")
     goal: str = Field(description="Goal of the agent")
     backstory: str = Field(description="Backstory of the agent")
-    llm: Optional[Union[str, InstanceOf[LLM], Any]] = Field(
+    llm: Optional[Union[str, InstanceOf[BaseLLM], Any]] = Field(
         default=None, description="Language model that will run the agent"
     )
     tools: List[BaseTool] = Field(
@@ -209,8 +209,8 @@ class LiteAgent(FlowTrackable, BaseModel):
     def setup_llm(self):
         """Set up the LLM and other components after initialization."""
         self.llm = create_llm(self.llm)
-        if not isinstance(self.llm, LLM):
-            raise ValueError("Unable to create LLM instance")
+        if not isinstance(self.llm, BaseLLM):
+            raise ValueError(f"Expected LLM instance of type BaseLLM, got {type(self.llm).__name__}")
 
         # Initialize callbacks
         token_callback = TokenCalcHandler(token_cost_process=self._token_process)
@@ -232,7 +232,8 @@ class LiteAgent(FlowTrackable, BaseModel):
         elif isinstance(self.guardrail, str):
             from crewai.tasks.llm_guardrail import LLMGuardrail
 
-            assert isinstance(self.llm, LLM)
+            if not isinstance(self.llm, BaseLLM):
+                raise TypeError(f"Guardrail requires LLM instance of type BaseLLM, got {type(self.llm).__name__}")
 
             self._guardrail = LLMGuardrail(description=self.guardrail, llm=self.llm)
 
@@ -304,6 +305,7 @@ class LiteAgent(FlowTrackable, BaseModel):
         """
         # Create agent info for event emission
         agent_info = {
+            "id": self.id,
             "role": self.role,
             "goal": self.goal,
             "backstory": self.backstory,
@@ -620,4 +622,4 @@ class LiteAgent(FlowTrackable, BaseModel):
 
     def _append_message(self, text: str, role: str = "assistant") -> None:
         """Append a message to the message list with the given role."""
-        self._messages.append(format_message_for_llm(text, role=role))
+        self._messages.append(format_message_for_llm(text, role=role))

src/crewai/task.py

@@ -67,6 +67,7 @@ class Task(BaseModel):
         description: Descriptive text detailing task's purpose and execution.
         expected_output: Clear definition of expected task outcome.
         output_file: File path for storing task output.
+        create_directory: Whether to create the directory for output_file if it doesn't exist.
         output_json: Pydantic model for structuring JSON output.
         output_pydantic: Pydantic model for task output.
         security_config: Security configuration including fingerprinting.
@@ -115,6 +116,10 @@ class Task(BaseModel):
         description="A file path to be used to create a file output.",
         default=None,
     )
+    create_directory: Optional[bool] = Field(
+        description="Whether to create the directory for output_file if it doesn't exist.",
+        default=True,
+    )
     output: Optional[TaskOutput] = Field(
         description="Task output, it's final result after being executed", default=None
     )
@@ -753,8 +758,10 @@ Follow these guidelines:
             resolved_path = Path(self.output_file).expanduser().resolve()
             directory = resolved_path.parent
 
-            if not directory.exists():
+            if self.create_directory and not directory.exists():
                 directory.mkdir(parents=True, exist_ok=True)
+            elif not self.create_directory and not directory.exists():
+                raise RuntimeError(f"Directory {directory} does not exist and create_directory is False")
 
             with resolved_path.open("w", encoding="utf-8") as file:
                 if isinstance(result, dict):

src/crewai/tasks/llm_guardrail.py

@@ -1,10 +1,9 @@
-from typing import Any, Optional, Tuple
+from typing import Any, Tuple
 
 from pydantic import BaseModel, Field
 
 from crewai.agent import Agent, LiteAgentOutput
-from crewai.llm import LLM
-from crewai.task import Task
+from crewai.llm import BaseLLM
 from crewai.tasks.task_output import TaskOutput
 
 
@@ -32,11 +31,11 @@ class LLMGuardrail:
     def __init__(
         self,
         description: str,
-        llm: LLM,
+        llm: BaseLLM,
     ):
         self.description = description
 
-        self.llm: LLM = llm
+        self.llm: BaseLLM = llm
 
     def _validate_output(self, task_output: TaskOutput) -> LiteAgentOutput:
         agent = Agent(

src/crewai/utilities/events/event_types.py

@@ -4,6 +4,7 @@ from .agent_events import (
     AgentExecutionCompletedEvent,
     AgentExecutionErrorEvent,
     AgentExecutionStartedEvent,
+    LiteAgentExecutionCompletedEvent,
 )
 from .crew_events import (
     CrewKickoffCompletedEvent,
@@ -80,6 +81,7 @@ EventTypes = Union[
     CrewTrainFailedEvent,
     AgentExecutionStartedEvent,
     AgentExecutionCompletedEvent,
+    LiteAgentExecutionCompletedEvent,
     TaskStartedEvent,
     TaskCompletedEvent,
     TaskFailedEvent,

tests/agent_test.py

@@ -1896,6 +1896,80 @@ def test_agent_with_knowledge_sources_generate_search_query():
         assert "red" in result.raw.lower()
 
 
+@pytest.mark.vcr(record_mode='none', filter_headers=["authorization"])
+def test_agent_with_knowledge_with_no_crewai_knowledge():
+    mock_knowledge = MagicMock(spec=Knowledge)
+
+    agent = Agent(
+        role="Information Agent",
+        goal="Provide information based on knowledge sources",
+        backstory="You have access to specific knowledge sources.",
+        llm=LLM(model="openrouter/openai/gpt-4o-mini",api_key=os.getenv('OPENROUTER_API_KEY')),
+        knowledge=mock_knowledge
+    )
+
+    # Create a task that requires the agent to use the knowledge
+    task = Task(
+        description="What is Vidit's favorite color?",
+        expected_output="Vidit's favorclearite color.",
+        agent=agent,
+    )
+
+    crew = Crew(agents=[agent], tasks=[task])
+    crew.kickoff()
+    mock_knowledge.query.assert_called_once()
+
+
+@pytest.mark.vcr(record_mode='none', filter_headers=["authorization"])
+def test_agent_with_only_crewai_knowledge():
+    mock_knowledge = MagicMock(spec=Knowledge)
+
+    agent = Agent(
+        role="Information Agent",
+        goal="Provide information based on knowledge sources",
+        backstory="You have access to specific knowledge sources.",
+        llm=LLM(model="openrouter/openai/gpt-4o-mini",api_key=os.getenv('OPENROUTER_API_KEY'))
+    )
+
+    # Create a task that requires the agent to use the knowledge
+    task = Task(
+        description="What is Vidit's favorite color?",
+        expected_output="Vidit's favorclearite color.",
+        agent=agent
+    )
+
+    crew = Crew(agents=[agent], tasks=[task],knowledge=mock_knowledge)
+    crew.kickoff()
+    mock_knowledge.query.assert_called_once()
+
+
+@pytest.mark.vcr(record_mode='none', filter_headers=["authorization"])
+def test_agent_knowledege_with_crewai_knowledge():
+    crew_knowledge = MagicMock(spec=Knowledge)
+    agent_knowledge = MagicMock(spec=Knowledge)
+
+
+    agent = Agent(
+        role="Information Agent",
+        goal="Provide information based on knowledge sources",
+        backstory="You have access to specific knowledge sources.",
+        llm=LLM(model="openrouter/openai/gpt-4o-mini",api_key=os.getenv('OPENROUTER_API_KEY')),
+        knowledge=agent_knowledge
+    )
+
+    # Create a task that requires the agent to use the knowledge
+    task = Task(
+        description="What is Vidit's favorite color?",
+        expected_output="Vidit's favorclearite color.",
+        agent=agent,
+    )
+
+    crew = Crew(agents=[agent],tasks=[task],knowledge=crew_knowledge)
+    crew.kickoff()
+    agent_knowledge.query.assert_called_once()
+    crew_knowledge.query.assert_called_once()
+
+
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_litellm_auth_error_handling():
     """Test that LiteLLM authentication errors are handled correctly and not retried."""

tests/cassettes/TestAgentEvaluator.test_eval_lite_agent.yaml

@@ -0,0 +1,237 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are Test Agent. An agent
+      created for testing purposes\nYour personal goal is: Complete test tasks successfully\n\nTo
+      give my best complete final answer to the task respond using the exact following
+      format:\n\nThought: I now can give a great answer\nFinal Answer: Your final
+      answer must be the great and the most complete as possible, it must be outcome
+      described.\n\nI MUST use these formats, my job depends on it!"}, {"role": "user",
+      "content": "Complete this task successfully"}], "model": "gpt-4o-mini", "stop":
+      ["\nObservation:"]}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '583'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.93.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.93.0
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.12
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFNNb9swDL3nVxA6J0U+HKTNbd0woMAOw7Bu6LbCUCXa1iqLgkgnzYr8
+        98FKWqdbB+wiQHx81OMj9TgCUM6qNSjTaDFt9JNL+TZ7N/dfrusPN01NyV6vPk3f/mrl5vLrXI17
+        Bt39RCNPrDNDbfQojsIBNgm1YF91tlrOl+fzxXKWgZYs+p5WR5kUNGldcJP5dF5MpqvJ7PzIbsgZ
+        ZLWG7yMAgMd89jqDxQe1hun4KdIis65RrZ+TAFQi30eUZnYsOogaD6ChIBiy9M8NdXUja7iCQFsw
+        OkDtNgga6l4/6MBbTAA/wnsXtIc3+b6Gjx41I8REG2cRWoStkwakQeCIxlXOgEXRzjNQgvzigwBV
+        OUU038OOOgiIFhr0MdPHoIOFK9g67wEDdwlBCI7OIjgB7oxB5qrzfpeznxRokIZS3wwk5EiB8ey0
+        54RVx7r3PXTenwA6BBLdzy27fXtE9s/+eqpjojv+g6oqFxw3ZULNFHovWSiqjO5HALd5jt2L0aiY
+        qI1SCt1jfu7i4lBODdszgEVxBIVE+yE+KxbjV8qVR79PFkEZbRq0A3XYGt1ZRyfA6KTpv9W8VvvQ
+        uAv1/5QfAGMwCtoyJrTOvOx4SEvYf65/pT2bnAUrxrRxBktxmPpBWKx05w8rr3jHgm1ZuVBjiskd
+        9r6K5aLQy0LjxcKo0X70GwAA//8DAMz2wVUFBAAA
+    headers:
+      CF-RAY:
+      - 95f93ea9af627e0b-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 15 Jul 2025 12:25:54 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=GRZmZLrjW5ZRHNmUJa4ccrMcy20D1rmeqK6Ptlv0mRY-1752582354-1.0.1.1-xKd_yga48Eedech5TRlThlEpDgsB2whxkWHlCyAGOVMqMcvH1Ju9FdXYbuQ9NdUQcVxPLgiGM35lYhqSLVQiXDyK01dnyp2Gvm560FBN9DY;
+        path=/; expires=Tue, 15-Jul-25 12:55:54 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=MYqswpSR7sqr4kGp6qZVkaL7HDYwMiww49PeN9QBP.A-1752582354973-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '4047'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '4440'
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999885'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_5704c0f206a927ddc12aa1a19b612a75
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are an expert evaluator
+      assessing how well an AI agent''s output aligns with its assigned task goal.\n\nScore
+      the agent''s goal alignment on a scale from 0-10 where:\n- 0: Complete misalignment,
+      agent did not understand or attempt the task goal\n- 5: Partial alignment, agent
+      attempted the task but missed key requirements\n- 10: Perfect alignment, agent
+      fully satisfied all task requirements\n\nConsider:\n1. Did the agent correctly
+      interpret the task goal?\n2. Did the final output directly address the requirements?\n3.
+      Did the agent focus on relevant aspects of the task?\n4. Did the agent provide
+      all requested information or deliverables?\n\nReturn your evaluation as JSON
+      with fields ''score'' (number) and ''feedback'' (string).\n"}, {"role": "user",
+      "content": "\nAgent role: Test Agent\nAgent goal: Complete test tasks successfully\n\n\nAgent''s
+      final output:\nPlease provide me with the specific details or context of the
+      task you need help with, and I will ensure to complete it successfully and provide
+      a thorough response.\n\nEvaluate how well the agent''s output aligns with the
+      assigned task goal.\n"}], "model": "gpt-4o-mini", "stop": []}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '1196'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=GRZmZLrjW5ZRHNmUJa4ccrMcy20D1rmeqK6Ptlv0mRY-1752582354-1.0.1.1-xKd_yga48Eedech5TRlThlEpDgsB2whxkWHlCyAGOVMqMcvH1Ju9FdXYbuQ9NdUQcVxPLgiGM35lYhqSLVQiXDyK01dnyp2Gvm560FBN9DY;
+        _cfuvid=MYqswpSR7sqr4kGp6qZVkaL7HDYwMiww49PeN9QBP.A-1752582354973-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.93.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.93.0
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.12
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA4xUy27bQAy8+yuIPdtGbMdN4FvbSxM0QIsEKNA6MJhdSmK82hWWVFwj8L8XKz/k
+        9AH0ogOHnOFjVq8DAMPOLMDYCtXWjR990O+TT7dfZs/v5OtFy/ef7++mxfu7j83t/cONGeaK+PRM
+        Vo9VYxvrxpNyDHvYJkKlzDq5mk/n19PZfN4BdXTkc1nZ6OgyjmoOPJpeTC9HF1ejyfWhuopsScwC
+        fgwAAF67b+4zOPppFnAxPEZqEsGSzOKUBGBS9DliUIRFMagZ9qCNQSl0rb8uA8DSiI2JlmYB0+E+
+        UBC5J7TrHFuah4oASwoKjh2EqOCojkE0oRIgWE+YoA2OUhZzHEqIBWhFoChrKCP6IWwqthWwgEY4
+        bItASbRLEpDWWhIpWu+3Y7gJooRuCKyAsiYHRUxQx0TgSJG9DIGDY4ua5RA82nVW5cDKqPxCWYhC
+        iSXBhrU69TOGbxV7ysxSxY0Awoa951AGkq69/do67QLZk8vBJsUXdgQYtoBWW/SQSJoYpFPq2Ptp
+        MLjTttC51DFXVIPjRFb9drw0y7A7v0uiohXM3git92cAhhAVs7c6RzwekN3JAz6WTYpP8lupKTiw
+        VKtEKDHke4vGxnTobgDw2HmtfWMf06RYN7rSuKZObjo7eM30Fu/R6yOoUdH38dnkCLzhWx1ud+ZW
+        Y9FW5PrS3trYOo5nwOBs6j+7+Rv3fnIO5f/Q94C11Ci5VZPIsX07cZ+WKP8B/pV22nLXsBFKL2xp
+        pUwpX8JRga3fv0sjW1GqVwWHklKTuHuc+ZKD3eAXAAAA//8DADksFsafBAAA
+    headers:
+      CF-RAY:
+      - 95f93ec73a1c7e0b-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 15 Jul 2025 12:25:57 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '1544'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '1546'
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999732'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_44930ba12ad8d1e3f0beed1d5e3d8b0c
+    status:
+      code: 200
+      message: OK
+version: 1

tests/cassettes/TestAgentEvaluator.test_evaluate_current_iteration.yaml

@@ -427,4 +427,140 @@ interactions:
     status:
       code: 200
       message: OK
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are an expert evaluator
+      assessing how well an AI agent''s output aligns with its assigned task goal.\n\nScore
+      the agent''s goal alignment on a scale from 0-10 where:\n- 0: Complete misalignment,
+      agent did not understand or attempt the task goal\n- 5: Partial alignment, agent
+      attempted the task but missed key requirements\n- 10: Perfect alignment, agent
+      fully satisfied all task requirements\n\nConsider:\n1. Did the agent correctly
+      interpret the task goal?\n2. Did the final output directly address the requirements?\n3.
+      Did the agent focus on relevant aspects of the task?\n4. Did the agent provide
+      all requested information or deliverables?\n\nReturn your evaluation as JSON
+      with fields ''score'' (number) and ''feedback'' (string).\n"}, {"role": "user",
+      "content": "\nAgent role: Test Agent\nAgent goal: Complete test tasks successfully\nTask
+      description: Test task description\nExpected output: Expected test output\n\nAgent''s
+      final output:\nThe expected test output is a comprehensive document that outlines
+      the specific parameters and criteria that define success for the task at hand.
+      It should include detailed descriptions of the tasks, the goals that need to
+      be achieved, and any specific formatting or structural requirements necessary
+      for the output. Each component of the task must be analyzed and addressed, providing
+      context as well as examples where applicable. Additionally, any tools or methodologies
+      that are relevant to executing the tasks successfully should be outlined, including
+      any potential risks or challenges that may arise during the process. This document
+      serves as a guiding framework to ensure that all aspects of the task are thoroughly
+      considered and executed to meet the high standards expected.\n\nEvaluate how
+      well the agent''s output aligns with the assigned task goal.\n"}], "model":
+      "gpt-4o-mini", "stop": []}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '1893'
+      content-type:
+      - application/json
+      cookie:
+      - _cfuvid=XwsgBfgvDGlKFQ4LiGYGIARIoSNTiwidqoo9UZcc.XY-1752087999227-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.93.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.93.0
+      x-stainless-raw-response:
+      - 'true'
+      x-stainless-read-timeout:
+      - '600.0'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.12
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFRNbxs5DL37VxA6jwPHddrUxxwWi2BRtEAPRevCYCSOh41GUkWOnTTI
+        fy8kf4zT5rCXOfCRT4+P5DxNAAw7swRjO1TbJz+90dvFxy//vX0za7dfr29+3eo/n75++Mh0O/za
+        maZUxLsfZPVYdWFjnzwpx7CHbSZUKqyX767mV/PL2eKqAn105EvZJul0Eac9B57OZ/PFdPZuenl9
+        qO4iWxKzhG8TAICn+i06g6MHs4RZc4z0JIIbMstTEoDJ0ZeIQREWxaCmGUEbg1Ko0p9WAWBlxMZM
+        K7OEq2YfaIncHdr7EluZzx0BbigopBy37MgBgiNF9uTAkdjMqbQOsYVdhwraEdBDIqvkIA6aBgXp
+        4uAdcLB+cNTArmPbAQfHFpUEJPYEQ3CUi2LHYVPoCpOi3EOmnwNn6imoXMC/cUdbyk3FWw7oj8+4
+        SAIhKkgiyy1b9P4RHHneUn4pTEn0WIYC6YDX5866aqDH+yKHFRJm5cqInjeB3AWM7vQsUgzhTFb9
+        48GtUlloSwMkZ4bEDMetOaSg1QH9XldVwSrk2wY4iBLWSs/hmG47zGiVMouylZP7WHkzdRSEtwQu
+        2qH4dhyBjcWKHWsXhzJTEgpVAwagByySirgzRSfLDrtzsTKr8Hy+VJnaQbAsdhi8PwMwhKhYfKzr
+        /P2APJ8W2MdNyvFO/ig1LQeWbp0JJYayrKIxmYo+TwC+10MZXuy+STn2Sdca76k+92ax2POZ8T5H
+        9P31AdSo6Mf4YjFvXuFb71dezk7NWLQdubF0vEscHMczYHLW9d9qXuPed85h83/oR8BaSkpunTI5
+        ti87HtMy/agTfT3t5HIVbITyli2tlSmXSThqcfD7n4qRR1Hq1y2HDeWUuf5ZyiQnz5PfAAAA//8D
+        AEfUP8BcBQAA
+    headers:
+      CF-RAY:
+      - 95f365f1bfc87ded-GRU
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Mon, 14 Jul 2025 19:24:07 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=PcC3_3T8.MK_WpZlQLdZfwpNv9Pe45AIYmrXOSgJ65E-1752521047-1.0.1.1-eyqwSWfQC7ZV6.JwTsTihK1ZWCrEmxd52CtNcfe.fw1UjjBN9rdTU4G7hRZiNqHQYo4sVZMmgRgqM9k7HRSzN2zln0bKmMiOuSQTZh6xF_I;
+        path=/; expires=Mon, 14-Jul-25 19:54:07 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=JvQ1c4qYZefNwOPoVNgAtX8ET7ObU.JKDvGc43LOR6g-1752521047741-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - crewai-iuxna1
+      openai-processing-ms:
+      - '2729'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '2789'
+      x-ratelimit-limit-requests:
+      - '30000'
+      x-ratelimit-limit-tokens:
+      - '150000000'
+      x-ratelimit-remaining-requests:
+      - '29999'
+      x-ratelimit-remaining-tokens:
+      - '149999559'
+      x-ratelimit-reset-requests:
+      - 2ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_74f6e8ff49db25dbea3d3525cc149e8e
+    status:
+      code: 200
+      message: OK
 version: 1

tests/cassettes/test_agent_knowledege_with_crewai_knowledge.yaml

@@ -0,0 +1,150 @@
+interactions:
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "Your goal is to rewrite the user query so that it is optimized for retrieval
+      from a vector database. Consider how the query will be used to find relevant
+      documents, and aim to make it more specific and context-aware. \n\n Do not include
+      any other text than the rewritten query, especially any preamble or postamble
+      and only add expected output format if its relevant to the rewritten query.
+      \n\n Focus on the key words of the intended task and to retrieve the most relevant
+      information. \n\n There will be some extra context provided that might need
+      to be removed such as expected_output formats structured_outputs and other instructions."},
+      {"role": "user", "content": "The original query is: What is Vidit''s favorite
+      color?\n\nThis is the expected criteria for your final answer: Vidit''s favorclearite
+      color.\nyou MUST return the actual complete content as the final answer, not
+      a summary.."}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1017'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA//90kE1vE0EMhv9K9V64TMrmgyadG8ceECAhhIrQarrj
+        3bidHY/GTgSK9r+jpUpaJLja78djn8ARHgPlxXK72a6X6+12szhq7Id72d2V8b58/nbzQb98gkOp
+        cuRIFR4fC+X3d3AYJVKChxTKgd8OxRYbWYycGQ7y8EidwaPbB7vuZCyJjCXDoasUjCL8S61Dtxfu
+        SOG/n5BkKFUeFD4fUnLoObPu20pBJcNDTQoccjA+UvufLedIP+Ebh5FUw0DwJ1RJBI+gymoh20wj
+        2SjPpF85sr3Rqz4cpbLRVSdJ6jUcKvUHDenM81zFeXgeTNMPB/2lRuMMM1Atlf8k9qVt1rer3WrV
+        3DZwOJw5SpWxWGvyRFnnR7ybQc4/usxvHEwspBfhbun+NreRLHDSObUL3Z7iRdxM/wh9rb/c8coy
+        Tb8BAAD//wMAqVt3JyMCAAA=
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402cb503aec46c0-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:56:14 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "You are Information Agent. You have access to specific knowledge sources.\nYour
+      personal goal is: Provide information based on knowledge sources\nTo give my
+      best complete final answer to the task respond using the exact following format:\n\nThought:
+      I now can give a great answer\nFinal Answer: Your final answer must be the great
+      and the most complete as possible, it must be outcome described.\n\nI MUST use
+      these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
+      Task: What is Vidit''s favorite color?\n\nThis is the expected criteria for
+      your final answer: Vidit''s favorclearite color.\nyou MUST return the actual
+      complete content as the final answer, not a summary.\n\nBegin! This is VERY
+      important to you, use the tools available and give your best Final Answer, your
+      job depends on it!\n\nThought:"}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '951'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA///iQjABAAAA//90kE9rG0EMxb/K8C69jNON7WJ7boFS
+        CD2ENm2g/1jGs/Ja7aw0zIydBuPvXjbBcQrtUU9P0u/pAO7g0JNMLhfzxexytli8mdy8r7c6/3Lb
+        v13eff00088fPj7AImXdc0cZDjeJ5OoaFoN2FOGgicTz6z7VyVwnAwvDQtc/KVQ4hK2vF0GHFKmy
+        CixCJl+pgzuftQhb5UAF7tsBUfuUdV3gZBejxYaFy7bN5IsKHErVBAvxlffU/qfL0tFvuMZioFJ8
+        T3AHZI0EB18Kl+qljjQqlWQkvTai9yZ4MT3vyXjTj6DGS7mnbMx3ecfio7l6rJ25447rq2I2fq+Z
+        K5mgUbPhYtZxRxewyLTZFR9PMZ4IWfon4Xj8YVEeSqVhzNBTTpkfQTapbWar6XI6bVYNLHYn/JR1
+        SLWt+oukjP9rRv7Ta8/6yqJq9fGsLFf27+m2o+o5lnFt8GFL3bO5Of5j60v/c5AXI8fjHwAAAP//
+        AwDEkP8dZgIAAA==
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402cb55c9fe46c0-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:56:15 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+version: 1

tests/cassettes/test_agent_with_knowledge_with_no_crewai_knowledge.yaml

@@ -0,0 +1,151 @@
+interactions:
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "Your goal is to rewrite the user query so that it is optimized for retrieval
+      from a vector database. Consider how the query will be used to find relevant
+      documents, and aim to make it more specific and context-aware. \n\n Do not include
+      any other text than the rewritten query, especially any preamble or postamble
+      and only add expected output format if its relevant to the rewritten query.
+      \n\n Focus on the key words of the intended task and to retrieve the most relevant
+      information. \n\n There will be some extra context provided that might need
+      to be removed such as expected_output formats structured_outputs and other instructions."},
+      {"role": "user", "content": "The original query is: What is Vidit''s favorite
+      color?\n\nThis is the expected criteria for your final answer: Vidit''s favorclearite
+      color.\nyou MUST return the actual complete content as the final answer, not
+      a summary.."}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1017'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA//90kE1vE0EMhv9K9V64TGCbNGQ7N46gIg6IXhBaTWed
+        Xbez49HYiaii/e9oqRKKBFf7/XjsE7iHx0B5db272W2uN++b3ep585k+jcmo/XqnYXvX5m/3cChV
+        jtxThceXQvnDRzhM0lOChxTKgd8NxVY3spo4Mxzk4ZGiwSOOwd5GmUoiY8lwiJWCUQ9/qW0d4igc
+        SeG/n5BkKFUeFD4fUnLYc2Ydu0pBJcNDTQoccjA+UvefLeeefsI3DhOphoHgT6iSCB5BldVCtoVG
+        slFeSO+5Z3ujV/twlMpGV1GSVDhU2h80pDPOSxPn4WUwzz8c9FmNpoVloFoq/w7cl67Z3K7b9bq5
+        beBwOGOUKlOxzuSJsi5/2C4c5xdd5lsHEwvpj7Bt3N/mricLnHRJjSGO1F/EzfyP0Nf6yx2vLPP8
+        CwAA//8DAOHu/cIiAgAA
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402c73df9d8859c-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:53:27 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "You are Information Agent. You have access to specific knowledge sources.\nYour
+      personal goal is: Provide information based on knowledge sources\nTo give my
+      best complete final answer to the task respond using the exact following format:\n\nThought:
+      I now can give a great answer\nFinal Answer: Your final answer must be the great
+      and the most complete as possible, it must be outcome described.\n\nI MUST use
+      these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
+      Task: What is Vidit''s favorite color?\n\nThis is the expected criteria for
+      your final answer: Vidit''s favorclearite color.\nyou MUST return the actual
+      complete content as the final answer, not a summary.\n\nBegin! This is VERY
+      important to you, use the tools available and give your best Final Answer, your
+      job depends on it!\n\nThought:"}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '951'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA///iQjABAAAA//90kUGPEzEMhf+K5QuXdJmlpbvkthIg
+        emFXQoIDoMpNPFNDJo6STLul6n9H09KyIDjmxc9+/rxH8Wix4zi5vpndTK+n8+Z2wo9vXj28fHff
+        vW4+PNT5j1l6/wkNpqwb8ZzR4n3ieLdAg716DmhRE0eS512qk5lOeomCBnX1jV1Fi25N9cppnwJX
+        0YgGXWaq7NH+HmvQrVUcF7Sf9xi0S1lXBW0cQjDYSpSyXmamohEtlqoJDUaqsuHlf34len5E2xjs
+        uRTqGO0eswZGi1SKlEqxjmk0Vo5j0gVE3YKjCJ1sGAi6MShQLFvOAF/iW4kU4O74tvBRvNRnBVra
+        aJbK4DRoBikQtcJWPIcdeHVDz7GyB4mQhlUQF3ZAG5JAq8BQdMiOi4GisBiHj+ZftIHA87hePeY5
+        5cjcUfYSO1hLgZLYSSvurxRXaDBzOxQKZ4gnPhK7k3A4fDVYdqVyPxLsOKcsRwxtWvoVOZo3vm3Q
+        4HCGl7L2qS6rfudYxus1I73zYS/69NZg1UrhorwYD/yHe+m5koQytnXk1uwvxc3hH12f1l8WeWI5
+        HH4CAAD//wMAhZKqO+QCAAA=
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402c7459f3f859c-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:53:28 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+version: 1

tests/cassettes/test_agent_with_only_crewai_knowledge.yaml

@@ -0,0 +1,150 @@
+interactions:
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "Your goal is to rewrite the user query so that it is optimized for retrieval
+      from a vector database. Consider how the query will be used to find relevant
+      documents, and aim to make it more specific and context-aware. \n\n Do not include
+      any other text than the rewritten query, especially any preamble or postamble
+      and only add expected output format if its relevant to the rewritten query.
+      \n\n Focus on the key words of the intended task and to retrieve the most relevant
+      information. \n\n There will be some extra context provided that might need
+      to be removed such as expected_output formats structured_outputs and other instructions."},
+      {"role": "user", "content": "The original query is: What is Vidit''s favorite
+      color?\n\nThis is the expected criteria for your final answer: Vidit''s favorclearite
+      color.\nyou MUST return the actual complete content as the final answer, not
+      a summary.."}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '1017'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA//90kE1PIzEMhv8Kei97Sdnplwq5gTgAF8ShcFitRmnG
+        nTFk4ihxq11V899Xs6gFJLja78djH8ANLFqKk+lqsZpP56vpYqJhublfP1eP65v1i79Lt9fdMwxS
+        lj03lGHxkChe3cGgl4YCLCRRdPyzTTpZyKTnyDCQzQt5hYXvnJ576VMgZYkw8JmcUgP7XmvgO2FP
+        BfbXAUHalGVTYOMuBIMtRy5dnckVibAoKgkG0Snvqf5my7GhP7CVQU+luJZgD8gSCBauFC7qoo40
+        EpXiSPrEDeuPcrZ1e8msdOYlSIZBpu2uuHDEeWvi2L4NhuG3QflblPqRpaWcMv8P3Ka6ml/OLmaz
+        6rKCwe6IkbL0SWuVV4pl/MNy5Di+6DRfGqioC+/Ci8p8NtcNqeNQxlTvfEfNSVwNX4R+1J/u+GAZ
+        hn8AAAD//wMAIwJ79CICAAA=
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402c9db99ec4722-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:55:14 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"model": "openai/gpt-4o-mini", "messages": [{"role": "system", "content":
+      "You are Information Agent. You have access to specific knowledge sources.\nYour
+      personal goal is: Provide information based on knowledge sources\nTo give my
+      best complete final answer to the task respond using the exact following format:\n\nThought:
+      I now can give a great answer\nFinal Answer: Your final answer must be the great
+      and the most complete as possible, it must be outcome described.\n\nI MUST use
+      these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
+      Task: What is Vidit''s favorite color?\n\nThis is the expected criteria for
+      your final answer: Vidit''s favorclearite color.\nyou MUST return the actual
+      complete content as the final answer, not a summary.\n\nBegin! This is VERY
+      important to you, use the tools available and give your best Final Answer, your
+      job depends on it!\n\nThought:"}], "stream": false, "stop": ["\nObservation:"]}'
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '951'
+      content-type:
+      - application/json
+      host:
+      - openrouter.ai
+      http-referer:
+      - https://litellm.ai
+      user-agent:
+      - litellm/1.68.0
+      x-title:
+      - liteLLM
+    method: POST
+    uri: https://openrouter.ai/api/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//4lKAAS4AAAAA///iQjABAAAA//90kN1qGzEQRl9FfNdyul4nday73ARy
+        VUpLE2jLIu+O15NoZ4QkOy1moa/R1+uTlE1wnEB7qU/zc84cwB0cepLZfHm+XMwXy/nF7II/3d7V
+        H+tOPvsS3le3d+keFjHpnjtKcPgQSa5uYDFoRwEOGkk8v+tjmZ3rbGBhWOj6ntoCh3bry1mrQwxU
+        WAUWbSJfqIM7rbVot8otZbivBwTtY9J1hpNdCBYbFs7bJpHPKnDIRSMsxBfeU/OfX5aOfsBVFgPl
+        7HuCOyBpIDj4nDkXL2WiUSkkE+mNEX00rRfT856MN/0EarzkR0rGfJNrFh/M1dPbmS/ccfnz63c2
+        G7/XxIVMq0GT4WzWYUdnsEi02WUfjiLPjCz9czCO3y3yz1xomCx6SjHxE8omNtViVV/WdbWqYLE7
+        CsSkQyxN0QeSPF2wmgyOxz3lK4uixYdTcrmyb7ubjornkKexrW+31L0UV+M/pr6ufxF51TKOfwEA
+        AP//AwBybekMaAIAAA==
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      CF-RAY:
+      - 9402c9e1b94a4722-BOM
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 15 May 2025 12:55:15 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      x-clerk-auth-message:
+      - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
+        token-carrier=header)
+      x-clerk-auth-reason:
+      - token-invalid
+      x-clerk-auth-status:
+      - signed-out
+    status:
+      code: 200
+      message: OK
+version: 1

tests/evaluation/test_agent_evaluator.py

@@ -1,95 +0,0 @@
-import pytest
-
-from crewai.agent import Agent
-from crewai.task import Task
-from crewai.crew import Crew
-from crewai.experimental.evaluation.agent_evaluator import AgentEvaluator
-from crewai.experimental.evaluation.base_evaluator import AgentEvaluationResult
-from crewai.experimental.evaluation import (
-    GoalAlignmentEvaluator,
-    SemanticQualityEvaluator,
-    ToolSelectionEvaluator,
-    ParameterExtractionEvaluator,
-    ToolInvocationEvaluator,
-    ReasoningEfficiencyEvaluator
-)
-
-from crewai.experimental.evaluation import create_default_evaluator
-class TestAgentEvaluator:
-    @pytest.fixture
-    def mock_crew(self):
-        agent = Agent(
-            role="Test Agent",
-            goal="Complete test tasks successfully",
-            backstory="An agent created for testing purposes",
-            allow_delegation=False,
-            verbose=False
-        )
-
-        task = Task(
-            description="Test task description",
-            agent=agent,
-            expected_output="Expected test output"
-        )
-
-        crew = Crew(
-            agents=[agent],
-            tasks=[task]
-        )
-        return crew
-
-    def test_set_iteration(self):
-        agent_evaluator = AgentEvaluator()
-
-        agent_evaluator.set_iteration(3)
-        assert agent_evaluator.iteration == 3
-
-    @pytest.mark.vcr(filter_headers=["authorization"])
-    def test_evaluate_current_iteration(self, mock_crew):
-        agent_evaluator = AgentEvaluator(crew=mock_crew, evaluators=[GoalAlignmentEvaluator()])
-
-        mock_crew.kickoff()
-
-        results = agent_evaluator.evaluate_current_iteration()
-
-        assert isinstance(results, dict)
-
-        agent, = mock_crew.agents
-        task, = mock_crew.tasks
-
-        assert len(mock_crew.agents) == 1
-        assert agent.role in results
-        assert len(results[agent.role]) == 1
-
-        result, = results[agent.role]
-        assert isinstance(result, AgentEvaluationResult)
-
-        assert result.agent_id == str(agent.id)
-        assert result.task_id == str(task.id)
-
-        goal_alignment, = result.metrics.values()
-        assert goal_alignment.score == 5.0
-
-        expected_feedback = "The agent's output demonstrates an understanding of the need for a comprehensive document"
-        assert expected_feedback in goal_alignment.feedback
-
-        assert goal_alignment.raw_response is not None
-        assert '"score": 5' in goal_alignment.raw_response
-
-    def test_create_default_evaluator(self, mock_crew):
-        agent_evaluator = create_default_evaluator(crew=mock_crew)
-        assert isinstance(agent_evaluator, AgentEvaluator)
-        assert agent_evaluator.crew == mock_crew
-
-        expected_types = [
-            GoalAlignmentEvaluator,
-            SemanticQualityEvaluator,
-            ToolSelectionEvaluator,
-            ParameterExtractionEvaluator,
-            ToolInvocationEvaluator,
-            ReasoningEfficiencyEvaluator
-        ]
-
-        assert len(agent_evaluator.evaluators) == len(expected_types)
-        for evaluator, expected_type in zip(agent_evaluator.evaluators, expected_types):
-            assert isinstance(evaluator, expected_type)

tests/experimental/evaluation/metrics/test_goal_metrics.py

@@ -1,5 +1,5 @@
 from unittest.mock import patch, MagicMock
-from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from tests.experimental.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
 
 from crewai.experimental.evaluation.base_evaluator import EvaluationScore
 from crewai.experimental.evaluation.metrics.goal_metrics import GoalAlignmentEvaluator

tests/experimental/evaluation/metrics/test_reasoning_metrics.py

@@ -6,7 +6,7 @@ from crewai.tasks.task_output import TaskOutput
 from crewai.experimental.evaluation.metrics.reasoning_metrics import (
     ReasoningEfficiencyEvaluator,
 )
-from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from tests.experimental.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
 from crewai.utilities.llm_utils import LLM
 from crewai.experimental.evaluation.base_evaluator import EvaluationScore
 

tests/experimental/evaluation/metrics/test_semantic_quality_metrics.py

@@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock
 
 from crewai.experimental.evaluation.base_evaluator import EvaluationScore
 from crewai.experimental.evaluation.metrics.semantic_quality_metrics import SemanticQualityEvaluator
-from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from tests.experimental.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
 from crewai.utilities.llm_utils import LLM
 
 class TestSemanticQualityEvaluator(BaseEvaluationMetricsTest):

tests/experimental/evaluation/metrics/test_tools_metrics.py

@@ -6,7 +6,7 @@ from crewai.experimental.evaluation.metrics.tools_metrics import (
     ToolInvocationEvaluator
 )
 from crewai.utilities.llm_utils import LLM
-from tests.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
+from tests.experimental.evaluation.metrics.base_evaluation_metrics_test import BaseEvaluationMetricsTest
 
 class TestToolSelectionEvaluator(BaseEvaluationMetricsTest):
     def test_no_tools_available(self, mock_task, mock_agent):

tests/experimental/evaluation/test_agent_evaluator.py

@@ -0,0 +1,164 @@
+import pytest
+
+from crewai.agent import Agent
+from crewai.task import Task
+from crewai.crew import Crew
+from crewai.experimental.evaluation.agent_evaluator import AgentEvaluator
+from crewai.experimental.evaluation.base_evaluator import AgentEvaluationResult
+from crewai.experimental.evaluation import (
+    GoalAlignmentEvaluator,
+    SemanticQualityEvaluator,
+    ToolSelectionEvaluator,
+    ParameterExtractionEvaluator,
+    ToolInvocationEvaluator,
+    ReasoningEfficiencyEvaluator
+)
+
+from crewai.experimental.evaluation import create_default_evaluator
+
+class TestAgentEvaluator:
+    @pytest.fixture
+    def mock_crew(self):
+        agent = Agent(
+            role="Test Agent",
+            goal="Complete test tasks successfully",
+            backstory="An agent created for testing purposes",
+            allow_delegation=False,
+            verbose=False
+        )
+
+        task = Task(
+            description="Test task description",
+            agent=agent,
+            expected_output="Expected test output"
+        )
+
+        crew = Crew(
+            agents=[agent],
+            tasks=[task]
+        )
+        return crew
+
+    def test_set_iteration(self):
+        agent_evaluator = AgentEvaluator(agents=[])
+
+        agent_evaluator.set_iteration(3)
+        assert agent_evaluator._execution_state.iteration == 3
+
+    @pytest.mark.vcr(filter_headers=["authorization"])
+    def test_evaluate_current_iteration(self, mock_crew):
+        agent_evaluator = AgentEvaluator(agents=mock_crew.agents, evaluators=[GoalAlignmentEvaluator()])
+
+        mock_crew.kickoff()
+
+        results = agent_evaluator.get_evaluation_results()
+
+        assert isinstance(results, dict)
+
+        agent, = mock_crew.agents
+        task, = mock_crew.tasks
+
+        assert len(mock_crew.agents) == 1
+        assert agent.role in results
+        assert len(results[agent.role]) == 1
+
+        result, = results[agent.role]
+        assert isinstance(result, AgentEvaluationResult)
+
+        assert result.agent_id == str(agent.id)
+        assert result.task_id == str(task.id)
+
+        goal_alignment, = result.metrics.values()
+        assert goal_alignment.score == 5.0
+
+        expected_feedback = "The agent's output demonstrates an understanding of the need for a comprehensive document outlining task"
+        assert expected_feedback in goal_alignment.feedback
+
+        assert goal_alignment.raw_response is not None
+        assert '"score": 5' in goal_alignment.raw_response
+
+    def test_create_default_evaluator(self, mock_crew):
+        agent_evaluator = create_default_evaluator(agents=mock_crew.agents)
+        assert isinstance(agent_evaluator, AgentEvaluator)
+        assert agent_evaluator.agents == mock_crew.agents
+
+        expected_types = [
+            GoalAlignmentEvaluator,
+            SemanticQualityEvaluator,
+            ToolSelectionEvaluator,
+            ParameterExtractionEvaluator,
+            ToolInvocationEvaluator,
+            ReasoningEfficiencyEvaluator
+        ]
+
+        assert len(agent_evaluator.evaluators) == len(expected_types)
+        for evaluator, expected_type in zip(agent_evaluator.evaluators, expected_types):
+            assert isinstance(evaluator, expected_type)
+
+    @pytest.mark.vcr(filter_headers=["authorization"])
+    def test_eval_lite_agent(self):
+        agent = Agent(
+            role="Test Agent",
+            goal="Complete test tasks successfully",
+            backstory="An agent created for testing purposes",
+        )
+        agent_evaluator = AgentEvaluator(agents=[agent], evaluators=[GoalAlignmentEvaluator()])
+
+        agent.kickoff(messages="Complete this task successfully")
+
+        results = agent_evaluator.get_evaluation_results()
+
+        assert isinstance(results, dict)
+
+        result, = results[agent.role]
+        assert isinstance(result, AgentEvaluationResult)
+
+        assert result.agent_id == str(agent.id)
+        assert result.task_id == "lite_task"
+
+        goal_alignment, = result.metrics.values()
+        assert goal_alignment.score == 2.0
+
+        expected_feedback = "The agent did not demonstrate a clear understanding of the task goal, which is to complete test tasks successfully"
+        assert expected_feedback in goal_alignment.feedback
+
+        assert goal_alignment.raw_response is not None
+        assert '"score": 2' in goal_alignment.raw_response
+
+    @pytest.mark.vcr(filter_headers=["authorization"])
+    def test_eval_specific_agents_from_crew(self, mock_crew):
+        agent = Agent(
+            role="Test Agent Eval",
+            goal="Complete test tasks successfully",
+            backstory="An agent created for testing purposes",
+        )
+        task = Task(
+            description="Test task description",
+            agent=agent,
+            expected_output="Expected test output"
+        )
+        mock_crew.agents.append(agent)
+        mock_crew.tasks.append(task)
+
+        agent_evaluator = AgentEvaluator(agents=[agent], evaluators=[GoalAlignmentEvaluator()])
+
+        mock_crew.kickoff()
+
+        results = agent_evaluator.get_evaluation_results()
+
+        assert isinstance(results, dict)
+        assert len(results.keys()) == 1
+        result, = results[agent.role]
+        assert isinstance(result, AgentEvaluationResult)
+
+        assert result.agent_id == str(agent.id)
+        assert result.task_id == str(task.id)
+
+        goal_alignment, = result.metrics.values()
+        assert goal_alignment.score == 5.0
+
+        expected_feedback = "The agent provided a thorough guide on how to conduct a test task but failed to produce specific expected output"
+        assert expected_feedback in goal_alignment.feedback
+
+        assert goal_alignment.raw_response is not None
+        assert '"score": 5' in goal_alignment.raw_response

tests/regression/results/test_financial_analysis_regression.json

@@ -0,0 +1,42 @@
+[
+  {
+    "timestamp": "2025-07-15T21:34:08.253410+00:00",
+    "metadata": {},
+    "results": [
+      {
+        "identifier": "72239c22b0cdde98ad5c588074ef6325",
+        "inputs": {
+          "company": "Apple Inc. (AAPL)"
+        },
+        "score": {
+          "goal_alignment": 10.0,
+          "semantic_quality": 9.0,
+          "tool_selection": 6.0,
+          "parameter_extraction": 5.0,
+          "tool_invocation": 10.0,
+          "reasoning_efficiency": 7.300000000000001
+        },
+        "expected_score": {
+          "goal_alignment": 8
+        },
+        "passed": true
+      },
+      {
+        "identifier": "test_2",
+        "inputs": {
+          "company": "Microsoft Corporation (MSFT)"
+        },
+        "score": {
+          "goal_alignment": 10.0,
+          "semantic_quality": 7.333333333333333,
+          "tool_selection": 6.25,
+          "parameter_extraction": 9.5,
+          "tool_invocation": 10.0,
+          "reasoning_efficiency": 6.0
+        },
+        "expected_score": 8,
+        "passed": true
+      }
+    ]
+  }
+]

tests/regression/results/test_history_teacher.json

@@ -0,0 +1,24 @@
+[
+  {
+    "timestamp": "2025-07-15T21:31:05.916161+00:00",
+    "metadata": {},
+    "results": [
+      {
+        "identifier": "df0ea31ac4a7fb4a908b8319ec7b3719",
+        "inputs": {
+          "messages": "How was the Battle of Waterloo?"
+        },
+        "score": {
+          "goal_alignment": 10.0,
+          "semantic_quality": 10.0,
+          "tool_selection": 10.0,
+          "parameter_extraction": 10.0,
+          "tool_invocation": 10.0,
+          "reasoning_efficiency": 5.5
+        },
+        "expected_score": 8,
+        "passed": true
+      }
+    ]
+  }
+]

tests/regression/test_financial_analysis.py

@@ -0,0 +1,144 @@
+import pytest
+from crewai import Agent, Crew, Process, Task
+from crewai_tools import SerperDevTool
+
+from crewai.experimental.evaluation.testing import (
+    assert_experiment_successfully,
+    run_experiment,
+)
+
+
+@pytest.fixture
+def financial_analysis_crew():
+    search_tool = SerperDevTool()
+
+    data_researcher = Agent(
+        role="Financial Data Researcher",
+        goal="Efficiently collect and structure key financial metrics using multiple search strategies. Using only the search tool.",
+        backstory=(
+            "You are a precision-focused financial analyst who uses multiple targeted searches "
+            "to cross-verify data and ensure comprehensive coverage. You leverage different "
+            "search approaches to gather financial information from various authoritative sources."
+        ),
+        tools=[search_tool],
+    )
+
+    financial_analyst = Agent(
+        role="Financial Analyst",
+        goal="Analyze financial data to assess company performance and outlook",
+        backstory=(
+            "You are a seasoned financial analyst with expertise in evaluating company "
+            "performance through quantitative analysis. You can interpret financial statements, "
+            "identify trends, and make reasoned assessments of a company's financial health."
+        ),
+        tools=[search_tool],
+    )
+
+    report_writer = Agent(
+        role="Financial Report Writer",
+        goal="Synthesize financial analysis into clear, actionable reports",
+        backstory=(
+            "You are an experienced financial writer who excels at turning complex financial "
+            "analyses into clear, concise, and impactful reports. You know how to highlight "
+            "key insights and present information in a way that's accessible to various audiences."
+        ),
+        tools=[],
+    )
+
+    research_task = Task(
+        description=(
+            "Research {company} financial data using multiple targeted search queries:\n\n"
+            "**Search Strategy - Execute these searches sequentially:**\n"
+            "1. '{company} quarterly earnings Q4 2024 Q1 2025 financial results'\n"
+            "2. '{company} financial metrics P/E ratio profit margin debt equity'\n"
+            "3. '{company} revenue growth year over year earnings growth rate'\n"
+            "4. '{company} recent financial news SEC filings analyst reports'\n"
+            "5. '{company} stock performance market cap valuation 2024 2025'\n\n"
+            "**Data Collection Guidelines:**\n"
+            "- Use multiple search queries to cross-verify financial figures\n"
+            "- Prioritize official sources (SEC filings, earnings calls, company reports)\n"
+            "- Compare data across different financial platforms for accuracy\n"
+            "- Present findings in the exact format specified in expected_output."
+        ),
+        expected_output=(
+            "Financial data summary in this structure:\n\n"
+            "## Company Financial Overview\n"
+            "**Data Sources Used:** [List 3-5 sources from multiple searches]\n\n"
+            "**Latest Quarter:** [Period]\n"
+            "- Revenue: $X (YoY: +/-X%) [Source verification]\n"
+            "- Net Income: $X (YoY: +/-X%) [Source verification]\n"
+            "- EPS: $X (YoY: +/-X%) [Source verification]\n\n"
+            "**Key Metrics:**\n"
+            "- P/E Ratio: X [Current vs Historical]\n"
+            "- Profit Margin: X% [Trend indicator]\n"
+            "- Debt-to-Equity: X [Industry comparison]\n\n"
+            "**Growth Analysis:**\n"
+            "- Revenue Growth: X% (3-year trend)\n"
+            "- Earnings Growth: X% (consistency check)\n\n"
+            "**Material Developments:** [1-2 key items with impact assessment]\n"
+            "**Data Confidence:** [High/Medium/Low based on source consistency]"
+        ),
+        agent=data_researcher,
+    )
+
+    analysis_task = Task(
+        description=(
+            "Analyze the collected financial data to assess the company's performance and outlook. "
+            "Include the following in your analysis:\n"
+            "1. Evaluation of financial health based on key metrics\n"
+            "2. Trend analysis showing growth or decline patterns\n"
+            "3. Comparison with industry benchmarks or competitors\n"
+            "4. Identification of strengths and potential areas of concern\n"
+            "5. Short-term financial outlook based on current trends"
+        ),
+        expected_output=(
+            "A detailed financial analysis that includes assessment of key metrics, trends, "
+            "comparative analysis, and a reasoned outlook for the company's financial future."
+        ),
+        agent=financial_analyst,
+        context=[research_task],
+    )
+
+    report_task = Task(
+        description=(
+            "Create a professional financial report based on the research and analysis. "
+            "The report should:\n"
+            "1. Begin with an executive summary highlighting key findings\n"
+            "2. Present the financial analysis in a clear, logical structure\n"
+            "3. Include visual representations of key data points (described textually)\n"
+            "4. Provide actionable insights for potential investors\n"
+            "5. Conclude with a clear investment recommendation (buy, hold, or sell)"
+        ),
+        expected_output=(
+            "A professional, comprehensive financial report with executive summary, "
+            "structured analysis, visual elements, actionable insights, and a clear recommendation."
+        ),
+        agent=report_writer,
+        context=[research_task, analysis_task],
+    )
+
+    crew = Crew(
+        agents=[data_researcher, financial_analyst, report_writer],
+        tasks=[research_task, analysis_task, report_task],
+        process=Process.sequential,
+    )
+
+    return crew
+
+
+def test_financial_analysis_regression(financial_analysis_crew):
+    dataset = [
+        {
+            "inputs": {"company": "Apple Inc. (AAPL)"},
+            "expected_score": {"goal_alignment": 8},
+        },
+        {
+            "identifier": "test_2",
+            "inputs": {"company": "Microsoft Corporation (MSFT)"},
+            "expected_score": 8,
+        },
+    ]
+
+    results = run_experiment(dataset=dataset, crew=financial_analysis_crew, verbose=True)
+
+    assert_experiment_successfully(results)

tests/regression/test_history_teacher.py

@@ -0,0 +1,33 @@
+import pytest
+from crewai import Agent
+from crewai_tools import SerperDevTool
+
+from crewai.experimental.evaluation.testing import (
+    assert_experiment_successfully,
+    run_experiment,
+)
+
+@pytest.fixture
+def history_teacher():
+    search_tool = SerperDevTool()
+    return Agent(
+        role="History Educator",
+        goal="Teach students about important historical events with clarity and context",
+        backstory=(
+            "As a renowned historian and educator, you have spent decades studying world history, "
+            "from ancient civilizations to modern events. You are passionate about making history "
+            "engaging and understandable for learners of all ages. Your mission is to educate, explain, "
+            "and spark curiosity about the past."
+        ),
+        tools=[search_tool],
+        verbose=True,
+    )
+def test_history_teacher(history_teacher):
+    dataset = [
+        {"inputs": {"messages": "How was the Battle of Waterloo?"}, "expected_score": 8}
+    ]
+    results = run_experiment(
+        dataset=dataset, agents=[history_teacher], verbose=True
+    )
+
+    assert_experiment_successfully(results)

tests/task_test.py

@@ -1133,6 +1133,119 @@ def test_output_file_validation():
         )
 
 
+def test_create_directory_true():
+    """Test that directories are created when create_directory=True."""
+    from pathlib import Path
+    
+    output_path = "test_create_dir/output.txt"
+    
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        output_file=output_path,
+        create_directory=True,
+    )
+    
+    resolved_path = Path(output_path).expanduser().resolve()
+    resolved_dir = resolved_path.parent
+    
+    if resolved_path.exists():
+        resolved_path.unlink()
+    if resolved_dir.exists():
+        import shutil
+        shutil.rmtree(resolved_dir)
+    
+    assert not resolved_dir.exists()
+    
+    task._save_file("test content")
+    
+    assert resolved_dir.exists()
+    assert resolved_path.exists()
+    
+    if resolved_path.exists():
+        resolved_path.unlink()
+    if resolved_dir.exists():
+        import shutil
+        shutil.rmtree(resolved_dir)
+
+
+def test_create_directory_false():
+    """Test that directories are not created when create_directory=False."""
+    from pathlib import Path
+    
+    output_path = "nonexistent_test_dir/output.txt"
+    
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        output_file=output_path,
+        create_directory=False,
+    )
+    
+    resolved_path = Path(output_path).expanduser().resolve()
+    resolved_dir = resolved_path.parent
+    
+    if resolved_dir.exists():
+        import shutil
+        shutil.rmtree(resolved_dir)
+    
+    assert not resolved_dir.exists()
+    
+    with pytest.raises(RuntimeError, match="Directory .* does not exist and create_directory is False"):
+        task._save_file("test content")
+
+
+def test_create_directory_default():
+    """Test that create_directory defaults to True for backward compatibility."""
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        output_file="output.txt",
+    )
+    
+    assert task.create_directory is True
+
+
+def test_create_directory_with_existing_directory():
+    """Test that create_directory=False works when directory already exists."""
+    from pathlib import Path
+    
+    output_path = "existing_test_dir/output.txt"
+    
+    resolved_path = Path(output_path).expanduser().resolve()
+    resolved_dir = resolved_path.parent
+    resolved_dir.mkdir(parents=True, exist_ok=True)
+    
+    task = Task(
+        description="Test task",
+        expected_output="Test output",
+        output_file=output_path,
+        create_directory=False,
+    )
+    
+    task._save_file("test content")
+    assert resolved_path.exists()
+    
+    if resolved_path.exists():
+        resolved_path.unlink()
+    if resolved_dir.exists():
+        import shutil
+        shutil.rmtree(resolved_dir)
+
+
+def test_github_issue_3149_reproduction():
+    """Test that reproduces the exact issue from GitHub issue #3149."""
+    task = Task(
+        description="Test task for issue reproduction",
+        expected_output="Test output",
+        output_file="test_output.txt",
+        create_directory=True,
+    )
+    
+    assert task.create_directory is True
+    assert task.output_file == "test_output.txt"
+
+
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_task_execution_times():
     researcher = Agent(

tests/test_lite_agent.py

@@ -12,6 +12,8 @@ from crewai.tools import BaseTool
 from crewai.utilities.events import crewai_event_bus
 from crewai.utilities.events.agent_events import LiteAgentExecutionStartedEvent
 from crewai.utilities.events.tool_usage_events import ToolUsageStartedEvent
+from crewai.llms.base_llm import BaseLLM
+from unittest.mock import patch
 
 
 # A simple test tool
@@ -418,3 +420,76 @@ def test_agent_output_when_guardrail_returns_base_model():
     result = agent.kickoff(messages="Top 10 best players in the world?")
 
     assert result.pydantic == Player(name="Lionel Messi", country="Argentina")
+
+def test_lite_agent_with_custom_llm_and_guardrails():
+    """Test that CustomLLM (inheriting from BaseLLM) works with guardrails."""
+    class CustomLLM(BaseLLM):
+        def __init__(self, response: str = "Custom response"):
+            super().__init__(model="custom-model")
+            self.response = response
+            self.call_count = 0
+
+        def call(self, messages, tools=None, callbacks=None, available_functions=None, from_task=None, from_agent=None) -> str:
+            self.call_count += 1
+
+            if "valid" in str(messages) and "feedback" in str(messages):
+                return '{"valid": true, "feedback": null}'
+
+            if "Thought:" in str(messages):
+                return f"Thought: I will analyze soccer players\nFinal Answer: {self.response}"
+
+            return self.response
+
+        def supports_function_calling(self) -> bool:
+            return False
+
+        def supports_stop_words(self) -> bool:
+            return False
+
+        def get_context_window_size(self) -> int:
+            return 4096
+
+    custom_llm = CustomLLM(response="Brazilian soccer players are the best!")
+
+    agent = LiteAgent(
+        role="Sports Analyst",
+        goal="Analyze soccer players",
+        backstory="You analyze soccer players and their performance.",
+        llm=custom_llm,
+        guardrail="Only include Brazilian players"
+    )
+
+    result = agent.kickoff("Tell me about the best soccer players")
+
+    assert custom_llm.call_count > 0
+    assert "Brazilian" in result.raw
+
+    custom_llm2 = CustomLLM(response="Original response")
+
+    def test_guardrail(output):
+        return (True, "Modified by guardrail")
+
+    agent2 = LiteAgent(
+        role="Test Agent",
+        goal="Test goal",
+        backstory="Test backstory",
+        llm=custom_llm2,
+        guardrail=test_guardrail
+    )
+
+    result2 = agent2.kickoff("Test message")
+    assert result2.raw == "Modified by guardrail"
+
+
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_lite_agent_with_invalid_llm():
+    """Test that LiteAgent raises proper error when create_llm returns None."""
+    with patch('crewai.lite_agent.create_llm', return_value=None):
+        with pytest.raises(ValueError) as exc_info:
+            LiteAgent(
+                role="Test Agent",
+                goal="Test goal", 
+                backstory="Test backstory",
+                llm="invalid-model"
+            )
+        assert "Expected LLM instance of type BaseLLM" in str(exc_info.value)