mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-28 01:28:14 +00:00
Compare commits
6 Commits
gl/feat/de
...
devin/1768
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b568bcee87 | ||
|
|
7fb8999545 | ||
|
|
d97d58d2c3 | ||
|
|
9edbf89b68 | ||
|
|
685f7b9af1 | ||
|
|
595fdfb6e7 |
@@ -291,6 +291,7 @@
|
||||
"en/observability/arize-phoenix",
|
||||
"en/observability/braintrust",
|
||||
"en/observability/datadog",
|
||||
"en/observability/galileo",
|
||||
"en/observability/langdb",
|
||||
"en/observability/langfuse",
|
||||
"en/observability/langtrace",
|
||||
@@ -742,6 +743,7 @@
|
||||
"pt-BR/observability/arize-phoenix",
|
||||
"pt-BR/observability/braintrust",
|
||||
"pt-BR/observability/datadog",
|
||||
"pt-BR/observability/galileo",
|
||||
"pt-BR/observability/langdb",
|
||||
"pt-BR/observability/langfuse",
|
||||
"pt-BR/observability/langtrace",
|
||||
@@ -1203,6 +1205,7 @@
|
||||
"ko/observability/arize-phoenix",
|
||||
"ko/observability/braintrust",
|
||||
"ko/observability/datadog",
|
||||
"ko/observability/galileo",
|
||||
"ko/observability/langdb",
|
||||
"ko/observability/langfuse",
|
||||
"ko/observability/langtrace",
|
||||
|
||||
115
docs/en/observability/galileo.mdx
Normal file
115
docs/en/observability/galileo.mdx
Normal file
@@ -0,0 +1,115 @@
|
||||
---
|
||||
title: Galileo
|
||||
description: Galileo integration for CrewAI tracing and evaluation
|
||||
icon: telescope
|
||||
mode: "wide"
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This guide demonstrates how to integrate **Galileo** with **CrewAI**
|
||||
for comprehensive tracing and Evaluation Engineering.
|
||||
By the end of this guide, you will be able to trace your CrewAI agents,
|
||||
monitor their performance, and evaluate their behaviour with
|
||||
Galileo's powerful observability platform.
|
||||
|
||||
> **What is Galileo?** [Galileo](https://galileo.ai) is AI evaluation and observability
|
||||
platform that delivers end-to-end tracing, evaluation,
|
||||
and monitoring for AI applications. It enables teams to capture ground truth,
|
||||
create robust guardrails, and run systematic experiments with
|
||||
built-in experiment tracking and performance analytics—ensuring reliability,
|
||||
transparency, and continuous improvement across the AI lifecycle.
|
||||
|
||||
## Getting started
|
||||
|
||||
This tutorial follows the [CrewAI quickstart](/en/quickstart) and shows how to add
|
||||
Galileo's [CrewAIEventListener](https://v2docs.galileo.ai/sdk-api/python/reference/handlers/crewai/handler),
|
||||
an event handler.
|
||||
For more information, see Galileo’s
|
||||
[Add Galileo to a CrewAI Application](https://v2docs.galileo.ai/how-to-guides/third-party-integrations/add-galileo-to-crewai/add-galileo-to-crewai)
|
||||
how-to guide.
|
||||
|
||||
> **Note** This tutorial assumes you have completed the [CrewAI quickstart](/en/quickstart).
|
||||
If you want a completed comprehensive example, see the Galileo
|
||||
[CrewAI sdk-example repo](https://github.com/rungalileo/sdk-examples/tree/main/python/agent/crew-ai).
|
||||
|
||||
### Step 1: Install dependencies
|
||||
|
||||
Install the required dependencies for your app.
|
||||
Create a virtual environment using your preferred method,
|
||||
then install dependencies inside that environment using your
|
||||
preferred tool:
|
||||
|
||||
```bash
|
||||
uv add galileo
|
||||
```
|
||||
|
||||
### Step 2: Add to the .env file from the [CrewAI quickstart](/en/quickstart)
|
||||
|
||||
```bash
|
||||
# Your Galileo API key
|
||||
GALILEO_API_KEY="your-galileo-api-key"
|
||||
|
||||
# Your Galileo project name
|
||||
GALILEO_PROJECT="your-galileo-project-name"
|
||||
|
||||
# The name of the Log stream you want to use for logging
|
||||
GALILEO_LOG_STREAM="your-galileo-log-stream "
|
||||
```
|
||||
|
||||
### Step 3: Add the Galileo event listener
|
||||
|
||||
To enable logging with Galileo, you need to create an instance of the `CrewAIEventListener`.
|
||||
Import the Galileo CrewAI handler package by
|
||||
adding the following code at the top of your main.py file:
|
||||
|
||||
```python
|
||||
from galileo.handlers.crewai.handler import CrewAIEventListener
|
||||
```
|
||||
|
||||
At the start of your run function, create the event listener:
|
||||
|
||||
```python
|
||||
def run():
|
||||
# Create the event listener
|
||||
CrewAIEventListener()
|
||||
# The rest of your existing code goes here
|
||||
```
|
||||
|
||||
When you create the listener instance, it is automatically
|
||||
registered with CrewAI.
|
||||
|
||||
### Step 4: Run your crew
|
||||
|
||||
Run your crew with the CrewAI CLI:
|
||||
|
||||
```bash
|
||||
crewai run
|
||||
```
|
||||
|
||||
### Step 5: View the traces in Galileo
|
||||
|
||||
Once your crew has finished, the traces will be flushed and appear in Galileo.
|
||||
|
||||

|
||||
|
||||
## Understanding the Galileo Integration
|
||||
|
||||
Galileo integrates with CrewAI by registering an event listener
|
||||
that captures Crew execution events (e.g., agent actions, tool calls, model responses)
|
||||
and forwards them to Galileo for observability and evaluation.
|
||||
|
||||
### Understanding the event listener
|
||||
|
||||
Creating a `CrewAIEventListener()` instance is all that’s
|
||||
required to enable Galileo for a CrewAI run. When instantiated, the listener:
|
||||
|
||||
- Automatically registers itself with CrewAI
|
||||
- Reads Galileo configuration from environment variables
|
||||
- Logs all run data to the Galileo project and log stream specified by
|
||||
`GALILEO_PROJECT` and `GALILEO_LOG_STREAM`
|
||||
|
||||
No additional configuration or code changes are required.
|
||||
All data from this run is logged to the Galileo project and
|
||||
log stream specified by your environment configuration
|
||||
(for example, GALILEO_PROJECT and GALILEO_LOG_STREAM).
|
||||
BIN
docs/images/galileo-trace-veiw.png
Normal file
BIN
docs/images/galileo-trace-veiw.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 239 KiB |
115
docs/ko/observability/galileo.mdx
Normal file
115
docs/ko/observability/galileo.mdx
Normal file
@@ -0,0 +1,115 @@
|
||||
---
|
||||
title: Galileo 갈릴레오
|
||||
description: CrewAI 추적 및 평가를 위한 Galileo 통합
|
||||
icon: telescope
|
||||
mode: "wide"
|
||||
---
|
||||
|
||||
## 개요
|
||||
|
||||
이 가이드는 **Galileo**를 **CrewAI**와 통합하는 방법을 보여줍니다.
|
||||
포괄적인 추적 및 평가 엔지니어링을 위한 것입니다.
|
||||
이 가이드가 끝나면 CrewAI 에이전트를 추적할 수 있게 됩니다.
|
||||
성과를 모니터링하고 행동을 평가합니다.
|
||||
Galileo의 강력한 관측 플랫폼.
|
||||
|
||||
> **갈릴레오(Galileo)란 무엇인가요?**[Galileo](https://galileo.ai/)는 AI 평가 및 관찰 가능성입니다.
|
||||
엔드투엔드 추적, 평가,
|
||||
AI 애플리케이션 모니터링. 이를 통해 팀은 실제 사실을 포착할 수 있습니다.
|
||||
견고한 가드레일을 만들고 체계적인 실험을 실행하세요.
|
||||
내장된 실험 추적 및 성능 분석으로 신뢰성 보장
|
||||
AI 수명주기 전반에 걸쳐 투명성과 지속적인 개선을 제공합니다.
|
||||
|
||||
## 시작하기
|
||||
|
||||
이 튜토리얼은 [CrewAI 빠른 시작](/ko/quickstart.mdx)을 따르며 추가하는 방법을 보여줍니다.
|
||||
갈릴레오의 [CrewAIEventListener](https://v2docs.galileo.ai/sdk-api/python/reference/handlers/crewai/handler),
|
||||
이벤트 핸들러.
|
||||
자세한 내용은 갈릴레오 문서를 참고하세요.
|
||||
[CrewAI 애플리케이션에 Galileo 추가](https://v2docs.galileo.ai/how-to-guides/third-party-integrations/add-galileo-to-crewai/add-galileo-to-crewai)
|
||||
방법 안내.
|
||||
|
||||
> **참고**이 튜토리얼에서는 [CrewAI 빠른 시작](/ko/quickstart.mdx)을 완료했다고 가정합니다.
|
||||
완전한 포괄적인 예제를 원한다면 Galileo
|
||||
[CrewAI SDK 예제 저장소](https://github.com/rungalileo/sdk-examples/tree/main/python/agent/crew-ai).
|
||||
|
||||
### 1단계: 종속성 설치
|
||||
|
||||
앱에 필요한 종속성을 설치합니다.
|
||||
원하는 방법으로 가상 환경을 생성하고,
|
||||
그런 다음 다음을 사용하여 해당 환경 내에 종속성을 설치하십시오.
|
||||
선호하는 도구:
|
||||
|
||||
```bash
|
||||
uv add galileo
|
||||
```
|
||||
|
||||
### 2단계: [CrewAI 빠른 시작](/ko/quickstart.mdx)에서 .env 파일에 추가
|
||||
|
||||
```bash
|
||||
# Your Galileo API key
|
||||
GALILEO_API_KEY="your-galileo-api-key"
|
||||
|
||||
# Your Galileo project name
|
||||
GALILEO_PROJECT="your-galileo-project-name"
|
||||
|
||||
# The name of the Log stream you want to use for logging
|
||||
GALILEO_LOG_STREAM="your-galileo-log-stream "
|
||||
```
|
||||
|
||||
### 3단계: Galileo 이벤트 리스너 추가
|
||||
|
||||
Galileo로 로깅을 활성화하려면 `CrewAIEventListener`의 인스턴스를 생성해야 합니다.
|
||||
다음을 통해 Galileo CrewAI 핸들러 패키지를 가져옵니다.
|
||||
main.py 파일 상단에 다음 코드를 추가하세요.
|
||||
|
||||
```python
|
||||
from galileo.handlers.crewai.handler import CrewAIEventListener
|
||||
```
|
||||
|
||||
실행 함수 시작 시 이벤트 리스너를 생성합니다.
|
||||
|
||||
```python
|
||||
def run():
|
||||
# Create the event listener
|
||||
CrewAIEventListener()
|
||||
# The rest of your existing code goes here
|
||||
```
|
||||
|
||||
리스너 인스턴스를 생성하면 자동으로
|
||||
CrewAI에 등록되었습니다.
|
||||
|
||||
### 4단계: Crew Agent 실행
|
||||
|
||||
CrewAI CLI를 사용하여 Crew Agent를 실행하세요.
|
||||
|
||||
```bash
|
||||
crewai run
|
||||
```
|
||||
|
||||
### 5단계: Galileo에서 추적 보기
|
||||
|
||||
승무원 에이전트가 완료되면 흔적이 플러시되어 Galileo에 나타납니다.
|
||||
|
||||

|
||||
|
||||
## 갈릴레오 통합 이해
|
||||
|
||||
Galileo는 이벤트 리스너를 등록하여 CrewAI와 통합됩니다.
|
||||
승무원 실행 이벤트(예: 에이전트 작업, 도구 호출, 모델 응답)를 캡처합니다.
|
||||
관찰 가능성과 평가를 위해 이를 갈릴레오에 전달합니다.
|
||||
|
||||
### 이벤트 리스너 이해
|
||||
|
||||
`CrewAIEventListener()` 인스턴스를 생성하는 것이 전부입니다.
|
||||
CrewAI 실행을 위해 Galileo를 활성화하는 데 필요합니다. 인스턴스화되면 리스너는 다음을 수행합니다.
|
||||
|
||||
-CrewAI에 자동으로 등록됩니다.
|
||||
-환경 변수에서 Galileo 구성을 읽습니다.
|
||||
-모든 실행 데이터를 Galileo 프로젝트 및 다음에서 지정한 로그 스트림에 기록합니다.
|
||||
`GALILEO_PROJECT` 및 `GALILEO_LOG_STREAM`
|
||||
|
||||
추가 구성이나 코드 변경이 필요하지 않습니다.
|
||||
이 실행의 모든 데이터는 Galileo 프로젝트에 기록되며
|
||||
환경 구성에 따라 지정된 로그 스트림
|
||||
(예: GALILEO_PROJECT 및 GALILEO_LOG_STREAM)
|
||||
115
docs/pt-BR/observability/galileo.mdx
Normal file
115
docs/pt-BR/observability/galileo.mdx
Normal file
@@ -0,0 +1,115 @@
|
||||
---
|
||||
title: Galileo Galileu
|
||||
description: Integração Galileo para rastreamento e avaliação CrewAI
|
||||
icon: telescope
|
||||
mode: "wide"
|
||||
---
|
||||
|
||||
## Visão geral
|
||||
|
||||
Este guia demonstra como integrar o **Galileo**com o **CrewAI**
|
||||
para rastreamento abrangente e engenharia de avaliação.
|
||||
Ao final deste guia, você será capaz de rastrear seus agentes CrewAI,
|
||||
monitorar seu desempenho e avaliar seu comportamento com
|
||||
A poderosa plataforma de observabilidade do Galileo.
|
||||
|
||||
> **O que é Galileo?**[Galileo](https://galileo.ai/) é avaliação e observabilidade de IA
|
||||
plataforma que oferece rastreamento, avaliação e
|
||||
e monitoramento de aplicações de IA. Ele permite que as equipes capturem a verdade,
|
||||
criar grades de proteção robustas e realizar experimentos sistemáticos com
|
||||
rastreamento de experimentos integrado e análise de desempenho -garantindo confiabilidade,
|
||||
transparência e melhoria contínua em todo o ciclo de vida da IA.
|
||||
|
||||
## Primeiros passos
|
||||
|
||||
Este tutorial segue o [CrewAI Quickstart](pt-BR/quickstart) e mostra como adicionar
|
||||
[CrewAIEventListener] do Galileo(https://v2docs.galileo.ai/sdk-api/python/reference/handlers/crewai/handler),
|
||||
um manipulador de eventos.
|
||||
Para mais informações, consulte Galileu
|
||||
[Adicionar Galileo a um aplicativo CrewAI](https://v2docs.galileo.ai/how-to-guides/third-party-integrations/add-galileo-to-crewai/add-galileo-to-crewai)
|
||||
guia prático.
|
||||
|
||||
> **Observação**Este tutorial pressupõe que você concluiu o [CrewAI Quickstart](pt-BR/quickstart).
|
||||
Se você quiser um exemplo completo e abrangente, consulte o Galileo
|
||||
[Repositório de exemplo SDK da CrewAI](https://github.com/rungalileo/sdk-examples/tree/main/python/agent/crew-ai).
|
||||
|
||||
### Etapa 1: instalar dependências
|
||||
|
||||
Instale as dependências necessárias para seu aplicativo.
|
||||
Crie um ambiente virtual usando seu método preferido,
|
||||
em seguida, instale dependências dentro desse ambiente usando seu
|
||||
ferramenta preferida:
|
||||
|
||||
```bash
|
||||
uv add galileo
|
||||
```
|
||||
|
||||
### Etapa 2: adicione ao arquivo .env do [CrewAI Quickstart](/pt-BR/quickstart)
|
||||
|
||||
```bash
|
||||
# Your Galileo API key
|
||||
GALILEO_API_KEY="your-galileo-api-key"
|
||||
|
||||
# Your Galileo project name
|
||||
GALILEO_PROJECT="your-galileo-project-name"
|
||||
|
||||
# The name of the Log stream you want to use for logging
|
||||
GALILEO_LOG_STREAM="your-galileo-log-stream "
|
||||
```
|
||||
|
||||
### Etapa 3: adicionar o ouvinte de eventos Galileo
|
||||
|
||||
Para habilitar o registro com Galileo, você precisa criar uma instância do `CrewAIEventListener`.
|
||||
Importe o pacote manipulador Galileo CrewAI por
|
||||
adicionando o seguinte código no topo do seu arquivo main.py:
|
||||
|
||||
```python
|
||||
from galileo.handlers.crewai.handler import CrewAIEventListener
|
||||
```
|
||||
|
||||
No início da sua função run, crie o ouvinte de evento:
|
||||
|
||||
```python
|
||||
def run():
|
||||
# Create the event listener
|
||||
CrewAIEventListener()
|
||||
# The rest of your existing code goes here
|
||||
```
|
||||
|
||||
Quando você cria a instância do listener, ela é automaticamente
|
||||
registrado na CrewAI.
|
||||
|
||||
### Etapa 4: administre sua Crew
|
||||
|
||||
Administre sua Crew com o CrewAI CLI:
|
||||
|
||||
```bash
|
||||
crewai run
|
||||
```
|
||||
|
||||
### Passo 5: Visualize os traços no Galileo
|
||||
|
||||
Assim que sua tripulação terminar, os rastros serão eliminados e aparecerão no Galileo.
|
||||
|
||||

|
||||
|
||||
## Compreendendo a integração do Galileo
|
||||
|
||||
Galileo se integra ao CrewAI registrando um ouvinte de evento
|
||||
que captura eventos de execução da tripulação (por exemplo, ações do agente, chamadas de ferramentas, respostas do modelo)
|
||||
e os encaminha ao Galileo para observabilidade e avaliação.
|
||||
|
||||
### Compreendendo o ouvinte de eventos
|
||||
|
||||
Criar uma instância `CrewAIEventListener()` é tudo o que você precisa
|
||||
necessário para habilitar o Galileo para uma execução do CrewAI. Quando instanciado, o ouvinte:
|
||||
|
||||
-Registra-se automaticamente no CrewAI
|
||||
-Lê a configuração do Galileo a partir de variáveis de ambiente
|
||||
-Registra todos os dados de execução no projeto Galileo e fluxo de log especificado por
|
||||
`GALILEO_PROJECT` e `GALILEO_LOG_STREAM`
|
||||
|
||||
Nenhuma configuração adicional ou alterações de código são necessárias.
|
||||
Todos os dados desta execução são registados no projecto Galileo e
|
||||
fluxo de log especificado pela configuração do seu ambiente
|
||||
(por exemplo, GALILEO_PROJECT e GALILEO_LOG_STREAM).
|
||||
@@ -1,4 +1,5 @@
|
||||
import inspect
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, InstanceOf, model_validator
|
||||
from typing_extensions import Self
|
||||
@@ -14,14 +15,14 @@ class FlowTrackable(BaseModel):
|
||||
inspecting the call stack.
|
||||
"""
|
||||
|
||||
parent_flow: InstanceOf[Flow] | None = Field(
|
||||
parent_flow: InstanceOf[Flow[Any]] | None = Field(
|
||||
default=None,
|
||||
description="The parent flow of the instance, if it was created inside a flow.",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _set_parent_flow(self) -> Self:
|
||||
max_depth = 5
|
||||
max_depth = 8
|
||||
frame = inspect.currentframe()
|
||||
|
||||
try:
|
||||
|
||||
@@ -443,7 +443,7 @@ class AzureCompletion(BaseLLM):
|
||||
params["presence_penalty"] = self.presence_penalty
|
||||
if self.max_tokens is not None:
|
||||
params["max_tokens"] = self.max_tokens
|
||||
if self.stop:
|
||||
if self.stop and self.supports_stop_words():
|
||||
params["stop"] = self.stop
|
||||
|
||||
# Handle tools/functions for Azure OpenAI models
|
||||
@@ -931,8 +931,28 @@ class AzureCompletion(BaseLLM):
|
||||
return self.is_openai_model
|
||||
|
||||
def supports_stop_words(self) -> bool:
|
||||
"""Check if the model supports stop words."""
|
||||
return True # Most Azure models support stop sequences
|
||||
"""Check if the model supports stop words.
|
||||
|
||||
Models using the Responses API (GPT-5 family, o-series reasoning models,
|
||||
computer-use-preview) do not support stop sequences.
|
||||
See: https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure
|
||||
"""
|
||||
model_lower = self.model.lower() if self.model else ""
|
||||
|
||||
if "gpt-5" in model_lower:
|
||||
return False
|
||||
|
||||
o_series_models = ["o1", "o3", "o4", "o1-mini", "o3-mini", "o4-mini"]
|
||||
|
||||
responses_api_models = ["computer-use-preview"]
|
||||
|
||||
unsupported_stop_models = o_series_models + responses_api_models
|
||||
|
||||
for unsupported in unsupported_stop_models:
|
||||
if unsupported in model_lower:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def get_context_window_size(self) -> int:
|
||||
"""Get the context window size for the model."""
|
||||
|
||||
@@ -229,6 +229,48 @@ def enforce_rpm_limit(
|
||||
request_within_rpm_limit()
|
||||
|
||||
|
||||
def _extract_tools_from_context(
|
||||
executor_context: CrewAgentExecutor | LiteAgent | None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Extract tools from executor context and convert to LLM-compatible format.
|
||||
|
||||
Args:
|
||||
executor_context: The executor context containing tools.
|
||||
|
||||
Returns:
|
||||
List of tool dictionaries in LLM-compatible format, or None if no tools.
|
||||
"""
|
||||
if executor_context is None:
|
||||
return None
|
||||
|
||||
# Get tools from executor context
|
||||
# CrewAgentExecutor has 'tools' attribute, LiteAgent has '_parsed_tools'
|
||||
tools: list[CrewStructuredTool] | None = None
|
||||
if hasattr(executor_context, "tools"):
|
||||
context_tools = executor_context.tools
|
||||
if isinstance(context_tools, list) and len(context_tools) > 0:
|
||||
tools = context_tools
|
||||
if tools is None and hasattr(executor_context, "_parsed_tools"):
|
||||
parsed_tools = executor_context._parsed_tools
|
||||
if isinstance(parsed_tools, list) and len(parsed_tools) > 0:
|
||||
tools = parsed_tools
|
||||
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
# Convert CrewStructuredTool to dict format expected by LLM
|
||||
tool_dicts: list[dict[str, Any]] = []
|
||||
for tool in tools:
|
||||
tool_dict: dict[str, Any] = {
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"args_schema": tool.args_schema,
|
||||
}
|
||||
tool_dicts.append(tool_dict)
|
||||
|
||||
return tool_dicts if tool_dicts else None
|
||||
|
||||
|
||||
def get_llm_response(
|
||||
llm: LLM | BaseLLM,
|
||||
messages: list[LLMMessage],
|
||||
@@ -264,14 +306,29 @@ def get_llm_response(
|
||||
raise ValueError("LLM call blocked by before_llm_call hook")
|
||||
messages = executor_context.messages
|
||||
|
||||
# Extract tools from executor context for native function calling support
|
||||
tools = _extract_tools_from_context(executor_context)
|
||||
|
||||
try:
|
||||
answer = llm.call(
|
||||
messages,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
# Only pass tools parameter if tools are available to maintain backward compatibility
|
||||
# with code that checks "tools" in kwargs
|
||||
if tools is not None:
|
||||
answer = llm.call(
|
||||
messages,
|
||||
tools=tools,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
else:
|
||||
answer = llm.call(
|
||||
messages,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
if not answer:
|
||||
@@ -292,7 +349,7 @@ async def aget_llm_response(
|
||||
from_task: Task | None = None,
|
||||
from_agent: Agent | LiteAgent | None = None,
|
||||
response_model: type[BaseModel] | None = None,
|
||||
executor_context: CrewAgentExecutor | None = None,
|
||||
executor_context: CrewAgentExecutor | LiteAgent | None = None,
|
||||
) -> str:
|
||||
"""Call the LLM asynchronously and return the response.
|
||||
|
||||
@@ -318,14 +375,29 @@ async def aget_llm_response(
|
||||
raise ValueError("LLM call blocked by before_llm_call hook")
|
||||
messages = executor_context.messages
|
||||
|
||||
# Extract tools from executor context for native function calling support
|
||||
tools = _extract_tools_from_context(executor_context)
|
||||
|
||||
try:
|
||||
answer = await llm.acall(
|
||||
messages,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
# Only pass tools parameter if tools are available to maintain backward compatibility
|
||||
# with code that checks "tools" in kwargs
|
||||
if tools is not None:
|
||||
answer = await llm.acall(
|
||||
messages,
|
||||
tools=tools,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
else:
|
||||
answer = await llm.acall(
|
||||
messages,
|
||||
callbacks=callbacks,
|
||||
from_task=from_task,
|
||||
from_agent=from_agent, # type: ignore[arg-type]
|
||||
response_model=response_model,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
if not answer:
|
||||
|
||||
@@ -1,370 +0,0 @@
|
||||
"""Lazy loader for Python packages.
|
||||
|
||||
Makes it easy to load subpackages and functions on demand.
|
||||
|
||||
Pulled from https://github.com/scientific-python/lazy-loader/blob/main/src/lazy_loader/__init__.py,
|
||||
modernized a little.
|
||||
"""
|
||||
|
||||
import ast
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
import importlib
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import inspect
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from typing import Any, NoReturn
|
||||
import warnings
|
||||
|
||||
import packaging.requirements
|
||||
|
||||
|
||||
_threadlock = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FrameData:
|
||||
"""Captured stack frame information for delayed error reporting."""
|
||||
|
||||
filename: str
|
||||
lineno: int
|
||||
function: str
|
||||
code_context: Sequence[str] | None
|
||||
|
||||
|
||||
def attach(
|
||||
package_name: str,
|
||||
submodules: set[str] | None = None,
|
||||
submod_attrs: dict[str, list[str]] | None = None,
|
||||
) -> tuple[Callable[[str], Any], Callable[[], list[str]], list[str]]:
|
||||
"""Attach lazily loaded submodules, functions, or other attributes.
|
||||
|
||||
Replaces a package's `__getattr__`, `__dir__`, and `__all__` such that
|
||||
imports work normally but occur upon first use.
|
||||
|
||||
Example:
|
||||
__getattr__, __dir__, __all__ = lazy.attach(
|
||||
__name__, ["mysubmodule"], {"foo": ["someattr"]}
|
||||
)
|
||||
|
||||
Args:
|
||||
package_name: The package name, typically ``__name__``.
|
||||
submodules: Set of submodule names to attach.
|
||||
submod_attrs: Mapping of submodule names to lists of attributes.
|
||||
These attributes are imported as they are used.
|
||||
|
||||
Returns:
|
||||
A tuple of (__getattr__, __dir__, __all__) to assign in the package.
|
||||
"""
|
||||
submod_attrs = submod_attrs or {}
|
||||
submodules = set(submodules) if submodules else set()
|
||||
attr_to_modules = {
|
||||
attr: mod for mod, attrs in submod_attrs.items() for attr in attrs
|
||||
}
|
||||
__all__ = sorted(submodules | attr_to_modules.keys())
|
||||
|
||||
def __getattr__(name: str) -> Any: # noqa: N807
|
||||
if name in submodules:
|
||||
return importlib.import_module(f"{package_name}.{name}")
|
||||
if name in attr_to_modules:
|
||||
submod_path = f"{package_name}.{attr_to_modules[name]}"
|
||||
submod = importlib.import_module(submod_path)
|
||||
attr = getattr(submod, name)
|
||||
|
||||
# If the attribute lives in a file (module) with the same
|
||||
# name as the attribute, ensure that the attribute and *not*
|
||||
# the module is accessible on the package.
|
||||
if name == attr_to_modules[name]:
|
||||
pkg = sys.modules[package_name]
|
||||
pkg.__dict__[name] = attr
|
||||
|
||||
return attr
|
||||
raise AttributeError(f"No {package_name} attribute {name}")
|
||||
|
||||
def __dir__() -> list[str]: # noqa: N807
|
||||
return __all__.copy()
|
||||
|
||||
if os.environ.get("EAGER_IMPORT"):
|
||||
for attr in set(attr_to_modules.keys()) | submodules:
|
||||
__getattr__(attr)
|
||||
|
||||
return __getattr__, __dir__, __all__.copy()
|
||||
|
||||
|
||||
class DelayedImportErrorModule(types.ModuleType):
|
||||
"""Module type that delays raising ModuleNotFoundError until attribute access.
|
||||
|
||||
Captures stack frame data to provide helpful error messages showing where
|
||||
the original import was attempted.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
frame_data: _FrameData,
|
||||
*args: Any,
|
||||
message: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the delayed error module.
|
||||
|
||||
Args:
|
||||
frame_data: Captured frame information for error reporting.
|
||||
*args: Positional arguments passed to ModuleType.
|
||||
message: The error message to display when accessed.
|
||||
**kwargs: Keyword arguments passed to ModuleType.
|
||||
"""
|
||||
self._frame_data = frame_data
|
||||
self._message = message
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def __getattr__(self, name: str) -> NoReturn:
|
||||
"""Raise ModuleNotFoundError with detailed context on any attribute access."""
|
||||
frame = self._frame_data
|
||||
code = "".join(frame.code_context) if frame.code_context else ""
|
||||
raise ModuleNotFoundError(
|
||||
f"{self._message}\n\n"
|
||||
"This error is lazily reported, having originally occurred in\n"
|
||||
f" File {frame.filename}, line {frame.lineno}, in {frame.function}\n\n"
|
||||
f"----> {code.strip()}"
|
||||
)
|
||||
|
||||
|
||||
def load(
|
||||
fullname: str,
|
||||
*,
|
||||
require: str | None = None,
|
||||
error_on_import: bool = False,
|
||||
suppress_warning: bool = False,
|
||||
) -> types.ModuleType:
|
||||
"""Return a lazily imported proxy for a module.
|
||||
|
||||
The proxy module delays actual import until first attribute access.
|
||||
|
||||
Example:
|
||||
np = lazy.load("numpy")
|
||||
|
||||
def myfunc():
|
||||
np.norm(...)
|
||||
|
||||
Warning:
|
||||
Lazily loading subpackages causes the parent package to be eagerly
|
||||
loaded. Use `lazy_loader.attach` instead for subpackages.
|
||||
|
||||
Args:
|
||||
fullname: The full name of the module to import (e.g., "scipy").
|
||||
require: A PEP-508 dependency requirement (e.g., "numpy >=1.24").
|
||||
If specified, raises an error if the installed version doesn't match.
|
||||
error_on_import: If True, raise import errors immediately.
|
||||
If False (default), delay errors until module is accessed.
|
||||
suppress_warning: If True, suppress the warning when loading subpackages.
|
||||
|
||||
Returns:
|
||||
A proxy module that loads on first attribute access.
|
||||
"""
|
||||
with _threadlock:
|
||||
module = sys.modules.get(fullname)
|
||||
|
||||
# Most common, short-circuit
|
||||
if module is not None and require is None:
|
||||
return module
|
||||
|
||||
have_module = module is not None
|
||||
|
||||
if not suppress_warning and "." in fullname:
|
||||
msg = (
|
||||
"subpackages can technically be lazily loaded, but it causes the "
|
||||
"package to be eagerly loaded even if it is already lazily loaded. "
|
||||
"So, you probably shouldn't use subpackages with this lazy feature."
|
||||
)
|
||||
warnings.warn(msg, RuntimeWarning, stacklevel=2)
|
||||
|
||||
spec = None
|
||||
|
||||
if not have_module:
|
||||
spec = importlib.util.find_spec(fullname)
|
||||
have_module = spec is not None
|
||||
|
||||
if not have_module:
|
||||
not_found_message = f"No module named '{fullname}'"
|
||||
elif require is not None:
|
||||
try:
|
||||
have_module = _check_requirement(require)
|
||||
except ModuleNotFoundError as e:
|
||||
raise ValueError(
|
||||
f"Found module '{fullname}' but cannot test "
|
||||
"requirement '{require}'. "
|
||||
"Requirements must match distribution name, not module name."
|
||||
) from e
|
||||
|
||||
not_found_message = f"No distribution can be found matching '{require}'"
|
||||
|
||||
if not have_module:
|
||||
if error_on_import:
|
||||
raise ModuleNotFoundError(not_found_message)
|
||||
|
||||
parent = inspect.stack()[1]
|
||||
frame_data = _FrameData(
|
||||
filename=parent.filename,
|
||||
lineno=parent.lineno,
|
||||
function=parent.function,
|
||||
code_context=parent.code_context,
|
||||
)
|
||||
del parent
|
||||
return DelayedImportErrorModule(
|
||||
frame_data,
|
||||
"DelayedImportErrorModule",
|
||||
message=not_found_message,
|
||||
)
|
||||
|
||||
if spec is not None:
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[fullname] = module
|
||||
|
||||
if spec.loader is not None:
|
||||
loader = importlib.util.LazyLoader(spec.loader)
|
||||
loader.exec_module(module)
|
||||
|
||||
if module is None:
|
||||
raise ModuleNotFoundError(f"No module named '{fullname}'")
|
||||
|
||||
return module
|
||||
|
||||
|
||||
def _check_requirement(require: str) -> bool:
|
||||
"""Verify that a package requirement is satisfied.
|
||||
|
||||
Args:
|
||||
require: A dependency requirement as defined in PEP-508.
|
||||
|
||||
Returns:
|
||||
True if the installed version matches the requirement, False otherwise.
|
||||
|
||||
Raises:
|
||||
ModuleNotFoundError: If the package is not installed.
|
||||
"""
|
||||
req = packaging.requirements.Requirement(require)
|
||||
return req.specifier.contains(
|
||||
importlib.metadata.version(req.name),
|
||||
prereleases=True,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StubVisitor(ast.NodeVisitor):
|
||||
"""AST visitor to parse a stub file for submodules and submod_attrs."""
|
||||
|
||||
_submodules: set[str] = field(default_factory=set)
|
||||
_submod_attrs: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
||||
"""Visit an ImportFrom node and extract submodule/attribute information.
|
||||
|
||||
Args:
|
||||
node: The AST ImportFrom node to visit.
|
||||
|
||||
Raises:
|
||||
ValueError: If the import is not a relative import or uses star import.
|
||||
"""
|
||||
if node.level != 1:
|
||||
raise ValueError(
|
||||
"Only within-module imports are supported (`from .* import`)"
|
||||
)
|
||||
names = [alias.name for alias in node.names]
|
||||
if node.module:
|
||||
if "*" in names:
|
||||
raise ValueError(
|
||||
f"lazy stub loader does not support star import "
|
||||
f"`from {node.module} import *`"
|
||||
)
|
||||
self._submod_attrs.setdefault(node.module, []).extend(names)
|
||||
else:
|
||||
self._submodules.update(names)
|
||||
|
||||
|
||||
def attach_stub(
|
||||
package_name: str,
|
||||
filename: str,
|
||||
) -> tuple[Callable[[str], Any], Callable[[], list[str]], list[str]]:
|
||||
"""Attach lazily loaded submodules and functions from a type stub.
|
||||
|
||||
Parses a `.pyi` stub file to infer submodules and attributes. This allows
|
||||
static type checkers to find imports while providing lazy loading at runtime.
|
||||
|
||||
Args:
|
||||
package_name: The package name, typically ``__name__``.
|
||||
filename: Path to `.py` file with an adjacent `.pyi` file.
|
||||
Typically use ``__file__``.
|
||||
|
||||
Returns:
|
||||
A tuple of (__getattr__, __dir__, __all__) to assign in the package.
|
||||
|
||||
Raises:
|
||||
ValueError: If stub file is not found or contains invalid imports.
|
||||
"""
|
||||
path = Path(filename)
|
||||
stubfile = path if path.suffix == ".pyi" else path.with_suffix(".pyi")
|
||||
|
||||
if not stubfile.exists():
|
||||
raise ValueError(f"Cannot load imports from non-existent stub {stubfile!r}")
|
||||
|
||||
visitor = _StubVisitor()
|
||||
visitor.visit(ast.parse(stubfile.read_text()))
|
||||
return attach(package_name, visitor._submodules, visitor._submod_attrs)
|
||||
|
||||
|
||||
def lazy_exports_stub(package_name: str, filename: str) -> None:
|
||||
"""Install lazy loading on a module based on its .pyi stub file.
|
||||
|
||||
Parses the adjacent `.pyi` stub file to determine what to export lazily.
|
||||
Type checkers see the stub, runtime gets lazy loading.
|
||||
|
||||
Example:
|
||||
# __init__.py
|
||||
from crewai.utilities.lazy import lazy_exports_stub
|
||||
lazy_exports_stub(__name__, __file__)
|
||||
|
||||
# __init__.pyi
|
||||
from .config import ChromaDBConfig, ChromaDBSettings
|
||||
from .types import EmbeddingType
|
||||
|
||||
Args:
|
||||
package_name: The package name, typically ``__name__``.
|
||||
filename: Path to the module file, typically ``__file__``.
|
||||
"""
|
||||
__getattr__, __dir__, __all__ = attach_stub(package_name, filename)
|
||||
module = sys.modules[package_name]
|
||||
module.__getattr__ = __getattr__ # type: ignore[method-assign]
|
||||
module.__dir__ = __dir__ # type: ignore[method-assign]
|
||||
module.__dict__["__all__"] = __all__
|
||||
|
||||
|
||||
def lazy_exports(
|
||||
package_name: str,
|
||||
submod_attrs: dict[str, list[str]],
|
||||
submodules: set[str] | None = None,
|
||||
) -> None:
|
||||
"""Install lazy loading on a module.
|
||||
|
||||
Example:
|
||||
from crewai.utilities.lazy import lazy_exports
|
||||
|
||||
lazy_exports(__name__, {
|
||||
'config': ['ChromaDBConfig', 'ChromaDBSettings'],
|
||||
'types': ['EmbeddingType'],
|
||||
})
|
||||
|
||||
Args:
|
||||
package_name: The package name, typically ``__name__``.
|
||||
submod_attrs: Mapping of submodule names to lists of attributes.
|
||||
submodules: Optional set of submodule names to expose directly.
|
||||
"""
|
||||
__getattr__, __dir__, __all__ = attach(package_name, submodules, submod_attrs)
|
||||
module = sys.modules[package_name]
|
||||
module.__getattr__ = __getattr__ # type: ignore[method-assign]
|
||||
module.__dir__ = __dir__ # type: ignore[method-assign]
|
||||
module.__dict__["__all__"] = __all__
|
||||
@@ -515,6 +515,94 @@ def test_azure_supports_stop_words():
|
||||
assert llm.supports_stop_words() == True
|
||||
|
||||
|
||||
def test_azure_gpt5_models_do_not_support_stop_words():
|
||||
"""
|
||||
Test that GPT-5 family models do not support stop words.
|
||||
GPT-5 models use the Responses API which doesn't support stop sequences.
|
||||
See: https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure
|
||||
"""
|
||||
# GPT-5 base models
|
||||
gpt5_models = [
|
||||
"azure/gpt-5",
|
||||
"azure/gpt-5-mini",
|
||||
"azure/gpt-5-nano",
|
||||
"azure/gpt-5-chat",
|
||||
# GPT-5.1 series
|
||||
"azure/gpt-5.1",
|
||||
"azure/gpt-5.1-chat",
|
||||
"azure/gpt-5.1-codex",
|
||||
"azure/gpt-5.1-codex-mini",
|
||||
# GPT-5.2 series
|
||||
"azure/gpt-5.2",
|
||||
"azure/gpt-5.2-chat",
|
||||
]
|
||||
|
||||
for model_name in gpt5_models:
|
||||
llm = LLM(model=model_name)
|
||||
assert llm.supports_stop_words() == False, f"Expected {model_name} to NOT support stop words"
|
||||
|
||||
|
||||
def test_azure_o_series_models_do_not_support_stop_words():
|
||||
"""
|
||||
Test that o-series reasoning models do not support stop words.
|
||||
"""
|
||||
o_series_models = [
|
||||
"azure/o1",
|
||||
"azure/o1-mini",
|
||||
"azure/o3",
|
||||
"azure/o3-mini",
|
||||
"azure/o4",
|
||||
"azure/o4-mini",
|
||||
]
|
||||
|
||||
for model_name in o_series_models:
|
||||
llm = LLM(model=model_name)
|
||||
assert llm.supports_stop_words() == False, f"Expected {model_name} to NOT support stop words"
|
||||
|
||||
|
||||
def test_azure_responses_api_models_do_not_support_stop_words():
|
||||
"""
|
||||
Test that models using the Responses API do not support stop words.
|
||||
"""
|
||||
responses_api_models = [
|
||||
"azure/computer-use-preview",
|
||||
]
|
||||
|
||||
for model_name in responses_api_models:
|
||||
llm = LLM(model=model_name)
|
||||
assert llm.supports_stop_words() == False, f"Expected {model_name} to NOT support stop words"
|
||||
|
||||
|
||||
def test_azure_stop_words_not_included_for_unsupported_models():
|
||||
"""
|
||||
Test that stop words are not included in completion params for models that don't support them.
|
||||
"""
|
||||
with patch.dict(os.environ, {
|
||||
"AZURE_API_KEY": "test-key",
|
||||
"AZURE_ENDPOINT": "https://models.inference.ai.azure.com"
|
||||
}):
|
||||
# Test GPT-5 model - stop should NOT be included even if set
|
||||
llm_gpt5 = LLM(
|
||||
model="azure/gpt-5-nano",
|
||||
stop=["STOP", "END"]
|
||||
)
|
||||
params = llm_gpt5._prepare_completion_params(
|
||||
messages=[{"role": "user", "content": "test"}]
|
||||
)
|
||||
assert "stop" not in params, "stop should not be included for GPT-5 models"
|
||||
|
||||
# Test regular model - stop SHOULD be included
|
||||
llm_gpt4 = LLM(
|
||||
model="azure/gpt-4",
|
||||
stop=["STOP", "END"]
|
||||
)
|
||||
params = llm_gpt4._prepare_completion_params(
|
||||
messages=[{"role": "user", "content": "test"}]
|
||||
)
|
||||
assert "stop" in params, "stop should be included for GPT-4 models"
|
||||
assert params["stop"] == ["STOP", "END"]
|
||||
|
||||
|
||||
def test_azure_context_window_size():
|
||||
"""
|
||||
Test that Azure models return correct context window sizes
|
||||
|
||||
@@ -4500,6 +4500,71 @@ def test_crew_copy_with_memory():
|
||||
pytest.fail(f"Copying crew raised an unexpected exception: {e}")
|
||||
|
||||
|
||||
def test_sets_parent_flow_when_using_crewbase_pattern_inside_flow():
|
||||
@CrewBase
|
||||
class TestCrew:
|
||||
agents_config = None
|
||||
tasks_config = None
|
||||
|
||||
agents: list[BaseAgent]
|
||||
tasks: list[Task]
|
||||
|
||||
@agent
|
||||
def researcher(self) -> Agent:
|
||||
return Agent(
|
||||
role="Researcher",
|
||||
goal="Research things",
|
||||
backstory="Expert researcher",
|
||||
)
|
||||
|
||||
@agent
|
||||
def writer_agent(self) -> Agent:
|
||||
return Agent(
|
||||
role="Writer",
|
||||
goal="Write things",
|
||||
backstory="Expert writer",
|
||||
)
|
||||
|
||||
@task
|
||||
def research_task(self) -> Task:
|
||||
return Task(
|
||||
description="Test task for researcher",
|
||||
expected_output="output",
|
||||
agent=self.researcher(),
|
||||
)
|
||||
|
||||
@task
|
||||
def write_task(self) -> Task:
|
||||
return Task(
|
||||
description="Test task for writer",
|
||||
expected_output="output",
|
||||
agent=self.writer_agent(),
|
||||
)
|
||||
|
||||
@crew
|
||||
def crew(self) -> Crew:
|
||||
return Crew(
|
||||
agents=self.agents,
|
||||
tasks=self.tasks,
|
||||
process=Process.sequential,
|
||||
)
|
||||
|
||||
captured_crew = None
|
||||
|
||||
class MyFlow(Flow):
|
||||
@start()
|
||||
def start_method(self):
|
||||
nonlocal captured_crew
|
||||
captured_crew = TestCrew().crew()
|
||||
return captured_crew
|
||||
|
||||
flow = MyFlow()
|
||||
flow.kickoff()
|
||||
|
||||
assert captured_crew is not None
|
||||
assert captured_crew.parent_flow is flow
|
||||
|
||||
|
||||
def test_sets_parent_flow_when_outside_flow(researcher, writer):
|
||||
crew = Crew(
|
||||
agents=[researcher, writer],
|
||||
|
||||
457
lib/crewai/tests/utilities/test_agent_utils.py
Normal file
457
lib/crewai/tests/utilities/test_agent_utils.py
Normal file
@@ -0,0 +1,457 @@
|
||||
"""Unit tests for agent_utils module.
|
||||
|
||||
Tests the utility functions for agent execution including tool extraction
|
||||
and LLM response handling.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from crewai.tools.structured_tool import CrewStructuredTool
|
||||
from crewai.utilities.agent_utils import (
|
||||
_extract_tools_from_context,
|
||||
aget_llm_response,
|
||||
get_llm_response,
|
||||
)
|
||||
from crewai.utilities.printer import Printer
|
||||
|
||||
|
||||
class MockArgsSchema(BaseModel):
|
||||
"""Mock args schema for testing."""
|
||||
|
||||
query: str = Field(description="The search query")
|
||||
|
||||
|
||||
class TestExtractToolsFromContext:
|
||||
"""Test _extract_tools_from_context function."""
|
||||
|
||||
def test_returns_none_when_context_is_none(self):
|
||||
"""Test that None is returned when executor_context is None."""
|
||||
result = _extract_tools_from_context(None)
|
||||
assert result is None
|
||||
|
||||
def test_returns_none_when_no_tools_attribute(self):
|
||||
"""Test that None is returned when context has no tools."""
|
||||
mock_context = Mock(spec=[])
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
assert result is None
|
||||
|
||||
def test_returns_none_when_tools_is_empty(self):
|
||||
"""Test that None is returned when tools list is empty."""
|
||||
mock_context = Mock()
|
||||
mock_context.tools = []
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
assert result is None
|
||||
|
||||
def test_extracts_tools_from_crew_agent_executor(self):
|
||||
"""Test tool extraction from CrewAgentExecutor (has 'tools' attribute)."""
|
||||
mock_tool = CrewStructuredTool(
|
||||
name="search_tool",
|
||||
description="A tool for searching",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: f"Results for {query}",
|
||||
)
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.tools = [mock_tool]
|
||||
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "search_tool"
|
||||
assert result[0]["description"] == "A tool for searching"
|
||||
assert result[0]["args_schema"] == MockArgsSchema
|
||||
|
||||
def test_extracts_tools_from_lite_agent(self):
|
||||
"""Test tool extraction from LiteAgent (has '_parsed_tools' attribute)."""
|
||||
mock_tool = CrewStructuredTool(
|
||||
name="calculator_tool",
|
||||
description="A tool for calculations",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: f"Calculated {query}",
|
||||
)
|
||||
|
||||
mock_context = Mock(spec=["_parsed_tools"])
|
||||
mock_context._parsed_tools = [mock_tool]
|
||||
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "calculator_tool"
|
||||
assert result[0]["description"] == "A tool for calculations"
|
||||
assert result[0]["args_schema"] == MockArgsSchema
|
||||
|
||||
def test_extracts_multiple_tools(self):
|
||||
"""Test extraction of multiple tools."""
|
||||
tool1 = CrewStructuredTool(
|
||||
name="tool1",
|
||||
description="First tool",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "result1",
|
||||
)
|
||||
tool2 = CrewStructuredTool(
|
||||
name="tool2",
|
||||
description="Second tool",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "result2",
|
||||
)
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.tools = [tool1, tool2]
|
||||
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == 2
|
||||
assert result[0]["name"] == "tool1"
|
||||
assert result[1]["name"] == "tool2"
|
||||
|
||||
def test_prefers_tools_over_parsed_tools(self):
|
||||
"""Test that 'tools' attribute is preferred over '_parsed_tools'."""
|
||||
tool_from_tools = CrewStructuredTool(
|
||||
name="from_tools",
|
||||
description="Tool from tools attribute",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "from_tools",
|
||||
)
|
||||
tool_from_parsed = CrewStructuredTool(
|
||||
name="from_parsed",
|
||||
description="Tool from _parsed_tools attribute",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "from_parsed",
|
||||
)
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.tools = [tool_from_tools]
|
||||
mock_context._parsed_tools = [tool_from_parsed]
|
||||
|
||||
result = _extract_tools_from_context(mock_context)
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "from_tools"
|
||||
|
||||
|
||||
class TestGetLlmResponse:
|
||||
"""Test get_llm_response function."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm(self):
|
||||
"""Create a mock LLM."""
|
||||
llm = Mock()
|
||||
llm.call = Mock(return_value="LLM response")
|
||||
return llm
|
||||
|
||||
@pytest.fixture
|
||||
def mock_printer(self):
|
||||
"""Create a mock printer."""
|
||||
return Mock(spec=Printer)
|
||||
|
||||
def test_passes_tools_to_llm_call(self, mock_llm, mock_printer):
|
||||
"""Test that tools are extracted and passed to llm.call()."""
|
||||
mock_tool = CrewStructuredTool(
|
||||
name="test_tool",
|
||||
description="A test tool",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "result",
|
||||
)
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.tools = [mock_tool]
|
||||
mock_context.messages = [{"role": "user", "content": "test"}]
|
||||
mock_context.before_llm_call_hooks = []
|
||||
mock_context.after_llm_call_hooks = []
|
||||
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_before_llm_call_hooks",
|
||||
return_value=True,
|
||||
):
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_after_llm_call_hooks",
|
||||
return_value="LLM response",
|
||||
):
|
||||
result = get_llm_response(
|
||||
llm=mock_llm,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=mock_context,
|
||||
)
|
||||
|
||||
# Verify llm.call was called with tools parameter
|
||||
mock_llm.call.assert_called_once()
|
||||
call_kwargs = mock_llm.call.call_args[1]
|
||||
assert "tools" in call_kwargs
|
||||
assert call_kwargs["tools"] is not None
|
||||
assert len(call_kwargs["tools"]) == 1
|
||||
assert call_kwargs["tools"][0]["name"] == "test_tool"
|
||||
|
||||
def test_does_not_pass_tools_when_no_context(self, mock_llm, mock_printer):
|
||||
"""Test that tools parameter is not passed when no executor_context."""
|
||||
result = get_llm_response(
|
||||
llm=mock_llm,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=None,
|
||||
)
|
||||
|
||||
mock_llm.call.assert_called_once()
|
||||
call_kwargs = mock_llm.call.call_args[1]
|
||||
# tools should NOT be in kwargs when there are no tools
|
||||
# This maintains backward compatibility with code that checks "tools" in kwargs
|
||||
assert "tools" not in call_kwargs
|
||||
|
||||
def test_does_not_pass_tools_when_context_has_no_tools(
|
||||
self, mock_llm, mock_printer
|
||||
):
|
||||
"""Test that tools parameter is not passed when context has no tools."""
|
||||
mock_context = Mock()
|
||||
mock_context.tools = []
|
||||
mock_context.messages = [{"role": "user", "content": "test"}]
|
||||
mock_context.before_llm_call_hooks = []
|
||||
mock_context.after_llm_call_hooks = []
|
||||
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_before_llm_call_hooks",
|
||||
return_value=True,
|
||||
):
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_after_llm_call_hooks",
|
||||
return_value="LLM response",
|
||||
):
|
||||
result = get_llm_response(
|
||||
llm=mock_llm,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=mock_context,
|
||||
)
|
||||
|
||||
mock_llm.call.assert_called_once()
|
||||
call_kwargs = mock_llm.call.call_args[1]
|
||||
# tools should NOT be in kwargs when there are no tools
|
||||
# This maintains backward compatibility with code that checks "tools" in kwargs
|
||||
assert "tools" not in call_kwargs
|
||||
|
||||
|
||||
class TestAgetLlmResponse:
|
||||
"""Test aget_llm_response async function."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm(self):
|
||||
"""Create a mock LLM with async call."""
|
||||
llm = Mock()
|
||||
llm.acall = AsyncMock(return_value="Async LLM response")
|
||||
return llm
|
||||
|
||||
@pytest.fixture
|
||||
def mock_printer(self):
|
||||
"""Create a mock printer."""
|
||||
return Mock(spec=Printer)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_tools_to_llm_acall(self, mock_llm, mock_printer):
|
||||
"""Test that tools are extracted and passed to llm.acall()."""
|
||||
mock_tool = CrewStructuredTool(
|
||||
name="async_test_tool",
|
||||
description="An async test tool",
|
||||
args_schema=MockArgsSchema,
|
||||
func=lambda query: "async result",
|
||||
)
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.tools = [mock_tool]
|
||||
mock_context.messages = [{"role": "user", "content": "async test"}]
|
||||
mock_context.before_llm_call_hooks = []
|
||||
mock_context.after_llm_call_hooks = []
|
||||
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_before_llm_call_hooks",
|
||||
return_value=True,
|
||||
):
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_after_llm_call_hooks",
|
||||
return_value="Async LLM response",
|
||||
):
|
||||
result = await aget_llm_response(
|
||||
llm=mock_llm,
|
||||
messages=[{"role": "user", "content": "async test"}],
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=mock_context,
|
||||
)
|
||||
|
||||
# Verify llm.acall was called with tools parameter
|
||||
mock_llm.acall.assert_called_once()
|
||||
call_kwargs = mock_llm.acall.call_args[1]
|
||||
assert "tools" in call_kwargs
|
||||
assert call_kwargs["tools"] is not None
|
||||
assert len(call_kwargs["tools"]) == 1
|
||||
assert call_kwargs["tools"][0]["name"] == "async_test_tool"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_pass_tools_when_no_context(self, mock_llm, mock_printer):
|
||||
"""Test that tools parameter is not passed when no executor_context."""
|
||||
result = await aget_llm_response(
|
||||
llm=mock_llm,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=None,
|
||||
)
|
||||
|
||||
mock_llm.acall.assert_called_once()
|
||||
call_kwargs = mock_llm.acall.call_args[1]
|
||||
# tools should NOT be in kwargs when there are no tools
|
||||
# This maintains backward compatibility with code that checks "tools" in kwargs
|
||||
assert "tools" not in call_kwargs
|
||||
|
||||
|
||||
class TestToolsPassedToGeminiModels:
|
||||
"""Test that tools are properly passed for Gemini models.
|
||||
|
||||
This test class specifically addresses GitHub issue #4238 where
|
||||
Gemini models fail with UNEXPECTED_TOOL_CALL errors because tools
|
||||
were not being passed to llm.call().
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_gemini_llm(self):
|
||||
"""Create a mock Gemini LLM."""
|
||||
llm = Mock()
|
||||
llm.model = "gemini/gemini-2.0-flash-exp"
|
||||
llm.call = Mock(return_value="Gemini response with tool call")
|
||||
return llm
|
||||
|
||||
@pytest.fixture
|
||||
def mock_printer(self):
|
||||
"""Create a mock printer."""
|
||||
return Mock(spec=Printer)
|
||||
|
||||
@pytest.fixture
|
||||
def delegation_tools(self):
|
||||
"""Create mock delegation tools similar to hierarchical crew setup."""
|
||||
|
||||
class DelegateWorkArgsSchema(BaseModel):
|
||||
task: str = Field(description="The task to delegate")
|
||||
context: str = Field(description="Context for the task")
|
||||
coworker: str = Field(description="The coworker to delegate to")
|
||||
|
||||
class AskQuestionArgsSchema(BaseModel):
|
||||
question: str = Field(description="The question to ask")
|
||||
context: str = Field(description="Context for the question")
|
||||
coworker: str = Field(description="The coworker to ask")
|
||||
|
||||
delegate_tool = CrewStructuredTool(
|
||||
name="Delegate work to coworker",
|
||||
description="Delegate a specific task to one of your coworkers",
|
||||
args_schema=DelegateWorkArgsSchema,
|
||||
func=lambda task, context, coworker: f"Delegated {task} to {coworker}",
|
||||
)
|
||||
|
||||
ask_question_tool = CrewStructuredTool(
|
||||
name="Ask question to coworker",
|
||||
description="Ask a specific question to one of your coworkers",
|
||||
args_schema=AskQuestionArgsSchema,
|
||||
func=lambda question, context, coworker: f"Asked {question} to {coworker}",
|
||||
)
|
||||
|
||||
return [delegate_tool, ask_question_tool]
|
||||
|
||||
def test_gemini_receives_tools_for_hierarchical_crew(
|
||||
self, mock_gemini_llm, mock_printer, delegation_tools
|
||||
):
|
||||
"""Test that Gemini models receive tools when used in hierarchical crew.
|
||||
|
||||
This test verifies the fix for issue #4238 where the manager agent
|
||||
in a hierarchical crew would fail because tools weren't passed to
|
||||
the Gemini model, causing UNEXPECTED_TOOL_CALL errors.
|
||||
"""
|
||||
mock_context = Mock()
|
||||
mock_context.tools = delegation_tools
|
||||
mock_context.messages = [
|
||||
{"role": "system", "content": "You are a manager agent"},
|
||||
{"role": "user", "content": "Coordinate the team to answer this question"},
|
||||
]
|
||||
mock_context.before_llm_call_hooks = []
|
||||
mock_context.after_llm_call_hooks = []
|
||||
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_before_llm_call_hooks",
|
||||
return_value=True,
|
||||
):
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_after_llm_call_hooks",
|
||||
return_value="Gemini response with tool call",
|
||||
):
|
||||
result = get_llm_response(
|
||||
llm=mock_gemini_llm,
|
||||
messages=mock_context.messages,
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=mock_context,
|
||||
)
|
||||
|
||||
# Verify that tools were passed to the Gemini model
|
||||
mock_gemini_llm.call.assert_called_once()
|
||||
call_kwargs = mock_gemini_llm.call.call_args[1]
|
||||
|
||||
assert "tools" in call_kwargs
|
||||
assert call_kwargs["tools"] is not None
|
||||
assert len(call_kwargs["tools"]) == 2
|
||||
|
||||
# Verify the delegation tools are properly formatted
|
||||
tool_names = [t["name"] for t in call_kwargs["tools"]]
|
||||
assert "Delegate work to coworker" in tool_names
|
||||
assert "Ask question to coworker" in tool_names
|
||||
|
||||
# Verify each tool has the required fields
|
||||
for tool_dict in call_kwargs["tools"]:
|
||||
assert "name" in tool_dict
|
||||
assert "description" in tool_dict
|
||||
assert "args_schema" in tool_dict
|
||||
|
||||
def test_tool_dict_format_compatible_with_llm_providers(
|
||||
self, mock_gemini_llm, mock_printer, delegation_tools
|
||||
):
|
||||
"""Test that extracted tools are in a format compatible with LLM providers.
|
||||
|
||||
The tool dictionaries should have 'name', 'description', and 'args_schema'
|
||||
fields that can be processed by the LLM's _prepare_completion_params method.
|
||||
"""
|
||||
mock_context = Mock()
|
||||
mock_context.tools = delegation_tools
|
||||
mock_context.messages = [{"role": "user", "content": "test"}]
|
||||
mock_context.before_llm_call_hooks = []
|
||||
mock_context.after_llm_call_hooks = []
|
||||
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_before_llm_call_hooks",
|
||||
return_value=True,
|
||||
):
|
||||
with patch(
|
||||
"crewai.utilities.agent_utils._setup_after_llm_call_hooks",
|
||||
return_value="response",
|
||||
):
|
||||
get_llm_response(
|
||||
llm=mock_gemini_llm,
|
||||
messages=mock_context.messages,
|
||||
callbacks=[],
|
||||
printer=mock_printer,
|
||||
executor_context=mock_context,
|
||||
)
|
||||
|
||||
call_kwargs = mock_gemini_llm.call.call_args[1]
|
||||
tools = call_kwargs["tools"]
|
||||
|
||||
for tool_dict in tools:
|
||||
# Verify the format matches what extract_tool_info() in common.py expects
|
||||
assert isinstance(tool_dict["name"], str)
|
||||
assert isinstance(tool_dict["description"], str)
|
||||
# args_schema should be a Pydantic model class
|
||||
assert hasattr(tool_dict["args_schema"], "model_json_schema")
|
||||
Reference in New Issue
Block a user