mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-03-15 00:08:14 +00:00
Compare commits
9 Commits
1.10.2rc1
...
cursor/cod
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3311988e81 | ||
|
|
e1d7de0dba | ||
|
|
96b07bfc84 | ||
|
|
b8d7942675 | ||
|
|
1ffbb3f76e | ||
|
|
4f37f59dc4 | ||
|
|
9365b270e0 | ||
|
|
31ab821bb6 | ||
|
|
6ee0cacd7c |
@@ -4,6 +4,25 @@ description: "Product updates, improvements, and bug fixes for CrewAI"
|
||||
icon: "clock"
|
||||
mode: "wide"
|
||||
---
|
||||
<Update label="Mar 14, 2026">
|
||||
## v1.10.2rc2
|
||||
|
||||
[View release on GitHub](https://github.com/crewAIInc/crewAI/releases/tag/1.10.2rc2)
|
||||
|
||||
## What's Changed
|
||||
|
||||
### Bug Fixes
|
||||
- Remove exclusive locks from read-only storage operations
|
||||
|
||||
### Documentation
|
||||
- Update changelog and version for v1.10.2rc1
|
||||
|
||||
## Contributors
|
||||
|
||||
@greysonlalonde
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="Mar 13, 2026">
|
||||
## v1.10.2rc1
|
||||
|
||||
|
||||
@@ -4,6 +4,25 @@ description: "CrewAI의 제품 업데이트, 개선 사항 및 버그 수정"
|
||||
icon: "clock"
|
||||
mode: "wide"
|
||||
---
|
||||
<Update label="2026년 3월 14일">
|
||||
## v1.10.2rc2
|
||||
|
||||
[GitHub 릴리스 보기](https://github.com/crewAIInc/crewAI/releases/tag/1.10.2rc2)
|
||||
|
||||
## 변경 사항
|
||||
|
||||
### 버그 수정
|
||||
- 읽기 전용 스토리지 작업에서 독점 잠금 제거
|
||||
|
||||
### 문서
|
||||
- v1.10.2rc1에 대한 변경 로그 및 버전 업데이트
|
||||
|
||||
## 기여자
|
||||
|
||||
@greysonlalonde
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="2026년 3월 13일">
|
||||
## v1.10.2rc1
|
||||
|
||||
|
||||
@@ -4,6 +4,25 @@ description: "Atualizações de produto, melhorias e correções do CrewAI"
|
||||
icon: "clock"
|
||||
mode: "wide"
|
||||
---
|
||||
<Update label="14 mar 2026">
|
||||
## v1.10.2rc2
|
||||
|
||||
[Ver release no GitHub](https://github.com/crewAIInc/crewAI/releases/tag/1.10.2rc2)
|
||||
|
||||
## O que Mudou
|
||||
|
||||
### Correções de Bugs
|
||||
- Remover bloqueios exclusivos de operações de armazenamento somente leitura
|
||||
|
||||
### Documentação
|
||||
- Atualizar changelog e versão para v1.10.2rc1
|
||||
|
||||
## Contribuidores
|
||||
|
||||
@greysonlalonde
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="13 mar 2026">
|
||||
## v1.10.2rc1
|
||||
|
||||
|
||||
@@ -152,4 +152,4 @@ __all__ = [
|
||||
"wrap_file_source",
|
||||
]
|
||||
|
||||
__version__ = "1.10.2rc1"
|
||||
__version__ = "1.10.2rc2"
|
||||
|
||||
@@ -11,7 +11,7 @@ dependencies = [
|
||||
"pytube~=15.0.0",
|
||||
"requests~=2.32.5",
|
||||
"docker~=7.1.0",
|
||||
"crewai==1.10.2rc1",
|
||||
"crewai==1.10.2rc2",
|
||||
"tiktoken~=0.8.0",
|
||||
"beautifulsoup4~=4.13.4",
|
||||
"python-docx~=1.2.0",
|
||||
|
||||
@@ -309,4 +309,4 @@ __all__ = [
|
||||
"ZapierActionTools",
|
||||
]
|
||||
|
||||
__version__ = "1.10.2rc1"
|
||||
__version__ = "1.10.2rc2"
|
||||
|
||||
@@ -46,13 +46,12 @@ class LanceDBAdapter(Adapter):
|
||||
|
||||
def query(self, question: str) -> str: # type: ignore[override]
|
||||
query = self.embedding_function([question])[0]
|
||||
with store_lock(self._lock_name):
|
||||
results = (
|
||||
self._table.search(query, vector_column_name=self.vector_column_name)
|
||||
.limit(self.top_k)
|
||||
.select([self.text_column_name])
|
||||
.to_list()
|
||||
)
|
||||
results = (
|
||||
self._table.search(query, vector_column_name=self.vector_column_name)
|
||||
.limit(self.top_k)
|
||||
.select([self.text_column_name])
|
||||
.to_list()
|
||||
)
|
||||
values = [result[self.text_column_name] for result in results]
|
||||
return "\n".join(values)
|
||||
|
||||
|
||||
@@ -173,13 +173,12 @@ class RAG(Adapter):
|
||||
try:
|
||||
question_embedding = self._embedding_service.embed_text(question)
|
||||
|
||||
with store_lock(self._lock_name):
|
||||
results = self._collection.query(
|
||||
query_embeddings=[question_embedding],
|
||||
n_results=self.top_k,
|
||||
where=where,
|
||||
include=["documents", "metadatas", "distances"],
|
||||
)
|
||||
results = self._collection.query(
|
||||
query_embeddings=[question_embedding],
|
||||
n_results=self.top_k,
|
||||
where=where,
|
||||
include=["documents", "metadatas", "distances"],
|
||||
)
|
||||
|
||||
if (
|
||||
not results
|
||||
|
||||
@@ -1,13 +1,27 @@
|
||||
# CodeInterpreterTool
|
||||
|
||||
## Description
|
||||
This tool is used to give the Agent the ability to run code (Python3) from the code generated by the Agent itself. The code is executed in a sandboxed environment, so it is safe to run any code.
|
||||
This tool is used to give the Agent the ability to run code (Python3) from the code generated by the Agent itself. The code is executed in a Docker container for secure isolation.
|
||||
|
||||
It is incredible useful since it allows the Agent to generate code, run it in the same environment, get the result and use it to make decisions.
|
||||
It is incredibly useful since it allows the Agent to generate code, run it in an isolated environment, get the result and use it to make decisions.
|
||||
|
||||
## ⚠️ Security Requirements
|
||||
|
||||
**Docker is REQUIRED** for safe code execution. The tool will refuse to execute code without Docker to prevent security vulnerabilities.
|
||||
|
||||
### Why Docker is Required
|
||||
|
||||
Previous versions included a "restricted sandbox" fallback when Docker was unavailable. This has been **removed** due to critical security vulnerabilities:
|
||||
|
||||
- The Python-based sandbox could be escaped via object introspection
|
||||
- Attackers could recover the original `__import__` function and access any module
|
||||
- This allowed arbitrary command execution on the host system
|
||||
|
||||
**Docker provides real process isolation** and is the only secure way to execute untrusted code.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Docker
|
||||
- **Docker (REQUIRED)** - Install from [docker.com](https://docs.docker.com/get-docker/)
|
||||
|
||||
## Installation
|
||||
Install the crewai_tools package
|
||||
@@ -17,7 +31,9 @@ pip install 'crewai[tools]'
|
||||
|
||||
## Example
|
||||
|
||||
Remember that when using this tool, the code must be generated by the Agent itself. The code must be a Python3 code. And it will take some time for the first time to run because it needs to build the Docker image.
|
||||
Remember that when using this tool, the code must be generated by the Agent itself. The code must be Python3 code. It will take some time the first time to run because it needs to build the Docker image.
|
||||
|
||||
### Basic Usage (Docker Container - Recommended)
|
||||
|
||||
```python
|
||||
from crewai_tools import CodeInterpreterTool
|
||||
@@ -28,7 +44,9 @@ Agent(
|
||||
)
|
||||
```
|
||||
|
||||
Or if you need to pass your own Dockerfile just do this
|
||||
### Custom Dockerfile
|
||||
|
||||
If you need to pass your own Dockerfile:
|
||||
|
||||
```python
|
||||
from crewai_tools import CodeInterpreterTool
|
||||
@@ -39,15 +57,39 @@ Agent(
|
||||
)
|
||||
```
|
||||
|
||||
If it is difficult to connect to docker daemon automatically (especially for macOS users), you can do this to setup docker host manually
|
||||
### Manual Docker Host Configuration
|
||||
|
||||
If it is difficult to connect to the Docker daemon automatically (especially for macOS users), you can set up the Docker host manually:
|
||||
|
||||
```python
|
||||
from crewai_tools import CodeInterpreterTool
|
||||
|
||||
Agent(
|
||||
...
|
||||
tools=[CodeInterpreterTool(user_docker_base_url="<Docker Host Base Url>",
|
||||
user_dockerfile_path="<Dockerfile_path>")],
|
||||
tools=[CodeInterpreterTool(
|
||||
user_docker_base_url="<Docker Host Base Url>",
|
||||
user_dockerfile_path="<Dockerfile_path>"
|
||||
)],
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
### Unsafe Mode (NOT RECOMMENDED)
|
||||
|
||||
If you absolutely cannot use Docker and **fully trust the code source**, you can use unsafe mode:
|
||||
|
||||
```python
|
||||
from crewai_tools import CodeInterpreterTool
|
||||
|
||||
# WARNING: Only use with fully trusted code!
|
||||
Agent(
|
||||
...
|
||||
tools=[CodeInterpreterTool(unsafe_mode=True)],
|
||||
)
|
||||
```
|
||||
|
||||
**⚠️ SECURITY WARNING:** `unsafe_mode=True` executes code directly on the host without any isolation. Only use this if:
|
||||
- You completely trust the code being executed
|
||||
- You understand the security risks
|
||||
- You cannot install Docker in your environment
|
||||
|
||||
For production use, **always use Docker** (the default mode).
|
||||
|
||||
@@ -50,11 +50,16 @@ class CodeInterpreterSchema(BaseModel):
|
||||
|
||||
|
||||
class SandboxPython:
|
||||
"""A restricted Python execution environment for running code safely.
|
||||
"""INSECURE: A restricted Python execution environment with known vulnerabilities.
|
||||
|
||||
This class provides methods to safely execute Python code by restricting access to
|
||||
potentially dangerous modules and built-in functions. It creates a sandboxed
|
||||
environment where harmful operations are blocked.
|
||||
WARNING: This class does NOT provide real security isolation and is vulnerable to
|
||||
sandbox escape attacks via Python object introspection. Attackers can recover the
|
||||
original __import__ function and bypass all restrictions.
|
||||
|
||||
DO NOT USE for untrusted code execution. Use Docker containers instead.
|
||||
|
||||
This class attempts to restrict access to dangerous modules and built-in functions
|
||||
but provides no real security boundary against a motivated attacker.
|
||||
"""
|
||||
|
||||
BLOCKED_MODULES: ClassVar[set[str]] = {
|
||||
@@ -299,8 +304,8 @@ class CodeInterpreterTool(BaseTool):
|
||||
def run_code_safety(self, code: str, libraries_used: list[str]) -> str:
|
||||
"""Runs code in the safest available environment.
|
||||
|
||||
Attempts to run code in Docker if available, falls back to a restricted
|
||||
sandbox if Docker is not available.
|
||||
Requires Docker to be available for secure code execution. Fails closed
|
||||
if Docker is not available to prevent sandbox escape vulnerabilities.
|
||||
|
||||
Args:
|
||||
code: The Python code to execute as a string.
|
||||
@@ -308,10 +313,24 @@ class CodeInterpreterTool(BaseTool):
|
||||
|
||||
Returns:
|
||||
The output of the executed code as a string.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If Docker is not available, as the restricted sandbox
|
||||
is vulnerable to escape attacks and should not be used
|
||||
for untrusted code execution.
|
||||
"""
|
||||
if self._check_docker_available():
|
||||
return self.run_code_in_docker(code, libraries_used)
|
||||
return self.run_code_in_restricted_sandbox(code)
|
||||
|
||||
error_msg = (
|
||||
"Docker is required for safe code execution but is not available. "
|
||||
"The restricted sandbox fallback has been removed due to security vulnerabilities "
|
||||
"that allow sandbox escape via Python object introspection. "
|
||||
"Please install Docker (https://docs.docker.com/get-docker/) or use unsafe_mode=True "
|
||||
"if you trust the code source and understand the security risks."
|
||||
)
|
||||
Printer.print(error_msg, color="bold_red")
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
def run_code_in_docker(self, code: str, libraries_used: list[str]) -> str:
|
||||
"""Runs Python code in a Docker container for safe isolation.
|
||||
@@ -342,10 +361,19 @@ class CodeInterpreterTool(BaseTool):
|
||||
|
||||
@staticmethod
|
||||
def run_code_in_restricted_sandbox(code: str) -> str:
|
||||
"""Runs Python code in a restricted sandbox environment.
|
||||
"""DEPRECATED AND INSECURE: Runs Python code in a restricted sandbox environment.
|
||||
|
||||
Executes the code with restricted access to potentially dangerous modules and
|
||||
built-in functions for basic safety when Docker is not available.
|
||||
WARNING: This method is vulnerable to sandbox escape attacks via Python object
|
||||
introspection and should NOT be used for untrusted code execution. It has been
|
||||
deprecated and is only kept for backward compatibility with trusted code.
|
||||
|
||||
The "restricted" environment can be bypassed by attackers who can:
|
||||
- Use object graph introspection to recover the original __import__ function
|
||||
- Access any Python module including os, subprocess, sys, etc.
|
||||
- Execute arbitrary commands on the host system
|
||||
|
||||
Use run_code_in_docker() for secure code execution, or run_code_unsafe()
|
||||
if you explicitly acknowledge the security risks.
|
||||
|
||||
Args:
|
||||
code: The Python code to execute as a string.
|
||||
@@ -354,7 +382,10 @@ class CodeInterpreterTool(BaseTool):
|
||||
The value of the 'result' variable from the executed code,
|
||||
or an error message if execution failed.
|
||||
"""
|
||||
Printer.print("Running code in restricted sandbox", color="yellow")
|
||||
Printer.print(
|
||||
"WARNING: Running code in INSECURE restricted sandbox (vulnerable to escape attacks)",
|
||||
color="bold_red"
|
||||
)
|
||||
exec_locals: dict[str, Any] = {}
|
||||
try:
|
||||
SandboxPython.exec(code=code, locals_=exec_locals)
|
||||
|
||||
@@ -76,24 +76,24 @@ print("This is line 2")"""
|
||||
)
|
||||
|
||||
|
||||
def test_restricted_sandbox_basic_code_execution(printer_mock, docker_unavailable_mock):
|
||||
"""Test basic code execution."""
|
||||
def test_docker_unavailable_raises_error(printer_mock, docker_unavailable_mock):
|
||||
"""Test that execution fails when Docker is unavailable in safe mode."""
|
||||
tool = CodeInterpreterTool()
|
||||
code = """
|
||||
result = 2 + 2
|
||||
print(result)
|
||||
"""
|
||||
result = tool.run(code=code, libraries_used=[])
|
||||
printer_mock.assert_called_with(
|
||||
"Running code in restricted sandbox", color="yellow"
|
||||
)
|
||||
assert result == 4
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
tool.run(code=code, libraries_used=[])
|
||||
|
||||
assert "Docker is required for safe code execution" in str(exc_info.value)
|
||||
assert "sandbox escape" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_restricted_sandbox_running_with_blocked_modules(
|
||||
printer_mock, docker_unavailable_mock
|
||||
):
|
||||
"""Test that restricted modules cannot be imported."""
|
||||
"""Test that restricted modules cannot be imported when using the deprecated sandbox directly."""
|
||||
tool = CodeInterpreterTool()
|
||||
restricted_modules = SandboxPython.BLOCKED_MODULES
|
||||
|
||||
@@ -102,18 +102,17 @@ def test_restricted_sandbox_running_with_blocked_modules(
|
||||
import {module}
|
||||
result = "Import succeeded"
|
||||
"""
|
||||
result = tool.run(code=code, libraries_used=[])
|
||||
printer_mock.assert_called_with(
|
||||
"Running code in restricted sandbox", color="yellow"
|
||||
)
|
||||
|
||||
# Note: run_code_in_restricted_sandbox is deprecated and insecure
|
||||
# This test verifies the old behavior but should not be used in production
|
||||
result = tool.run_code_in_restricted_sandbox(code)
|
||||
|
||||
assert f"An error occurred: Importing '{module}' is not allowed" in result
|
||||
|
||||
|
||||
def test_restricted_sandbox_running_with_blocked_builtins(
|
||||
printer_mock, docker_unavailable_mock
|
||||
):
|
||||
"""Test that restricted builtins are not available."""
|
||||
"""Test that restricted builtins are not available when using the deprecated sandbox directly."""
|
||||
tool = CodeInterpreterTool()
|
||||
restricted_builtins = SandboxPython.UNSAFE_BUILTINS
|
||||
|
||||
@@ -122,25 +121,23 @@ def test_restricted_sandbox_running_with_blocked_builtins(
|
||||
{builtin}("test")
|
||||
result = "Builtin available"
|
||||
"""
|
||||
result = tool.run(code=code, libraries_used=[])
|
||||
printer_mock.assert_called_with(
|
||||
"Running code in restricted sandbox", color="yellow"
|
||||
)
|
||||
# Note: run_code_in_restricted_sandbox is deprecated and insecure
|
||||
# This test verifies the old behavior but should not be used in production
|
||||
result = tool.run_code_in_restricted_sandbox(code)
|
||||
assert f"An error occurred: name '{builtin}' is not defined" in result
|
||||
|
||||
|
||||
def test_restricted_sandbox_running_with_no_result_variable(
|
||||
printer_mock, docker_unavailable_mock
|
||||
):
|
||||
"""Test behavior when no result variable is set."""
|
||||
"""Test behavior when no result variable is set in deprecated sandbox."""
|
||||
tool = CodeInterpreterTool()
|
||||
code = """
|
||||
x = 10
|
||||
"""
|
||||
result = tool.run(code=code, libraries_used=[])
|
||||
printer_mock.assert_called_with(
|
||||
"Running code in restricted sandbox", color="yellow"
|
||||
)
|
||||
# Note: run_code_in_restricted_sandbox is deprecated and insecure
|
||||
# This test verifies the old behavior but should not be used in production
|
||||
result = tool.run_code_in_restricted_sandbox(code)
|
||||
assert result == "No result variable found."
|
||||
|
||||
|
||||
@@ -172,3 +169,40 @@ result = eval("5/1")
|
||||
"WARNING: Running code in unsafe mode", color="bold_magenta"
|
||||
)
|
||||
assert 5.0 == result
|
||||
|
||||
|
||||
def test_sandbox_escape_vulnerability_demonstration(printer_mock):
|
||||
"""Demonstrate that the restricted sandbox is vulnerable to escape attacks.
|
||||
|
||||
This test shows that an attacker can use Python object introspection to bypass
|
||||
the restricted sandbox and access blocked modules like 'os'. This is why the
|
||||
sandbox should never be used for untrusted code execution.
|
||||
|
||||
NOTE: This test uses the deprecated run_code_in_restricted_sandbox directly
|
||||
to demonstrate the vulnerability. In production, Docker is now required.
|
||||
"""
|
||||
tool = CodeInterpreterTool()
|
||||
|
||||
# Classic Python sandbox escape via object introspection
|
||||
escape_code = """
|
||||
# Recover the real __import__ function via object introspection
|
||||
for cls in ().__class__.__bases__[0].__subclasses__():
|
||||
if cls.__name__ == 'catch_warnings':
|
||||
# Get the real builtins module
|
||||
real_builtins = cls()._module.__builtins__
|
||||
real_import = real_builtins['__import__']
|
||||
# Now we can import os and execute commands
|
||||
os = real_import('os')
|
||||
# Demonstrate we have escaped the sandbox
|
||||
result = "SANDBOX_ESCAPED" if hasattr(os, 'system') else "FAILED"
|
||||
break
|
||||
"""
|
||||
|
||||
# The deprecated sandbox is vulnerable to this attack
|
||||
result = tool.run_code_in_restricted_sandbox(escape_code)
|
||||
|
||||
# This demonstrates the vulnerability - the attacker can escape
|
||||
assert result == "SANDBOX_ESCAPED", (
|
||||
"The restricted sandbox was bypassed via object introspection. "
|
||||
"This is why Docker is now required for safe code execution."
|
||||
)
|
||||
|
||||
@@ -53,7 +53,7 @@ Repository = "https://github.com/crewAIInc/crewAI"
|
||||
|
||||
[project.optional-dependencies]
|
||||
tools = [
|
||||
"crewai-tools==1.10.2rc1",
|
||||
"crewai-tools==1.10.2rc2",
|
||||
]
|
||||
embeddings = [
|
||||
"tiktoken~=0.8.0"
|
||||
|
||||
@@ -41,7 +41,7 @@ def _suppress_pydantic_deprecation_warnings() -> None:
|
||||
|
||||
_suppress_pydantic_deprecation_warnings()
|
||||
|
||||
__version__ = "1.10.2rc1"
|
||||
__version__ = "1.10.2rc2"
|
||||
_telemetry_submitted = False
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ description = "{{name}} using crewAI"
|
||||
authors = [{ name = "Your Name", email = "you@example.com" }]
|
||||
requires-python = ">=3.10,<3.14"
|
||||
dependencies = [
|
||||
"crewai[tools]==1.10.2rc1"
|
||||
"crewai[tools]==1.10.2rc2"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -5,7 +5,7 @@ description = "{{name}} using crewAI"
|
||||
authors = [{ name = "Your Name", email = "you@example.com" }]
|
||||
requires-python = ">=3.10,<3.14"
|
||||
dependencies = [
|
||||
"crewai[tools]==1.10.2rc1"
|
||||
"crewai[tools]==1.10.2rc2"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -5,7 +5,7 @@ description = "Power up your crews with {{folder_name}}"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
dependencies = [
|
||||
"crewai[tools]==1.10.2rc1"
|
||||
"crewai[tools]==1.10.2rc2"
|
||||
]
|
||||
|
||||
[tool.crewai]
|
||||
|
||||
@@ -2716,7 +2716,9 @@ class Flow(Generic[T], metaclass=FlowMeta):
|
||||
from crewai.flow.async_feedback.types import HumanFeedbackPending
|
||||
|
||||
if not isinstance(e, HumanFeedbackPending):
|
||||
logger.error(f"Error executing listener {listener_name}: {e}")
|
||||
if not getattr(e, "_flow_listener_logged", False):
|
||||
logger.error(f"Error executing listener {listener_name}: {e}")
|
||||
e._flow_listener_logged = True # type: ignore[attr-defined]
|
||||
raise
|
||||
|
||||
# ── User Input (self.ask) ────────────────────────────────────────
|
||||
|
||||
@@ -13,6 +13,7 @@ from __future__ import annotations
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
import contextvars
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import math
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -29,6 +30,8 @@ from crewai.memory.analyze import (
|
||||
from crewai.memory.types import MemoryConfig, MemoryRecord, embed_texts
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State models
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -188,7 +191,15 @@ class EncodingFlow(Flow[EncodingState]):
|
||||
|
||||
if len(active) == 1:
|
||||
_, item = active[0]
|
||||
raw = _search_one(item)
|
||||
try:
|
||||
raw = _search_one(item)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Storage search failed in parallel_find_similar, "
|
||||
"treating item as new",
|
||||
exc_info=True,
|
||||
)
|
||||
raw = []
|
||||
item.similar_records = [r for r, _ in raw]
|
||||
item.top_similarity = float(raw[0][1]) if raw else 0.0
|
||||
else:
|
||||
@@ -202,7 +213,15 @@ class EncodingFlow(Flow[EncodingState]):
|
||||
for i, item in active
|
||||
]
|
||||
for _, item, future in futures:
|
||||
raw = future.result()
|
||||
try:
|
||||
raw = future.result()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Storage search failed in parallel_find_similar, "
|
||||
"treating item as new",
|
||||
exc_info=True,
|
||||
)
|
||||
raw = []
|
||||
item.similar_records = [r for r, _ in raw]
|
||||
item.top_similarity = float(raw[0][1]) if raw else 0.0
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from __future__ import annotations
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import contextvars
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -30,6 +31,9 @@ from crewai.memory.types import (
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RecallState(BaseModel):
|
||||
"""State for the recall flow."""
|
||||
|
||||
@@ -125,7 +129,14 @@ class RecallFlow(Flow[RecallState]):
|
||||
|
||||
if len(tasks) <= 1:
|
||||
for emb, sc in tasks:
|
||||
scope, results = _search_one(emb, sc)
|
||||
try:
|
||||
scope, results = _search_one(emb, sc)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Storage search failed in recall flow, skipping scope",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
if results:
|
||||
top_composite, _ = compute_composite_score(
|
||||
results[0][0], results[0][1], self._config
|
||||
@@ -147,7 +158,14 @@ class RecallFlow(Flow[RecallState]):
|
||||
for emb, sc in tasks
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
scope, results = future.result()
|
||||
try:
|
||||
scope, results = future.result()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Storage search failed in recall flow, skipping scope",
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
if results:
|
||||
top_composite, _ = compute_composite_score(
|
||||
results[0][0], results[0][1], self._config
|
||||
@@ -246,13 +264,17 @@ class RecallFlow(Flow[RecallState]):
|
||||
if analysis and analysis.suggested_scopes:
|
||||
candidates = [s for s in analysis.suggested_scopes if s]
|
||||
else:
|
||||
candidates = self._storage.list_scopes(scope_prefix)
|
||||
try:
|
||||
candidates = self._storage.list_scopes(scope_prefix)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Storage list_scopes failed in filter_and_chunk, "
|
||||
"falling back to scope prefix",
|
||||
exc_info=True,
|
||||
)
|
||||
candidates = []
|
||||
if not candidates:
|
||||
info = self._storage.get_scope_info(scope_prefix)
|
||||
if info.record_count > 0:
|
||||
candidates = [scope_prefix]
|
||||
else:
|
||||
candidates = [scope_prefix]
|
||||
candidates = [scope_prefix]
|
||||
self.state.candidate_scopes = candidates[:20]
|
||||
return self.state.candidate_scopes
|
||||
|
||||
|
||||
@@ -350,12 +350,11 @@ class LanceDBStorage:
|
||||
"""Return a single record by ID, or None if not found."""
|
||||
if self._table is None:
|
||||
return None
|
||||
with store_lock(self._lock_name):
|
||||
safe_id = str(record_id).replace("'", "''")
|
||||
rows = self._table.search().where(f"id = '{safe_id}'").limit(1).to_list()
|
||||
if not rows:
|
||||
return None
|
||||
return self._row_to_record(rows[0])
|
||||
safe_id = str(record_id).replace("'", "''")
|
||||
rows = self._table.search().where(f"id = '{safe_id}'").limit(1).to_list()
|
||||
if not rows:
|
||||
return None
|
||||
return self._row_to_record(rows[0])
|
||||
|
||||
def search(
|
||||
self,
|
||||
@@ -368,15 +367,14 @@ class LanceDBStorage:
|
||||
) -> list[tuple[MemoryRecord, float]]:
|
||||
if self._table is None:
|
||||
return []
|
||||
with store_lock(self._lock_name):
|
||||
query = self._table.search(query_embedding)
|
||||
if scope_prefix is not None and scope_prefix.strip("/"):
|
||||
prefix = scope_prefix.rstrip("/")
|
||||
like_val = prefix + "%"
|
||||
query = query.where(f"scope LIKE '{like_val}'")
|
||||
results = query.limit(
|
||||
limit * 3 if (categories or metadata_filter) else limit
|
||||
).to_list()
|
||||
query = self._table.search(query_embedding)
|
||||
if scope_prefix is not None and scope_prefix.strip("/"):
|
||||
prefix = scope_prefix.rstrip("/")
|
||||
like_val = prefix + "%"
|
||||
query = query.where(f"scope LIKE '{like_val}'")
|
||||
results = query.limit(
|
||||
limit * 3 if (categories or metadata_filter) else limit
|
||||
).to_list()
|
||||
out: list[tuple[MemoryRecord, float]] = []
|
||||
for row in results:
|
||||
record = self._row_to_record(row)
|
||||
@@ -460,8 +458,6 @@ class LanceDBStorage:
|
||||
Uses a full table scan (no vector query) so the limit is applied after
|
||||
the scope filter, not to ANN candidates before filtering.
|
||||
|
||||
Caller must hold ``store_lock(self._lock_name)``.
|
||||
|
||||
Args:
|
||||
scope_prefix: Optional scope path prefix to filter by.
|
||||
limit: Maximum number of rows to return (applied after filtering).
|
||||
@@ -492,8 +488,7 @@ class LanceDBStorage:
|
||||
Returns:
|
||||
List of MemoryRecord, ordered by created_at descending.
|
||||
"""
|
||||
with store_lock(self._lock_name):
|
||||
rows = self._scan_rows(scope_prefix, limit=limit + offset)
|
||||
rows = self._scan_rows(scope_prefix, limit=limit + offset)
|
||||
records = [self._row_to_record(r) for r in rows]
|
||||
records.sort(key=lambda r: r.created_at, reverse=True)
|
||||
return records[offset : offset + limit]
|
||||
@@ -503,11 +498,10 @@ class LanceDBStorage:
|
||||
prefix = scope if scope != "/" else ""
|
||||
if prefix and not prefix.startswith("/"):
|
||||
prefix = "/" + prefix
|
||||
with store_lock(self._lock_name):
|
||||
rows = self._scan_rows(
|
||||
prefix or None,
|
||||
columns=["scope", "categories_str", "created_at"],
|
||||
)
|
||||
rows = self._scan_rows(
|
||||
prefix or None,
|
||||
columns=["scope", "categories_str", "created_at"],
|
||||
)
|
||||
if not rows:
|
||||
return ScopeInfo(
|
||||
path=scope or "/",
|
||||
@@ -558,8 +552,7 @@ class LanceDBStorage:
|
||||
def list_scopes(self, parent: str = "/") -> list[str]:
|
||||
parent = parent.rstrip("/") or ""
|
||||
prefix = (parent + "/") if parent else "/"
|
||||
with store_lock(self._lock_name):
|
||||
rows = self._scan_rows(prefix if prefix != "/" else None, columns=["scope"])
|
||||
rows = self._scan_rows(prefix if prefix != "/" else None, columns=["scope"])
|
||||
children: set[str] = set()
|
||||
for row in rows:
|
||||
sc = str(row.get("scope", ""))
|
||||
@@ -571,8 +564,7 @@ class LanceDBStorage:
|
||||
return sorted(children)
|
||||
|
||||
def list_categories(self, scope_prefix: str | None = None) -> dict[str, int]:
|
||||
with store_lock(self._lock_name):
|
||||
rows = self._scan_rows(scope_prefix, columns=["categories_str"])
|
||||
rows = self._scan_rows(scope_prefix, columns=["categories_str"])
|
||||
counts: dict[str, int] = {}
|
||||
for row in rows:
|
||||
cat_str = row.get("categories_str") or "[]"
|
||||
@@ -588,8 +580,7 @@ class LanceDBStorage:
|
||||
if self._table is None:
|
||||
return 0
|
||||
if scope_prefix is None or scope_prefix.strip("/") == "":
|
||||
with store_lock(self._lock_name):
|
||||
return int(self._table.count_rows())
|
||||
return int(self._table.count_rows())
|
||||
info = self.get_scope_info(scope_prefix)
|
||||
return info.record_count
|
||||
|
||||
|
||||
@@ -446,30 +446,29 @@ class ChromaDBClient(BaseClient):
|
||||
|
||||
params = _extract_search_params(kwargs)
|
||||
|
||||
with self._locked():
|
||||
collection = self.client.get_or_create_collection(
|
||||
name=_sanitize_collection_name(params.collection_name),
|
||||
embedding_function=self.embedding_function,
|
||||
collection = self.client.get_or_create_collection(
|
||||
name=_sanitize_collection_name(params.collection_name),
|
||||
embedding_function=self.embedding_function,
|
||||
)
|
||||
|
||||
where = params.where if params.where is not None else params.metadata_filter
|
||||
|
||||
with suppress_logging(
|
||||
"chromadb.segment.impl.vector.local_persistent_hnsw", logging.ERROR
|
||||
):
|
||||
results: QueryResult = collection.query(
|
||||
query_texts=[params.query],
|
||||
n_results=params.limit,
|
||||
where=where,
|
||||
where_document=params.where_document,
|
||||
include=params.include,
|
||||
)
|
||||
|
||||
where = params.where if params.where is not None else params.metadata_filter
|
||||
|
||||
with suppress_logging(
|
||||
"chromadb.segment.impl.vector.local_persistent_hnsw", logging.ERROR
|
||||
):
|
||||
results: QueryResult = collection.query(
|
||||
query_texts=[params.query],
|
||||
n_results=params.limit,
|
||||
where=where,
|
||||
where_document=params.where_document,
|
||||
include=params.include,
|
||||
)
|
||||
|
||||
return _process_query_results(
|
||||
collection=collection,
|
||||
results=results,
|
||||
params=params,
|
||||
)
|
||||
return _process_query_results(
|
||||
collection=collection,
|
||||
results=results,
|
||||
params=params,
|
||||
)
|
||||
|
||||
async def asearch(
|
||||
self, **kwargs: Unpack[ChromaDBCollectionSearchParams]
|
||||
@@ -510,30 +509,29 @@ class ChromaDBClient(BaseClient):
|
||||
|
||||
params = _extract_search_params(kwargs)
|
||||
|
||||
async with self._alocked():
|
||||
collection = await self.client.get_or_create_collection(
|
||||
name=_sanitize_collection_name(params.collection_name),
|
||||
embedding_function=self.embedding_function,
|
||||
collection = await self.client.get_or_create_collection(
|
||||
name=_sanitize_collection_name(params.collection_name),
|
||||
embedding_function=self.embedding_function,
|
||||
)
|
||||
|
||||
where = params.where if params.where is not None else params.metadata_filter
|
||||
|
||||
with suppress_logging(
|
||||
"chromadb.segment.impl.vector.local_persistent_hnsw", logging.ERROR
|
||||
):
|
||||
results: QueryResult = await collection.query(
|
||||
query_texts=[params.query],
|
||||
n_results=params.limit,
|
||||
where=where,
|
||||
where_document=params.where_document,
|
||||
include=params.include,
|
||||
)
|
||||
|
||||
where = params.where if params.where is not None else params.metadata_filter
|
||||
|
||||
with suppress_logging(
|
||||
"chromadb.segment.impl.vector.local_persistent_hnsw", logging.ERROR
|
||||
):
|
||||
results: QueryResult = await collection.query(
|
||||
query_texts=[params.query],
|
||||
n_results=params.limit,
|
||||
where=where,
|
||||
where_document=params.where_document,
|
||||
include=params.include,
|
||||
)
|
||||
|
||||
return _process_query_results(
|
||||
collection=collection,
|
||||
results=results,
|
||||
params=params,
|
||||
)
|
||||
return _process_query_results(
|
||||
collection=collection,
|
||||
results=results,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def delete_collection(self, **kwargs: Unpack[BaseCollectionParams]) -> None:
|
||||
"""Delete a collection and all its data.
|
||||
|
||||
@@ -10,17 +10,21 @@ from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from functools import lru_cache
|
||||
from hashlib import md5
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import portalocker
|
||||
import portalocker.exceptions
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import redis
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_REDIS_URL: str | None = os.environ.get("REDIS_URL")
|
||||
|
||||
_DEFAULT_TIMEOUT: Final[int] = 120
|
||||
@@ -57,5 +61,16 @@ def lock(name: str, *, timeout: float = _DEFAULT_TIMEOUT) -> Iterator[None]:
|
||||
else:
|
||||
lock_dir = tempfile.gettempdir()
|
||||
lock_path = os.path.join(lock_dir, f"{channel}.lock")
|
||||
with portalocker.Lock(lock_path, timeout=timeout):
|
||||
try:
|
||||
pl = portalocker.Lock(lock_path, timeout=timeout)
|
||||
pl.acquire()
|
||||
except portalocker.exceptions.BaseLockException as exc:
|
||||
raise portalocker.exceptions.LockException(
|
||||
f"Failed to acquire lock '{name}' at {lock_path} "
|
||||
f"(timeout={timeout}s). This commonly occurs in "
|
||||
f"multi-process environments. "
|
||||
) from exc
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pl.release() # type: ignore[no-untyped-call]
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""CrewAI development tools."""
|
||||
|
||||
__version__ = "1.10.2rc1"
|
||||
__version__ = "1.10.2rc2"
|
||||
|
||||
Reference in New Issue
Block a user