mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-26 08:38:15 +00:00
Compare commits
2 Commits
devin/1769
...
devin/1765
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ec39a70a9 | ||
|
|
2a62ac0a23 |
@@ -61,6 +61,7 @@ openpyxl = [
|
|||||||
"openpyxl~=3.1.5",
|
"openpyxl~=3.1.5",
|
||||||
]
|
]
|
||||||
mem0 = ["mem0ai~=0.1.94"]
|
mem0 = ["mem0ai~=0.1.94"]
|
||||||
|
openmemory = ["openmemory-py>=1.0.0"]
|
||||||
docling = [
|
docling = [
|
||||||
"docling~=2.63.0",
|
"docling~=2.63.0",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from crewai.rag.embeddings.types import ProviderSpec
|
|||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from crewai.memory.storage.mem0_storage import Mem0Storage
|
from crewai.memory.storage.mem0_storage import Mem0Storage
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
|
||||||
class ExternalMemory(Memory):
|
class ExternalMemory(Memory):
|
||||||
@@ -32,10 +33,19 @@ class ExternalMemory(Memory):
|
|||||||
|
|
||||||
return Mem0Storage(type="external", crew=crew, config=config) # type: ignore[no-untyped-call]
|
return Mem0Storage(type="external", crew=crew, config=config) # type: ignore[no-untyped-call]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _configure_openmemory(
|
||||||
|
crew: Any, config: dict[str, Any]
|
||||||
|
) -> OpenMemoryStorage:
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
return OpenMemoryStorage(type="external", crew=crew, config=config)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def external_supported_storages() -> dict[str, Any]:
|
def external_supported_storages() -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"mem0": ExternalMemory._configure_mem0,
|
"mem0": ExternalMemory._configure_mem0,
|
||||||
|
"openmemory": ExternalMemory._configure_openmemory,
|
||||||
}
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
158
lib/crewai/src/crewai/memory/storage/openmemory_storage.py
Normal file
158
lib/crewai/src/crewai/memory/storage/openmemory_storage.py
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from crewai.memory.storage.interface import Storage
|
||||||
|
from crewai.utilities.paths import get_crewai_storage_dir
|
||||||
|
|
||||||
|
|
||||||
|
class OpenMemoryStorage(Storage):
|
||||||
|
"""
|
||||||
|
Extends Storage to handle embedding and searching using OpenMemory.
|
||||||
|
|
||||||
|
OpenMemory is a local-first persistent memory engine for AI applications.
|
||||||
|
It provides cognitive memory capabilities without requiring external vector databases.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
type: str,
|
||||||
|
crew: Any | None = None,
|
||||||
|
config: dict[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__()
|
||||||
|
|
||||||
|
self._validate_type(type)
|
||||||
|
self.memory_type = type
|
||||||
|
self.crew = crew
|
||||||
|
self.config = config or {}
|
||||||
|
|
||||||
|
self._extract_config_values()
|
||||||
|
self._initialize_memory()
|
||||||
|
|
||||||
|
def _validate_type(self, type: str) -> None:
|
||||||
|
supported_types = {"short_term", "long_term", "entities", "external"}
|
||||||
|
if type not in supported_types:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid type '{type}' for OpenMemoryStorage. "
|
||||||
|
f"Must be one of: {', '.join(supported_types)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _extract_config_values(self) -> None:
|
||||||
|
self.user_id = self.config.get("user_id")
|
||||||
|
self.path = self.config.get("path")
|
||||||
|
self.tier = self.config.get("tier", "fast")
|
||||||
|
self.embeddings = self.config.get(
|
||||||
|
"embeddings", {"provider": "synthetic"}
|
||||||
|
)
|
||||||
|
|
||||||
|
if not self.path:
|
||||||
|
storage_dir = get_crewai_storage_dir()
|
||||||
|
self.path = os.path.join(
|
||||||
|
storage_dir, f"openmemory_{self.memory_type}.sqlite"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _initialize_memory(self) -> None:
|
||||||
|
try:
|
||||||
|
from openmemory import (
|
||||||
|
OpenMemory, # type: ignore[import-untyped,import-not-found]
|
||||||
|
)
|
||||||
|
except ImportError as e:
|
||||||
|
raise ImportError(
|
||||||
|
"OpenMemory is not installed. "
|
||||||
|
"Please install it with `pip install openmemory-py`."
|
||||||
|
) from e
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(self.path), exist_ok=True)
|
||||||
|
|
||||||
|
self.memory = OpenMemory(
|
||||||
|
mode="local",
|
||||||
|
path=self.path,
|
||||||
|
tier=self.tier,
|
||||||
|
embeddings=self.embeddings,
|
||||||
|
)
|
||||||
|
|
||||||
|
def save(self, value: Any, metadata: dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Save a memory item to OpenMemory.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value: The content to save.
|
||||||
|
metadata: Additional metadata to associate with the memory.
|
||||||
|
"""
|
||||||
|
params: dict[str, Any] = {}
|
||||||
|
|
||||||
|
if self.user_id:
|
||||||
|
params["userId"] = self.user_id
|
||||||
|
|
||||||
|
tags = metadata.pop("tags", None)
|
||||||
|
if tags:
|
||||||
|
params["tags"] = tags
|
||||||
|
|
||||||
|
params["metadata"] = {
|
||||||
|
"type": self.memory_type,
|
||||||
|
**metadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.memory.add(str(value), **params)
|
||||||
|
|
||||||
|
def search(
|
||||||
|
self, query: str, limit: int = 5, score_threshold: float = 0.6
|
||||||
|
) -> list[Any]:
|
||||||
|
"""
|
||||||
|
Search for relevant memories in OpenMemory.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: The search query.
|
||||||
|
limit: Maximum number of results to return.
|
||||||
|
score_threshold: Minimum similarity score for results.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of matching memory entries with 'content' field.
|
||||||
|
"""
|
||||||
|
params: dict[str, Any] = {
|
||||||
|
"k": limit,
|
||||||
|
}
|
||||||
|
|
||||||
|
filters: dict[str, Any] = {}
|
||||||
|
if self.user_id:
|
||||||
|
filters["user_id"] = self.user_id
|
||||||
|
|
||||||
|
if filters:
|
||||||
|
params["filters"] = filters
|
||||||
|
|
||||||
|
results = self.memory.query(query, **params)
|
||||||
|
|
||||||
|
normalized_results = []
|
||||||
|
if isinstance(results, list):
|
||||||
|
for result in results:
|
||||||
|
if isinstance(result, dict):
|
||||||
|
normalized = dict(result)
|
||||||
|
if "content" not in normalized and "memory" in normalized:
|
||||||
|
normalized["content"] = normalized["memory"]
|
||||||
|
elif "content" not in normalized:
|
||||||
|
normalized["content"] = str(result)
|
||||||
|
|
||||||
|
score = normalized.get("score", 1.0)
|
||||||
|
if score >= score_threshold:
|
||||||
|
normalized_results.append(normalized)
|
||||||
|
else:
|
||||||
|
normalized_results.append({"content": str(result)})
|
||||||
|
|
||||||
|
return normalized_results
|
||||||
|
|
||||||
|
def reset(self) -> None:
|
||||||
|
"""
|
||||||
|
Reset the OpenMemory storage by closing and deleting the database file.
|
||||||
|
"""
|
||||||
|
if hasattr(self, "memory") and self.memory:
|
||||||
|
try:
|
||||||
|
self.memory.close()
|
||||||
|
except Exception:
|
||||||
|
self.memory = None
|
||||||
|
|
||||||
|
if self.path and os.path.exists(self.path):
|
||||||
|
os.remove(self.path)
|
||||||
|
|
||||||
|
self._initialize_memory()
|
||||||
257
lib/crewai/tests/storage/test_openmemory_storage.py
Normal file
257
lib/crewai/tests/storage/test_openmemory_storage.py
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class MockOpenMemory:
|
||||||
|
def __init__(self, mode=None, path=None, tier=None, embeddings=None):
|
||||||
|
self.mode = mode
|
||||||
|
self.path = path
|
||||||
|
self.tier = tier
|
||||||
|
self.embeddings = embeddings
|
||||||
|
self._memories: list[dict] = []
|
||||||
|
|
||||||
|
def add(self, content, userId=None, tags=None, metadata=None, **kwargs):
|
||||||
|
memory_id = f"mem_{len(self._memories)}"
|
||||||
|
self._memories.append(
|
||||||
|
{
|
||||||
|
"id": memory_id,
|
||||||
|
"content": content,
|
||||||
|
"userId": userId,
|
||||||
|
"tags": tags,
|
||||||
|
"metadata": metadata,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {"id": memory_id, "primarySector": "semantic", "sectors": ["semantic"]}
|
||||||
|
|
||||||
|
def query(self, query, k=10, filters=None, **kwargs):
|
||||||
|
results = []
|
||||||
|
for mem in self._memories:
|
||||||
|
if query.lower() in mem["content"].lower():
|
||||||
|
results.append(
|
||||||
|
{
|
||||||
|
"content": mem["content"],
|
||||||
|
"score": 0.9,
|
||||||
|
"metadata": mem["metadata"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return results[:k]
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class MockCrew:
|
||||||
|
def __init__(self):
|
||||||
|
self.agents = [MagicMock(role="Test Agent")]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_openmemory():
|
||||||
|
return MockOpenMemory
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_storage_path():
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
yield os.path.join(tmpdir, "test_memory.sqlite")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def openmemory_storage_with_mock(mock_openmemory, temp_storage_path):
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", mock_openmemory
|
||||||
|
):
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"path": temp_storage_path,
|
||||||
|
"tier": "fast",
|
||||||
|
"embeddings": {"provider": "synthetic"},
|
||||||
|
"user_id": "test_user",
|
||||||
|
}
|
||||||
|
crew = MockCrew()
|
||||||
|
storage = OpenMemoryStorage(type="external", crew=crew, config=config)
|
||||||
|
return storage
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_initialization(openmemory_storage_with_mock):
|
||||||
|
storage = openmemory_storage_with_mock
|
||||||
|
assert storage.memory_type == "external"
|
||||||
|
assert storage.tier == "fast"
|
||||||
|
assert storage.user_id == "test_user"
|
||||||
|
assert storage.embeddings == {"provider": "synthetic"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_invalid_type():
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Invalid type"):
|
||||||
|
OpenMemoryStorage(type="invalid_type", config={"path": "/tmp/test.sqlite"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_save(openmemory_storage_with_mock):
|
||||||
|
storage = openmemory_storage_with_mock
|
||||||
|
|
||||||
|
test_value = "This is a test memory about AI agents"
|
||||||
|
test_metadata = {"description": "Test description", "agent": "Test Agent"}
|
||||||
|
|
||||||
|
storage.save(test_value, test_metadata)
|
||||||
|
|
||||||
|
assert len(storage.memory._memories) == 1
|
||||||
|
saved_memory = storage.memory._memories[0]
|
||||||
|
assert saved_memory["content"] == test_value
|
||||||
|
assert saved_memory["userId"] == "test_user"
|
||||||
|
assert saved_memory["metadata"]["type"] == "external"
|
||||||
|
assert saved_memory["metadata"]["description"] == "Test description"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_save_with_tags(openmemory_storage_with_mock):
|
||||||
|
storage = openmemory_storage_with_mock
|
||||||
|
|
||||||
|
test_value = "Memory with tags"
|
||||||
|
test_metadata = {"tags": ["tag1", "tag2"], "description": "Tagged memory"}
|
||||||
|
|
||||||
|
storage.save(test_value, test_metadata)
|
||||||
|
|
||||||
|
saved_memory = storage.memory._memories[0]
|
||||||
|
assert saved_memory["tags"] == ["tag1", "tag2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_search(openmemory_storage_with_mock):
|
||||||
|
storage = openmemory_storage_with_mock
|
||||||
|
|
||||||
|
storage.save("Memory about Python programming", {"description": "Python"})
|
||||||
|
storage.save("Memory about JavaScript", {"description": "JavaScript"})
|
||||||
|
storage.save("Memory about Python frameworks", {"description": "Frameworks"})
|
||||||
|
|
||||||
|
results = storage.search("Python", limit=5, score_threshold=0.5)
|
||||||
|
|
||||||
|
assert len(results) == 2
|
||||||
|
assert all("content" in r for r in results)
|
||||||
|
assert any("Python programming" in r["content"] for r in results)
|
||||||
|
assert any("Python frameworks" in r["content"] for r in results)
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_search_with_score_threshold(openmemory_storage_with_mock):
|
||||||
|
storage = openmemory_storage_with_mock
|
||||||
|
|
||||||
|
storage.save("Test memory", {"description": "Test"})
|
||||||
|
|
||||||
|
results = storage.search("Test", limit=5, score_threshold=0.95)
|
||||||
|
assert len(results) == 0
|
||||||
|
|
||||||
|
results = storage.search("Test", limit=5, score_threshold=0.5)
|
||||||
|
assert len(results) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_reset(temp_storage_path):
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"path": temp_storage_path,
|
||||||
|
"tier": "fast",
|
||||||
|
"embeddings": {"provider": "synthetic"},
|
||||||
|
}
|
||||||
|
storage = OpenMemoryStorage(type="external", config=config)
|
||||||
|
|
||||||
|
storage.save("Test memory", {"description": "Test"})
|
||||||
|
assert len(storage.memory._memories) == 1
|
||||||
|
|
||||||
|
storage.reset()
|
||||||
|
|
||||||
|
assert len(storage.memory._memories) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_default_path():
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"tier": "fast",
|
||||||
|
"embeddings": {"provider": "synthetic"},
|
||||||
|
}
|
||||||
|
storage = OpenMemoryStorage(type="external", config=config)
|
||||||
|
|
||||||
|
assert storage.path is not None
|
||||||
|
assert "openmemory_external.sqlite" in storage.path
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_storage_different_memory_types():
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.storage.openmemory_storage import OpenMemoryStorage
|
||||||
|
|
||||||
|
for memory_type in ["short_term", "long_term", "entities", "external"]:
|
||||||
|
config = {
|
||||||
|
"tier": "fast",
|
||||||
|
"embeddings": {"provider": "synthetic"},
|
||||||
|
}
|
||||||
|
storage = OpenMemoryStorage(type=memory_type, config=config)
|
||||||
|
assert storage.memory_type == memory_type
|
||||||
|
assert f"openmemory_{memory_type}.sqlite" in storage.path
|
||||||
|
|
||||||
|
|
||||||
|
def test_openmemory_import_error():
|
||||||
|
import sys
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
original_modules = sys.modules.copy()
|
||||||
|
sys.modules["openmemory"] = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
try:
|
||||||
|
if "crewai.memory.storage.openmemory_storage" in sys.modules:
|
||||||
|
del sys.modules["crewai.memory.storage.openmemory_storage"]
|
||||||
|
|
||||||
|
with pytest.raises(ImportError, match="OpenMemory is not installed"):
|
||||||
|
from crewai.memory.storage import openmemory_storage
|
||||||
|
|
||||||
|
importlib.reload(openmemory_storage)
|
||||||
|
openmemory_storage.OpenMemoryStorage(
|
||||||
|
type="external", config={"path": "/tmp/test.sqlite"}
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
sys.modules.update(original_modules)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_memory_openmemory_provider():
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.external.external_memory import ExternalMemory
|
||||||
|
|
||||||
|
supported = ExternalMemory.external_supported_storages()
|
||||||
|
assert "openmemory" in supported
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_memory_create_openmemory_storage(temp_storage_path):
|
||||||
|
with patch(
|
||||||
|
"crewai.memory.storage.openmemory_storage.OpenMemory", MockOpenMemory
|
||||||
|
):
|
||||||
|
from crewai.memory.external.external_memory import ExternalMemory
|
||||||
|
|
||||||
|
crew = MockCrew()
|
||||||
|
embedder_config = {
|
||||||
|
"provider": "openmemory",
|
||||||
|
"config": {
|
||||||
|
"path": temp_storage_path,
|
||||||
|
"tier": "fast",
|
||||||
|
"embeddings": {"provider": "synthetic"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
storage = ExternalMemory.create_storage(crew, embedder_config)
|
||||||
|
|
||||||
|
assert storage is not None
|
||||||
|
assert storage.memory_type == "external"
|
||||||
Reference in New Issue
Block a user