Add support for External Memory (the future replacement for UserMemory) (#2510)

* fix: surfacing properly supported types by Mem0Storage

* feat: prepare Mem0Storage to accept config paramenter

We're planning to remove `memory_config` soon. This commit kindly prepare this storage to accept the config provided directly

* feat: add external memory

* fix: cleanup Mem0 warning while adding messages to the memory

* feat: support set the current crew in memory

This can be useful when a memory is initialized before the crew, but the crew might still be a very relevant attribute

* fix: allow to reset only an external_memory from crew

* test: add external memory test

* test: ensure the config takes precedence over memory_config when setting mem0

* fix: support to provide a custom storage to External Memory

* docs: add docs about external memory

* chore: add warning messages about the deprecation of UserMemory

* fix: fix typing check

---------

Co-authored-by: Lorenze Jay <63378463+lorenzejay@users.noreply.github.com>

tests/memory/external/test_external_memory.py

@@ -0,0 +1,180 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from mem0.memory.main import Memory
+
+from crewai.agent import Agent
+from crewai.crew import Crew, Process
+from crewai.memory.external.external_memory import ExternalMemory
+from crewai.memory.external.external_memory_item import ExternalMemoryItem
+from crewai.memory.storage.interface import Storage
+from crewai.task import Task
+
+
+@pytest.fixture
+def mock_mem0_memory():
+    mock_memory = MagicMock(spec=Memory)
+    return mock_memory
+
+
+@pytest.fixture
+def patch_configure_mem0(mock_mem0_memory):
+    with patch(
+        "crewai.memory.external.external_memory.ExternalMemory._configure_mem0",
+        return_value=mock_mem0_memory,
+    ) as mocked:
+        yield mocked
+
+
+@pytest.fixture
+def external_memory_with_mocked_config(patch_configure_mem0):
+    embedder_config = {"provider": "mem0"}
+    external_memory = ExternalMemory(embedder_config=embedder_config)
+    return external_memory
+
+
+@pytest.fixture
+def crew_with_external_memory(external_memory_with_mocked_config, patch_configure_mem0):
+    agent = Agent(
+        role="Researcher",
+        goal="Search relevant data and provide results",
+        backstory="You are a researcher at a leading tech think tank.",
+        tools=[],
+        verbose=True,
+    )
+
+    task = Task(
+        description="Perform a search on specific topics.",
+        expected_output="A list of relevant URLs based on the search query.",
+        agent=agent,
+    )
+
+    crew = Crew(
+        agents=[agent],
+        tasks=[task],
+        verbose=True,
+        process=Process.sequential,
+        memory=True,
+        external_memory=external_memory_with_mocked_config,
+    )
+
+    return crew
+
+
+def test_external_memory_initialization(external_memory_with_mocked_config):
+    assert external_memory_with_mocked_config is not None
+    assert isinstance(external_memory_with_mocked_config, ExternalMemory)
+
+
+def test_external_memory_save(external_memory_with_mocked_config):
+    memory_item = ExternalMemoryItem(
+        value="test value", metadata={"task": "test_task"}, agent="test_agent"
+    )
+
+    with patch.object(ExternalMemory, "save") as mock_save:
+        external_memory_with_mocked_config.save(
+            value=memory_item.value,
+            metadata=memory_item.metadata,
+            agent=memory_item.agent,
+        )
+
+        mock_save.assert_called_once_with(
+            value=memory_item.value,
+            metadata=memory_item.metadata,
+            agent=memory_item.agent,
+        )
+
+
+def test_external_memory_reset(external_memory_with_mocked_config):
+    with patch(
+        "crewai.memory.external.external_memory.ExternalMemory.reset"
+    ) as mock_reset:
+        external_memory_with_mocked_config.reset()
+        mock_reset.assert_called_once()
+
+
+def test_external_memory_supported_storages():
+    supported_storages = ExternalMemory.external_supported_storages()
+    assert "mem0" in supported_storages
+    assert callable(supported_storages["mem0"])
+
+
+def test_external_memory_create_storage_invalid_provider():
+    embedder_config = {"provider": "invalid_provider", "config": {}}
+
+    with pytest.raises(ValueError, match="Provider invalid_provider not supported"):
+        ExternalMemory.create_storage(None, embedder_config)
+
+
+def test_external_memory_create_storage_missing_provider():
+    embedder_config = {"config": {}}
+
+    with pytest.raises(
+        ValueError, match="embedder_config must include a 'provider' key"
+    ):
+        ExternalMemory.create_storage(None, embedder_config)
+
+
+def test_external_memory_create_storage_missing_config():
+    with pytest.raises(ValueError, match="embedder_config is required"):
+        ExternalMemory.create_storage(None, None)
+
+
+def test_crew_with_external_memory_initialization(crew_with_external_memory):
+    assert crew_with_external_memory._external_memory is not None
+    assert isinstance(crew_with_external_memory._external_memory, ExternalMemory)
+    assert crew_with_external_memory._external_memory.crew == crew_with_external_memory
+
+
+@pytest.mark.parametrize("mem_type", ["external", "all"])
+def test_crew_external_memory_reset(mem_type, crew_with_external_memory):
+    with patch(
+        "crewai.memory.external.external_memory.ExternalMemory.reset"
+    ) as mock_reset:
+        crew_with_external_memory.reset_memories(mem_type)
+        mock_reset.assert_called_once()
+
+
+@pytest.mark.parametrize("mem_method", ["search", "save"])
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_crew_external_memory_save(mem_method, crew_with_external_memory):
+    with patch(
+        f"crewai.memory.external.external_memory.ExternalMemory.{mem_method}"
+    ) as mock_method:
+        crew_with_external_memory.kickoff()
+        assert mock_method.call_count > 0
+
+
+def test_external_memory_custom_storage(crew_with_external_memory):
+    class CustomStorage(Storage):
+        def __init__(self):
+            self.memories = []
+
+        def save(self, value, metadata=None, agent=None):
+            self.memories.append({"value": value, "metadata": metadata, "agent": agent})
+
+        def search(self, query, limit=10, score_threshold=0.5):
+            return self.memories
+
+        def reset(self):
+            self.memories = []
+
+    custom_storage = CustomStorage()
+    external_memory = ExternalMemory(storage=custom_storage)
+
+    # by ensuring the crew is set, we can test that the storage is used
+    external_memory.set_crew(crew_with_external_memory)
+
+    test_value = "test value"
+    test_metadata = {"source": "test"}
+    test_agent = "test_agent"
+    external_memory.save(value=test_value, metadata=test_metadata, agent=test_agent)
+
+    results = external_memory.search("test")
+    assert len(results) == 1
+    assert results[0]["value"] == test_value
+    assert results[0]["metadata"] == test_metadata | {"agent": test_agent}
+
+    external_memory.reset()
+    results = external_memory.search("test")
+    assert len(results) == 0