From 7b0467899ca03bd5b7c758825130add5d8f0c6c2 Mon Sep 17 00:00:00 2001 From: lucasgomide Date: Mon, 31 Mar 2025 16:58:40 -0300 Subject: [PATCH] feat: add external memory --- src/crewai/agent.py | 1 + .../base_agent_executor_mixin.py | 20 +++++++ src/crewai/agents/crew_agent_executor.py | 1 + src/crewai/crew.py | 14 +++++ src/crewai/memory/__init__.py | 9 ++- .../memory/contextual/contextual_memory.py | 32 ++++++++++- src/crewai/memory/external/__init__.py | 0 src/crewai/memory/external/external_memory.py | 56 +++++++++++++++++++ .../memory/external/external_memory_item.py | 13 +++++ src/crewai/memory/storage/mem0_storage.py | 23 +++++--- 10 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 src/crewai/memory/external/__init__.py create mode 100644 src/crewai/memory/external/external_memory.py create mode 100644 src/crewai/memory/external/external_memory_item.py diff --git a/src/crewai/agent.py b/src/crewai/agent.py index a40841db1..fe18ec0af 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -205,6 +205,7 @@ class Agent(BaseAgent): self.crew._long_term_memory, self.crew._entity_memory, self.crew._user_memory, + self.crew._external_memory, ) memory = contextual_memory.build_context_for_task(task, context) if memory.strip() != "": diff --git a/src/crewai/agents/agent_builder/base_agent_executor_mixin.py b/src/crewai/agents/agent_builder/base_agent_executor_mixin.py index e7917f2bd..51f03bd60 100644 --- a/src/crewai/agents/agent_builder/base_agent_executor_mixin.py +++ b/src/crewai/agents/agent_builder/base_agent_executor_mixin.py @@ -47,6 +47,26 @@ class CrewAgentExecutorMixin: print(f"Failed to add to short term memory: {e}") pass + def _create_external_memory(self, output) -> None: + """Create and save a external-term memory item if conditions are met.""" + if ( + self.crew + and self.agent + and self.task + and getattr(self.crew, "_external_memory", None) + ): + try: + self.crew._external_memory.save( + value=output.text, + metadata={ + "description": self.task.description, + }, + agent=self.agent.role, + ) + except Exception as e: + print(f"Failed to add to external memory: {e}") + pass + def _create_long_term_memory(self, output) -> None: """Create and save long-term and entity memory items based on evaluation.""" if ( diff --git a/src/crewai/agents/crew_agent_executor.py b/src/crewai/agents/crew_agent_executor.py index eb0cb22ad..a907e4520 100644 --- a/src/crewai/agents/crew_agent_executor.py +++ b/src/crewai/agents/crew_agent_executor.py @@ -130,6 +130,7 @@ class CrewAgentExecutor(CrewAgentExecutorMixin): self._create_short_term_memory(formatted_answer) self._create_long_term_memory(formatted_answer) + self._create_external_memory(formatted_answer) return {"output": formatted_answer.output} def _invoke_loop(self) -> AgentFinish: diff --git a/src/crewai/crew.py b/src/crewai/crew.py index b5f3e3ff5..fd825c335 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -28,6 +28,7 @@ from crewai.knowledge.knowledge import Knowledge from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource from crewai.llm import LLM, BaseLLM from crewai.memory.entity.entity_memory import EntityMemory +from crewai.memory.external.external_memory import ExternalMemory from crewai.memory.long_term.long_term_memory import LongTermMemory from crewai.memory.short_term.short_term_memory import ShortTermMemory from crewai.memory.user.user_memory import UserMemory @@ -105,6 +106,7 @@ class Crew(BaseModel): _long_term_memory: Optional[InstanceOf[LongTermMemory]] = PrivateAttr() _entity_memory: Optional[InstanceOf[EntityMemory]] = PrivateAttr() _user_memory: Optional[InstanceOf[UserMemory]] = PrivateAttr() + _external_memory: Optional[InstanceOf[ExternalMemory]] = PrivateAttr() _train: Optional[bool] = PrivateAttr(default=False) _train_iteration: Optional[int] = PrivateAttr() _inputs: Optional[Dict[str, Any]] = PrivateAttr(default=None) @@ -145,6 +147,10 @@ class Crew(BaseModel): default=None, description="An instance of the UserMemory to be used by the Crew to store/fetch memories of a specific user.", ) + external_memory: Optional[InstanceOf[ExternalMemory]] = Field( + default=None, + description="An Instance of the ExternalMemory to be used by the Crew", + ) embedder: Optional[dict] = Field( default=None, description="Configuration for the embedder to be used for the crew.", @@ -289,6 +295,11 @@ class Crew(BaseModel): if self.entity_memory else EntityMemory(crew=self, embedder_config=self.embedder) ) + self._external_memory = ( + self.external_memory + if self.external_memory + else ExternalMemory(crew=self, embedder_config=self.embedder) + ) if ( self.memory_config and "user_memory" in self.memory_config @@ -1167,6 +1178,7 @@ class Crew(BaseModel): "_short_term_memory", "_long_term_memory", "_entity_memory", + "_external_memory", "_telemetry", "agents", "tasks", @@ -1339,6 +1351,7 @@ class Crew(BaseModel): memory_systems = [ ("short term", getattr(self, "_short_term_memory", None)), ("entity", getattr(self, "_entity_memory", None)), + ("external", getattr(self, "_external_memory", None)), ("long term", getattr(self, "_long_term_memory", None)), ("task output", getattr(self, "_task_output_handler", None)), ("knowledge", getattr(self, "knowledge", None)), @@ -1366,6 +1379,7 @@ class Crew(BaseModel): "entity": (self._entity_memory, "entity"), "knowledge": (self.knowledge, "knowledge"), "kickoff_outputs": (self._task_output_handler, "task output"), + "external": (self._external_memory, "external"), } memory_system, name = reset_functions[memory_type] diff --git a/src/crewai/memory/__init__.py b/src/crewai/memory/__init__.py index 3f7ca2ad6..b6b0cc025 100644 --- a/src/crewai/memory/__init__.py +++ b/src/crewai/memory/__init__.py @@ -2,5 +2,12 @@ from .entity.entity_memory import EntityMemory from .long_term.long_term_memory import LongTermMemory from .short_term.short_term_memory import ShortTermMemory from .user.user_memory import UserMemory +from .external.external_memory import ExternalMemory -__all__ = ["UserMemory", "EntityMemory", "LongTermMemory", "ShortTermMemory"] +__all__ = [ + "UserMemory", + "EntityMemory", + "LongTermMemory", + "ShortTermMemory", + "ExternalMemory", +] diff --git a/src/crewai/memory/contextual/contextual_memory.py b/src/crewai/memory/contextual/contextual_memory.py index a9f657d8a..23e79986f 100644 --- a/src/crewai/memory/contextual/contextual_memory.py +++ b/src/crewai/memory/contextual/contextual_memory.py @@ -1,6 +1,12 @@ from typing import Any, Dict, Optional -from crewai.memory import EntityMemory, LongTermMemory, ShortTermMemory, UserMemory +from crewai.memory import ( + EntityMemory, + ExternalMemory, + LongTermMemory, + ShortTermMemory, + UserMemory, +) class ContextualMemory: @@ -11,6 +17,7 @@ class ContextualMemory: ltm: LongTermMemory, em: EntityMemory, um: UserMemory, + exm: ExternalMemory, ): if memory_config is not None: self.memory_provider = memory_config.get("provider") @@ -20,6 +27,7 @@ class ContextualMemory: self.ltm = ltm self.em = em self.um = um + self.exm = exm def build_context_for_task(self, task, context) -> str: """ @@ -35,6 +43,7 @@ class ContextualMemory: context.append(self._fetch_ltm_context(task.description)) context.append(self._fetch_stm_context(query)) context.append(self._fetch_entity_context(query)) + context.append(self._fetch_external_context(query)) if self.memory_provider == "mem0": context.append(self._fetch_user_context(query)) return "\n".join(filter(None, context)) @@ -106,3 +115,24 @@ class ContextualMemory: f"- {result['memory']}" for result in user_memories ) return f"User memories/preferences:\n{formatted_memories}" + + def _fetch_external_context(self, query: str) -> str: + """ + Fetches and formats relevant information from External Memory. + Args: + query (str): The search query to find relevant information. + Returns: + str: Formatted information as bullet points, or an empty string if none found. + """ + if self.exm is None: + return "" + + external_memories = self.exm.search(query) + + if not external_memories: + return "" + + formatted_memories = "\n".join( + f"- {result['memory']}" for result in external_memories + ) + return f"External memories:\n{formatted_memories}" diff --git a/src/crewai/memory/external/__init__.py b/src/crewai/memory/external/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/crewai/memory/external/external_memory.py b/src/crewai/memory/external/external_memory.py new file mode 100644 index 000000000..c0f430769 --- /dev/null +++ b/src/crewai/memory/external/external_memory.py @@ -0,0 +1,56 @@ +from typing import Any, Dict, Optional + +from crewai.memory.external.external_memory_item import ExternalMemoryItem +from crewai.memory.memory import Memory +from crewai.memory.storage.interface import Storage + + +class ExternalMemory(Memory): + def __init__(self, crew=None, embedder_config=None, storage=None): + storage = ( + storage + if storage + else self._create_storage(crew=crew, embedder_config=embedder_config) + ) + + super().__init__(storage=storage) + + @staticmethod + def _configure_mem0(crew, config) -> "Mem0Storage": + from crewai.memory.storage.mem0_storage import Mem0Storage + + return Mem0Storage(type="external", crew=crew, config=config) + + @property + def external_supported_storages(self): + return { + "mem0": self._configure_mem0, + } + + def _create_storage(self, crew, embedder_config): + if not embedder_config: + return Storage() + + if embedder_config and "provider" not in embedder_config: + raise ValueError("embedder_config must include a 'provider' key") + + provider = embedder_config["provider"] + if provider not in self.external_supported_storages: + raise ValueError(f"Provider {provider} not supported") + + return self.external_supported_storages[provider]( + crew, embedder_config.get("config", {}) + ) + + def save( + self, + value: Any, + metadata: Optional[Dict[str, Any]] = None, + agent: Optional[str] = None, + ) -> None: + """Saves a value into the external storage.""" + item = ExternalMemoryItem(value=value, metadata=metadata, agent=agent) + super().save(value=item.value, metadata=item.metadata, agent=item.agent) + + def reset(self) -> None: + self.storage.reset() diff --git a/src/crewai/memory/external/external_memory_item.py b/src/crewai/memory/external/external_memory_item.py new file mode 100644 index 000000000..c97cccd59 --- /dev/null +++ b/src/crewai/memory/external/external_memory_item.py @@ -0,0 +1,13 @@ +from typing import Any, Dict, Optional + + +class ExternalMemoryItem: + def __init__( + self, + value: Any, + metadata: Optional[Dict[str, Any]] = None, + agent: Optional[str] = None, + ): + self.value = value + self.metadata = metadata + self.agent = agent diff --git a/src/crewai/memory/storage/mem0_storage.py b/src/crewai/memory/storage/mem0_storage.py index ce526a457..c2c11d03e 100644 --- a/src/crewai/memory/storage/mem0_storage.py +++ b/src/crewai/memory/storage/mem0_storage.py @@ -13,10 +13,10 @@ class Mem0Storage(Storage): def __init__(self, type, crew=None, config=None): super().__init__() - supported_types = ["user", "short_term", "long_term", "entities"] + supported_types = ["user", "short_term", "long_term", "entities", "external"] if type not in supported_types: raise ValueError( - "Invalid type for Mem0Storage. Must be one of: " + f"Invalid type '{type}' for Mem0Storage. Must be one of: " + ", ".join(supported_types) ) @@ -61,9 +61,6 @@ class Mem0Storage(Storage): def save(self, value: Any, metadata: Dict[str, Any]) -> None: user_id = self._get_user_id() agent_name = self._get_agent_name() - if user_id: - self.memory.add(value, user_id=user_id, metadata={**metadata}) - if self.memory_type == "short_term": self.memory.add( value, agent_id=agent_name, metadata={"type": "short_term", **metadata} @@ -77,7 +74,14 @@ class Mem0Storage(Storage): ) elif self.memory_type == "entities": self.memory.add( - value, user_id=agent_name, metadata={"type": "entity", **metadata} + value, agent_id=agent_name, metadata={"type": "entity", **metadata} + ) + elif self.memory_type == "external": + self.memory.add( + value, + user_id=user_id, + agent_id=agent_name, + metadata={"type": "external", **metadata}, ) def search( @@ -90,18 +94,19 @@ class Mem0Storage(Storage): if user_id := self._get_user_id(): params["user_id"] = user_id + agent_name = self._get_agent_name() if self.memory_type == "short_term": - agent_name = self._get_agent_name() params["agent_id"] = agent_name params["metadata"] = {"type": "short_term"} elif self.memory_type == "long_term": - agent_name = self._get_agent_name() params["agent_id"] = agent_name params["metadata"] = {"type": "long_term"} elif self.memory_type == "entities": - agent_name = self._get_agent_name() params["agent_id"] = agent_name params["metadata"] = {"type": "entity"} + elif self.memory_type == "external": + params["agent_id"] = agent_name + params["metadata"] = {"type": "external"} # Discard the filters for now since we create the filters # automatically when the crew is created.