Knowledge (#1567)

* initial knowledge

* WIP

* Adding core knowledge sources

* Improve types and better support for file paths

* added additional sources

* fix linting

* update yaml to include optional deps

* adding in lorenze feedback

* ensure embeddings are persisted

* improvements all around Knowledge class

* return this

* properly reset memory

* properly reset memory+knowledge

* consolodation and improvements

* linted

* cleanup rm unused embedder

* fix test

* fix duplicate

* generating cassettes for knowledge test

* updated default embedder

* None embedder to use default on pipeline cloning

* improvements

* fixed text_file_knowledge

* mypysrc fixes

* type check fixes

* added extra cassette

* just mocks

* linted

* mock knowledge query to not spin up db

* linted

* verbose run

* put a flag

* fix

* adding docs

* better docs

* improvements from review

* more docs

* linted

* rm print

* more fixes

* clearer docs

* added docstrings and type hints for cli

---------

Co-authored-by: João Moura <joaomdmoura@gmail.com>
Co-authored-by: Lorenze Jay <lorenzejaytech@gmail.com>

src/crewai/knowledge/source/base_file_knowledge_source.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+from typing import Union, List
+
+from pydantic import Field
+
+from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
+from typing import Dict, Any
+from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
+
+
+class BaseFileKnowledgeSource(BaseKnowledgeSource):
+    """Base class for knowledge sources that load content from files."""
+
+    file_path: Union[Path, List[Path]] = Field(...)
+    content: Dict[Path, str] = Field(init=False, default_factory=dict)
+    storage: KnowledgeStorage = Field(default_factory=KnowledgeStorage)
+
+    def model_post_init(self, _):
+        """Post-initialization method to load content."""
+        self.content = self.load_content()
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess file content. Should be overridden by subclasses."""
+        paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path
+
+        for path in paths:
+            if not path.exists():
+                raise FileNotFoundError(f"File not found: {path}")
+            if not path.is_file():
+                raise ValueError(f"Path is not a file: {path}")
+        return {}
+
+    def save_documents(self, metadata: Dict[str, Any]):
+        """Save the documents to the storage."""
+        chunk_metadatas = [metadata.copy() for _ in self.chunks]
+        self.storage.save(self.chunks, chunk_metadatas)

src/crewai/knowledge/source/base_knowledge_source.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, Field
+
+from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
+
+
+class BaseKnowledgeSource(BaseModel, ABC):
+    """Abstract base class for knowledge sources."""
+
+    chunk_size: int = 4000
+    chunk_overlap: int = 200
+    chunks: List[str] = Field(default_factory=list)
+    chunk_embeddings: List[np.ndarray] = Field(default_factory=list)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    storage: KnowledgeStorage = Field(default_factory=KnowledgeStorage)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    @abstractmethod
+    def load_content(self) -> Dict[Any, str]:
+        """Load and preprocess content from the source."""
+        pass
+
+    @abstractmethod
+    def add(self) -> None:
+        """Process content, chunk it, compute embeddings, and save them."""
+        pass
+
+    def get_embeddings(self) -> List[np.ndarray]:
+        """Return the list of embeddings for the chunks."""
+        return self.chunk_embeddings
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]
+
+    def save_documents(self, metadata: Dict[str, Any]):
+        """
+        Save the documents to the storage.
+        This method should be called after the chunks and embeddings are generated.
+        """
+        self.storage.save(self.chunks, metadata)

src/crewai/knowledge/source/csv_knowledge_source.py

@@ -0,0 +1,44 @@
+import csv
+from typing import Dict, List
+from pathlib import Path
+
+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
+
+
+class CSVKnowledgeSource(BaseFileKnowledgeSource):
+    """A knowledge source that stores and queries CSV file content using embeddings."""
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess CSV file content."""
+        super().load_content()  # Validate the file path
+
+        file_path = (
+            self.file_path[0] if isinstance(self.file_path, list) else self.file_path
+        )
+        file_path = Path(file_path) if isinstance(file_path, str) else file_path
+
+        with open(file_path, "r", encoding="utf-8") as csvfile:
+            reader = csv.reader(csvfile)
+            content = ""
+            for row in reader:
+                content += " ".join(row) + "\n"
+        return {file_path: content}
+
+    def add(self) -> None:
+        """
+        Add CSV file content to the knowledge source, chunk it, compute embeddings,
+        and save the embeddings.
+        """
+        content_str = (
+            str(self.content) if isinstance(self.content, dict) else self.content
+        )
+        new_chunks = self._chunk_text(content_str)
+        self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]

src/crewai/knowledge/source/excel_knowledge_source.py

@@ -0,0 +1,56 @@
+from typing import Dict, List
+from pathlib import Path
+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
+
+
+class ExcelKnowledgeSource(BaseFileKnowledgeSource):
+    """A knowledge source that stores and queries Excel file content using embeddings."""
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess Excel file content."""
+        super().load_content()  # Validate the file path
+        pd = self._import_dependencies()
+
+        if isinstance(self.file_path, list):
+            file_path = self.file_path[0]
+        else:
+            file_path = self.file_path
+
+        df = pd.read_excel(file_path)
+        content = df.to_csv(index=False)
+        return {file_path: content}
+
+    def _import_dependencies(self):
+        """Dynamically import dependencies."""
+        try:
+            import openpyxl  # noqa
+            import pandas as pd
+
+            return pd
+        except ImportError as e:
+            missing_package = str(e).split()[-1]
+            raise ImportError(
+                f"{missing_package} is not installed. Please install it with: pip install {missing_package}"
+            )
+
+    def add(self) -> None:
+        """
+        Add Excel file content to the knowledge source, chunk it, compute embeddings,
+        and save the embeddings.
+        """
+        # Convert dictionary values to a single string if content is a dictionary
+        if isinstance(self.content, dict):
+            content_str = "\n".join(str(value) for value in self.content.values())
+        else:
+            content_str = str(self.content)
+
+        new_chunks = self._chunk_text(content_str)
+        self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]

src/crewai/knowledge/source/json_knowledge_source.py

@@ -0,0 +1,54 @@
+import json
+from typing import Any, Dict, List
+from pathlib import Path
+
+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
+
+
+class JSONKnowledgeSource(BaseFileKnowledgeSource):
+    """A knowledge source that stores and queries JSON file content using embeddings."""
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess JSON file content."""
+        super().load_content()  # Validate the file path
+        paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path
+
+        content: Dict[Path, str] = {}
+        for path in paths:
+            with open(path, "r", encoding="utf-8") as json_file:
+                data = json.load(json_file)
+            content[path] = self._json_to_text(data)
+        return content
+
+    def _json_to_text(self, data: Any, level: int = 0) -> str:
+        """Recursively convert JSON data to a text representation."""
+        text = ""
+        indent = "  " * level
+        if isinstance(data, dict):
+            for key, value in data.items():
+                text += f"{indent}{key}: {self._json_to_text(value, level + 1)}\n"
+        elif isinstance(data, list):
+            for item in data:
+                text += f"{indent}- {self._json_to_text(item, level + 1)}\n"
+        else:
+            text += f"{str(data)}"
+        return text
+
+    def add(self) -> None:
+        """
+        Add JSON file content to the knowledge source, chunk it, compute embeddings,
+        and save the embeddings.
+        """
+        content_str = (
+            str(self.content) if isinstance(self.content, dict) else self.content
+        )
+        new_chunks = self._chunk_text(content_str)
+        self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]

src/crewai/knowledge/source/pdf_knowledge_source.py

@@ -0,0 +1,54 @@
+from typing import List, Dict
+from pathlib import Path
+
+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
+
+
+class PDFKnowledgeSource(BaseFileKnowledgeSource):
+    """A knowledge source that stores and queries PDF file content using embeddings."""
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess PDF file content."""
+        super().load_content()  # Validate the file paths
+        pdfplumber = self._import_pdfplumber()
+
+        paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path
+        content = {}
+
+        for path in paths:
+            text = ""
+            with pdfplumber.open(path) as pdf:
+                for page in pdf.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+            content[path] = text
+        return content
+
+    def _import_pdfplumber(self):
+        """Dynamically import pdfplumber."""
+        try:
+            import pdfplumber
+
+            return pdfplumber
+        except ImportError:
+            raise ImportError(
+                "pdfplumber is not installed. Please install it with: pip install pdfplumber"
+            )
+
+    def add(self) -> None:
+        """
+        Add PDF file content to the knowledge source, chunk it, compute embeddings,
+        and save the embeddings.
+        """
+        for _, text in self.content.items():
+            new_chunks = self._chunk_text(text)
+            self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]

src/crewai/knowledge/source/string_knowledge_source.py

@@ -0,0 +1,33 @@
+from typing import List
+
+from pydantic import Field
+
+from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
+
+
+class StringKnowledgeSource(BaseKnowledgeSource):
+    """A knowledge source that stores and queries plain text content using embeddings."""
+
+    content: str = Field(...)
+
+    def model_post_init(self, _):
+        """Post-initialization method to validate content."""
+        self.load_content()
+
+    def load_content(self):
+        """Validate string content."""
+        if not isinstance(self.content, str):
+            raise ValueError("StringKnowledgeSource only accepts string content")
+
+    def add(self) -> None:
+        """Add string content to the knowledge source, chunk it, compute embeddings, and save them."""
+        new_chunks = self._chunk_text(self.content)
+        self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]

src/crewai/knowledge/source/text_file_knowledge_source.py

@@ -0,0 +1,35 @@
+from typing import Dict, List
+from pathlib import Path
+
+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
+
+
+class TextFileKnowledgeSource(BaseFileKnowledgeSource):
+    """A knowledge source that stores and queries text file content using embeddings."""
+
+    def load_content(self) -> Dict[Path, str]:
+        """Load and preprocess text file content."""
+        super().load_content()
+        paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path
+        content = {}
+        for path in paths:
+            with path.open("r", encoding="utf-8") as f:
+                content[path] = f.read()  # type: ignore
+        return content
+
+    def add(self) -> None:
+        """
+        Add text file content to the knowledge source, chunk it, compute embeddings,
+        and save the embeddings.
+        """
+        for _, text in self.content.items():
+            new_chunks = self._chunk_text(text)
+            self.chunks.extend(new_chunks)
+        self.save_documents(metadata=self.metadata)
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """Utility method to split text into chunks."""
+        return [
+            text[i : i + self.chunk_size]
+            for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
+        ]