mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-23 15:18:14 +00:00
feat: complete monorepo transformation with tools integration
- Add crewai-tools as git subtree preserving full history - Move tools to proper src/ directory structure with git mv - Configure tools pyproject.toml for workspace dependency on crewai-core - Update workspace configuration to include both packages - Fix build configurations for both packages
This commit is contained in:
46
packages/tools/src/crewai_tools/rag/source_content.py
Normal file
46
packages/tools/src/crewai_tools/rag/source_content.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
from typing import TYPE_CHECKING
|
||||
from pathlib import Path
|
||||
from functools import cached_property
|
||||
|
||||
from crewai_tools.rag.misc import compute_sha256
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crewai_tools.rag.data_types import DataType
|
||||
|
||||
|
||||
class SourceContent:
|
||||
def __init__(self, source: str | Path):
|
||||
self.source = str(source)
|
||||
|
||||
def is_url(self) -> bool:
|
||||
if not isinstance(self.source, str):
|
||||
return False
|
||||
try:
|
||||
parsed_url = urlparse(self.source)
|
||||
return bool(parsed_url.scheme and parsed_url.netloc)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def path_exists(self) -> bool:
|
||||
return os.path.exists(self.source)
|
||||
|
||||
@cached_property
|
||||
def data_type(self) -> "DataType":
|
||||
from crewai_tools.rag.data_types import DataTypes
|
||||
|
||||
return DataTypes.from_content(self.source)
|
||||
|
||||
@cached_property
|
||||
def source_ref(self) -> str:
|
||||
""""
|
||||
Returns the source reference for the content.
|
||||
If the content is a URL or a local file, returns the source.
|
||||
Otherwise, returns the hash of the content.
|
||||
"""
|
||||
|
||||
if self.is_url() or self.path_exists():
|
||||
return self.source
|
||||
|
||||
return compute_sha256(self.source)
|
||||
Reference in New Issue
Block a user