fix: add tiktoken as explicit dependency and document Rust requirement (#1826)

* feat: add tiktoken as explicit dependency and document Rust requirement

- Add tiktoken>=0.8.0 as explicit dependency to ensure pre-built wheels are used
- Document Rust compiler requirement as fallback in README.md
- Addresses issue #1824 tiktoken build failure

Co-Authored-By: Joe Moura <joao@crewai.com>

* fix: adjust tiktoken version to ~=0.7.0 for dependency compatibility

- Update tiktoken dependency to ~=0.7.0 to resolve conflict with embedchain
- Maintain compatibility with crewai-tools dependency chain
- Addresses CI build failures

Co-Authored-By: Joe Moura <joao@crewai.com>

* docs: add troubleshooting section and make tiktoken optional

Co-Authored-By: Joe Moura <joao@crewai.com>

* Update README.md

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Joe Moura <joao@crewai.com>
Co-authored-by: João Moura <joaomdmoura@gmail.com>

pyproject.toml

@@ -8,27 +8,38 @@ authors = [
     { name = "Joao Moura", email = "joao@crewai.com" }
 ]
 dependencies = [
+    # Core Dependencies
     "pydantic>=2.4.2",
     "openai>=1.13.3",
+    "litellm>=1.44.22",
+    "instructor>=1.3.3",
+    
+    # Text Processing
+    "pdfplumber>=0.11.4",
+    "regex>=2024.9.11",
+    
+    # Telemetry and Monitoring
     "opentelemetry-api>=1.22.0",
     "opentelemetry-sdk>=1.22.0",
     "opentelemetry-exporter-otlp-proto-http>=1.22.0",
-    "instructor>=1.3.3",
-    "regex>=2024.9.11",
-    "click>=8.1.7",
+    
+    # Data Handling
+    "chromadb>=0.5.23",
+    "openpyxl>=3.1.5",
+    "pyvis>=0.3.2",
+    
+    # Authentication and Security
+    "auth0-python>=4.7.1",
     "python-dotenv>=1.0.0",
+    
+    # Configuration and Utils
+    "click>=8.1.7",
     "appdirs>=1.4.4",
     "jsonref>=1.1.0",
     "json-repair>=0.25.2",
-    "auth0-python>=4.7.1",
-    "litellm>=1.44.22",
-    "pyvis>=0.3.2",
     "uv>=0.4.25",
     "tomli-w>=1.1.0",
     "tomli>=2.0.2",
-    "chromadb>=0.5.23",
-    "pdfplumber>=0.11.4",
-    "openpyxl>=3.1.5",
     "blinker>=1.9.0",
 ]
 
@@ -39,6 +50,9 @@ Repository = "https://github.com/crewAIInc/crewAI"
 
 [project.optional-dependencies]
 tools = ["crewai-tools>=0.17.0"]
+embeddings = [
+    "tiktoken~=0.7.0"
+]
 agentops = ["agentops>=0.3.0"]
 fastembed = ["fastembed>=0.4.1"]
 pdfplumber = [