docs: add troubleshooting section and make tiktoken optional

Co-Authored-By: Joe Moura <joao@crewai.com>

README.md

@@ -86,7 +86,44 @@ First, install CrewAI:
 pip install crewai
 ```
 
-> **Note**: While pre-built wheels are available for most common platforms and Python versions, some environments might require building `tiktoken` from source. In such cases, you'll need the [Rust compiler](https://rustup.rs/) installed on your system.
+> **Note**: While pre-built wheels are available for most common platforms and Python versions, some environments might require building `tiktoken` from source. In such cases:
+> 1. Install the [Rust compiler](https://rustup.rs/) following your platform's instructions below
+> 2. Windows users also need [Microsoft Visual C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
+> 3. For troubleshooting, refer to the [tiktoken guide](https://github.com/openai/tiktoken#troubleshooting)
+>
+> **Platform-Specific Rust Installation:**
+> - **Windows**: 
+>   1. Download and run [rustup-init.exe](https://rustup.rs/)
+>   2. Install Visual C++ Build Tools (select "Desktop development with C++")
+> - **Linux**: Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
+> - **macOS**: Use `brew install rust` or install via [rustup](https://rustup.rs/)
+>
+> After installing Rust, you may need to restart your terminal for the changes to take effect.
+
+### Troubleshooting Dependencies
+
+If you encounter issues during installation or usage, here are some common solutions:
+
+#### Common Issues
+
+1. **ModuleNotFoundError: No module named 'tiktoken'**
+   - Install tiktoken explicitly: `pip install 'crewai[embeddings]'`
+   - If using embedchain or other tools: `pip install 'crewai[tools]'`
+
+2. **Failed building wheel for tiktoken**
+   - Ensure Rust compiler is installed (see installation steps above)
+   - For Windows: Verify Visual C++ Build Tools are installed
+   - Try upgrading pip: `pip install --upgrade pip`
+   - If issues persist, use a pre-built wheel: `pip install tiktoken --prefer-binary`
+
+3. **Version Conflicts**
+   - If you see dependency conflicts with tiktoken, try installing specific extras:
+     ```bash
+     pip install 'crewai[tools,embeddings]'  # For full functionality
+     pip install crewai  # For core features only
+     ```
+
+For additional help, check the [tiktoken troubleshooting guide](https://github.com/openai/tiktoken#troubleshooting).
 
 If you want to install the 'crewai' package along with its optional features that include additional tools for agents, you can do so by using the following command:
 

pyproject.toml

@@ -8,29 +8,39 @@ authors = [
     { name = "Joao Moura", email = "joao@crewai.com" }
 ]
 dependencies = [
+    # Core Dependencies
     "pydantic>=2.4.2",
     "openai>=1.13.3",
+    "litellm>=1.44.22",
+    "instructor>=1.3.3",
+    
+    # Text Processing
+    "pdfplumber>=0.11.4",
+    "regex>=2024.9.11",
+    
+    # Telemetry and Monitoring
     "opentelemetry-api>=1.22.0",
     "opentelemetry-sdk>=1.22.0",
     "opentelemetry-exporter-otlp-proto-http>=1.22.0",
-    "instructor>=1.3.3",
-    "regex>=2024.9.11",
-    "click>=8.1.7",
+    
+    # Data Handling
+    "chromadb>=0.5.23",
+    "openpyxl>=3.1.5",
+    "pyvis>=0.3.2",
+    
+    # Authentication and Security
+    "auth0-python>=4.7.1",
     "python-dotenv>=1.0.0",
+    
+    # Configuration and Utils
+    "click>=8.1.7",
     "appdirs>=1.4.4",
     "jsonref>=1.1.0",
     "json-repair>=0.25.2",
-    "auth0-python>=4.7.1",
-    "litellm>=1.44.22",
-    "pyvis>=0.3.2",
     "uv>=0.4.25",
     "tomli-w>=1.1.0",
     "tomli>=2.0.2",
-    "chromadb>=0.5.23",
-    "pdfplumber>=0.11.4",
-    "openpyxl>=3.1.5",
     "blinker>=1.9.0",
-    "tiktoken~=0.7.0",
 ]
 
 [project.urls]
@@ -40,6 +50,9 @@ Repository = "https://github.com/crewAIInc/crewAI"
 
 [project.optional-dependencies]
 tools = ["crewai-tools>=0.17.0"]
+embeddings = [
+    "tiktoken~=0.7.0"
+]
 agentops = ["agentops>=0.3.0"]
 fastembed = ["fastembed>=0.4.1"]
 pdfplumber = [