Fix UnicodeDecodeError in litellm when loading JSON files on Windows

Co-Authored-By: Joe Moura <joao@crewai.com>

src/crewai/__init__.py

@@ -1,5 +1,8 @@
 import warnings
 
+from crewai.patches.litellm_patch import apply_patches
+apply_patches()
+
 from crewai.agent import Agent
 from crewai.crew import Crew
 from crewai.crews.crew_output import CrewOutput

src/crewai/patches/__init__.py

@@ -0,0 +1 @@
+# This file is intentionally left empty to make the directory a package

src/crewai/patches/litellm_patch.py

@@ -0,0 +1,49 @@
+"""
+Patch for litellm to fix UnicodeDecodeError on Windows systems.
+
+This patch ensures that all file open operations in litellm use UTF-8 encoding,
+which prevents UnicodeDecodeError when loading JSON files on Windows systems
+where the default encoding is cp1252 or cp1254.
+"""
+
+import builtins
+import functools
+import io
+import json
+import logging
+import os
+from importlib import resources
+from typing import Any, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+
+def apply_patches():
+    """Apply all patches to fix litellm encoding issues."""
+    logger.info("Applying litellm encoding patches")
+    
+    original_open = builtins.open
+    
+    @functools.wraps(original_open)
+    def patched_open(
+        file, mode='r', buffering=-1, encoding=None, 
+        errors=None, newline=None, closefd=True, opener=None
+    ):
+        if 'r' in mode and encoding is None and 'b' not in mode:
+            encoding = 'utf-8'
+        
+        return original_open(
+            file, mode, buffering, encoding, 
+            errors, newline, closefd, opener
+        )
+    
+    builtins.open = patched_open
+    
+    logger.info("Successfully applied litellm encoding patches")
+
+
+def remove_patches():
+    """Remove all patches (for testing purposes)."""
+    if hasattr(builtins, '_original_open'):
+        builtins.open = builtins._original_open
+        logger.info("Removed litellm encoding patches")

tests/litellm_tests/test_encoding.py

@@ -0,0 +1,27 @@
+import json
+import os
+import sys
+import unittest
+from unittest.mock import patch, mock_open
+
+import pytest
+
+from crewai.llm import LLM
+
+
+class TestLitellmEncoding(unittest.TestCase):
+    """Test that the litellm encoding patch works correctly."""
+
+    def test_json_load_with_utf8_encoding(self):
+        """Test that json.load is called with UTF-8 encoding."""
+        
+        mock_content = '{"test": "日本語テキスト"}'  # Japanese text that would fail with cp1252
+        
+        with patch('builtins.open', mock_open(read_data=mock_content)):
+            import litellm
+            
+            self.assertTrue(hasattr(litellm.utils, 'json_data'))
+            
+            with open('test.json', 'r') as f:
+                data = json.load(f)
+                self.assertEqual(data['test'], '日本語テキスト')