Compare commits

...

6 Commits

Author SHA1 Message Date
Devin AI
b1d0072c13 Fix failing tests: update expectations for new URL-as-is behavior
- Update test inputs to provide complete URLs with /api/embeddings endpoint
- Remove tests for automatic URL construction that no longer exists
- Tests now reflect the requirement for users to provide complete URLs
- Addresses CI failures after removing _construct_embeddings_url helper

Co-Authored-By: João <joao@crewai.com>
2025-07-02 17:07:33 +00:00
Devin AI
b022e06e0d Fix tests to match new URL behavior: use complete URLs as-is
- Update test expectations to provide complete URLs with /api/embeddings endpoint
- Remove assumptions about automatic URL construction per lucasgomide's feedback
- Tests now reflect the new behavior where URLs are used exactly as provided
- Maintains backward compatibility for users who provide complete URLs

Co-Authored-By: João <joao@crewai.com>
2025-07-02 17:04:27 +00:00
Devin AI
2ede9ca9be Address GitHub review: remove URL construction helper and use provided URLs as-is
- Remove _construct_embeddings_url() method as requested by lucasgomide
- Use URLs exactly as provided by config or environment variable
- Add comprehensive documentation about API_BASE environment variable support
- Maintain URL validation for security
- Simplify implementation to avoid assumptions about URL formats across providers

Co-Authored-By: João <joao@crewai.com>
2025-07-02 17:00:29 +00:00
Devin AI
f4807ee858 Fix URL validation to only accept HTTP/HTTPS schemes for Ollama embeddings
- Update _validate_url() to restrict schemes to 'http' and 'https' only
- Fixes CI test failures for ftp://invalid-scheme test cases
- Maintains security by preventing non-web protocols

Co-Authored-By: João <joao@crewai.com>
2025-07-01 22:13:10 +00:00
Devin AI
abd1d341da Address GitHub review feedback: add URL validation, helper functions, and edge case tests
- Add _validate_url() helper function with proper URL validation using urllib.parse
- Add _construct_embeddings_url() helper function to refactor URL construction logic
- Add comprehensive error handling with ValueError for invalid URLs
- Fix test mocking to use correct chromadb import path
- Add edge case tests for invalid URLs with pytest markers
- Organize tests with @pytest.mark.url_configuration and @pytest.mark.error_handling
- Remove unused imports (pytest, MagicMock) to fix lint issues

This addresses all suggestions from joaomdmoura's AI review while maintaining backward compatibility.

Co-Authored-By: João <joao@crewai.com>
2025-07-01 22:07:13 +00:00
Devin AI
d422439b7a Fix Ollama embedding configurator to respect API_BASE environment variable
- Add support for API_BASE environment variable in _configure_ollama method
- Implement priority order: url config > api_base config > API_BASE env var > default
- Handle URL construction with proper /api/embeddings endpoint appending
- Maintain backward compatibility with existing url parameter
- Add comprehensive tests covering all configuration scenarios
- Fixes issue #3099: Ollama integration problems with API_BASE configuration

The fix ensures consistent behavior with other embedding providers and
allows users to configure Ollama embeddings using environment variables
for better deployment flexibility.

Co-Authored-By: João <joao@crewai.com>
2025-07-01 22:01:45 +00:00
2 changed files with 215 additions and 1 deletions

View File

@@ -91,14 +91,52 @@ class EmbeddingConfigurator:
organization_id=config.get("organization_id"),
)
@staticmethod
def _validate_url(url):
"""Validate that a URL is properly formatted and uses HTTP/HTTPS scheme."""
if not url:
return False
from urllib.parse import urlparse
try:
result = urlparse(url)
return all([
result.scheme in ('http', 'https'),
result.netloc
])
except ValueError:
return False
@staticmethod
def _configure_ollama(config, model_name):
"""Configure Ollama embedding function.
Supports configuration via:
1. config.url - Direct URL to Ollama embeddings endpoint
2. config.api_base - Base URL for Ollama API
3. API_BASE environment variable - Base URL from environment
4. Default: http://localhost:11434/api/embeddings
Note: When using api_base or API_BASE, ensure the URL includes the full
embeddings endpoint path (e.g., http://localhost:11434/api/embeddings)
"""
from chromadb.utils.embedding_functions.ollama_embedding_function import (
OllamaEmbeddingFunction,
)
url = (
config.get("url")
or config.get("api_base")
or os.getenv("API_BASE")
or "http://localhost:11434/api/embeddings"
)
if not EmbeddingConfigurator._validate_url(url):
raise ValueError(f"Invalid Ollama API URL: {url}")
return OllamaEmbeddingFunction(
url=config.get("url", "http://localhost:11434/api/embeddings"),
url=url,
model_name=model_name,
)

View File

@@ -0,0 +1,176 @@
import os
import pytest
from unittest.mock import patch
from crewai.utilities.embedding_configurator import EmbeddingConfigurator
@pytest.mark.url_configuration
class TestOllamaEmbeddingConfigurator:
def setup_method(self):
self.configurator = EmbeddingConfigurator()
@patch.dict(os.environ, {}, clear=True)
def test_ollama_default_url(self):
config = {"provider": "ollama", "config": {"model": "llama2"}}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://localhost:11434/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {"API_BASE": "http://custom-ollama:8080/api/embeddings"}, clear=True)
def test_ollama_respects_api_base_env_var(self):
config = {"provider": "ollama", "config": {"model": "llama2"}}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://custom-ollama:8080/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {"API_BASE": "http://env-ollama:8080/api/embeddings"}, clear=True)
def test_ollama_config_url_overrides_env_var(self):
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"url": "http://config-ollama:9090/api/embeddings"
}
}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://config-ollama:9090/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {"API_BASE": "http://env-ollama:8080/api/embeddings"}, clear=True)
def test_ollama_config_api_base_overrides_env_var(self):
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"api_base": "http://config-ollama:9090/api/embeddings"
}
}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://config-ollama:9090/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {}, clear=True)
def test_ollama_url_priority_order(self):
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"url": "http://url-config:1111/api/embeddings",
"api_base": "http://api-base-config:2222/api/embeddings"
}
}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://url-config:1111/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {"API_BASE": "http://localhost:11434/api/embeddings"}, clear=True)
def test_ollama_uses_provided_url_as_is(self):
config = {"provider": "ollama", "config": {"model": "llama2"}}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://localhost:11434/api/embeddings",
model_name="llama2"
)
@patch.dict(os.environ, {"API_BASE": "http://custom-base:9000"}, clear=True)
def test_ollama_requires_complete_url_in_api_base(self):
"""Test that demonstrates users must provide complete URLs including endpoint."""
config = {"provider": "ollama", "config": {"model": "llama2"}}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://custom-base:9000",
model_name="llama2"
)
@patch.dict(os.environ, {}, clear=True)
def test_ollama_config_api_base_without_url(self):
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"api_base": "http://config-ollama:9090/api/embeddings"
}
}
with patch("chromadb.utils.embedding_functions.ollama_embedding_function.OllamaEmbeddingFunction") as mock_ollama:
self.configurator.configure_embedder(config)
mock_ollama.assert_called_once_with(
url="http://config-ollama:9090/api/embeddings",
model_name="llama2"
)
@pytest.mark.error_handling
class TestOllamaErrorHandling:
def setup_method(self):
self.configurator = EmbeddingConfigurator()
@pytest.mark.parametrize("invalid_url", [
"not-a-url",
"ftp://invalid-scheme",
"http://",
"://missing-scheme",
"http:///missing-netloc",
])
def test_invalid_url_raises_error(self, invalid_url):
"""Test that invalid URLs raise ValueError with clear error message."""
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"url": invalid_url
}
}
with pytest.raises(ValueError, match="Invalid Ollama API URL"):
self.configurator.configure_embedder(config)
@pytest.mark.parametrize("invalid_api_base", [
"not-a-url",
"ftp://invalid-scheme",
"http://",
"://missing-scheme",
])
def test_invalid_api_base_raises_error(self, invalid_api_base):
"""Test that invalid api_base URLs raise ValueError with clear error message."""
config = {
"provider": "ollama",
"config": {
"model": "llama2",
"api_base": invalid_api_base
}
}
with pytest.raises(ValueError, match="Invalid Ollama API URL"):
self.configurator.configure_embedder(config)
@patch.dict(os.environ, {"API_BASE": "not-a-valid-url"}, clear=True)
def test_invalid_env_var_raises_error(self):
"""Test that invalid API_BASE environment variable raises ValueError."""
config = {"provider": "ollama", "config": {"model": "llama2"}}
with pytest.raises(ValueError, match="Invalid Ollama API URL"):
self.configurator.configure_embedder(config)