Compare commits

...

6 Commits

Author SHA1 Message Date
Devin AI
e78efb047f style: fix import block formatting with ruff
Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:38:03 +00:00
Devin AI
d58dc08511 style: fix import sorting in base_token_process.py
Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:36:52 +00:00
Devin AI
9ed21c4b0e feat: add logging for None values and improve documentation
- Add logging for None token values
- Improve test documentation and structure
- Fix import sorting in tests

Part of #2198

Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:36:00 +00:00
Devin AI
4e84b98ac2 refactor: improve token counter implementation
- Fix import sorting in tests
- Add docstrings and type validation
- Add comprehensive test cases
- Add validation for negative token counts

Addresses review feedback on #2198

Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:32:43 +00:00
Devin AI
9f7f1cdb54 style: fix import sorting in test_token_process.py
Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:30:52 +00:00
Devin AI
3f02d10626 fix: handle None values in token counter
- Update sum_cached_prompt_tokens to handle None values gracefully
- Add unit tests for token counting with None values
- Fixes #2197

Co-Authored-By: Joe Moura <joao@crewai.com>
2025-02-22 01:29:47 +00:00
2 changed files with 68 additions and 1 deletions

View File

@@ -1,5 +1,9 @@
import logging
from crewai.types.usage_metrics import UsageMetrics
logger = logging.getLogger(__name__)
class TokenProcess:
def __init__(self) -> None:
@@ -17,7 +21,21 @@ class TokenProcess:
self.completion_tokens += tokens
self.total_tokens += tokens
def sum_cached_prompt_tokens(self, tokens: int) -> None:
def sum_cached_prompt_tokens(self, tokens: int | None) -> None:
"""
Adds the given token count to cached prompt tokens.
Args:
tokens (int | None): Number of tokens to add. None values are ignored.
Raises:
ValueError: If tokens is negative.
"""
if tokens is None:
logger.debug("Received None value for token count")
return
if tokens < 0:
raise ValueError("Token count cannot be negative")
self.cached_prompt_tokens += tokens
def sum_successful_requests(self, requests: int) -> None:

View File

@@ -0,0 +1,49 @@
import unittest
from crewai.agents.agent_builder.utilities.base_token_process import TokenProcess
class TestTokenProcess(unittest.TestCase):
"""Test suite for TokenProcess class token counting functionality."""
def setUp(self):
"""Initialize a fresh TokenProcess instance before each test."""
self.token_process = TokenProcess()
def test_sum_cached_prompt_tokens_with_none(self):
"""Test that passing None to sum_cached_prompt_tokens doesn't modify the counter."""
initial_tokens = self.token_process.cached_prompt_tokens
self.token_process.sum_cached_prompt_tokens(None)
self.assertEqual(self.token_process.cached_prompt_tokens, initial_tokens)
def test_sum_cached_prompt_tokens_with_int(self):
"""Test that passing an integer correctly increments the counter."""
initial_tokens = self.token_process.cached_prompt_tokens
self.token_process.sum_cached_prompt_tokens(5)
self.assertEqual(self.token_process.cached_prompt_tokens, initial_tokens + 5)
def test_sum_cached_prompt_tokens_with_zero(self):
"""Test that passing zero doesn't modify the counter."""
initial_tokens = self.token_process.cached_prompt_tokens
self.token_process.sum_cached_prompt_tokens(0)
self.assertEqual(self.token_process.cached_prompt_tokens, initial_tokens)
def test_sum_cached_prompt_tokens_with_large_number(self):
"""Test that the counter works with large numbers."""
initial_tokens = self.token_process.cached_prompt_tokens
self.token_process.sum_cached_prompt_tokens(1000000)
self.assertEqual(self.token_process.cached_prompt_tokens, initial_tokens + 1000000)
def test_sum_cached_prompt_tokens_multiple_calls(self):
"""Test that multiple calls accumulate correctly, ignoring None values."""
initial_tokens = self.token_process.cached_prompt_tokens
self.token_process.sum_cached_prompt_tokens(5)
self.token_process.sum_cached_prompt_tokens(None)
self.token_process.sum_cached_prompt_tokens(3)
self.assertEqual(self.token_process.cached_prompt_tokens, initial_tokens + 8)
def test_sum_cached_prompt_tokens_with_negative(self):
"""Test that negative values raise ValueError."""
with self.assertRaises(ValueError) as context:
self.token_process.sum_cached_prompt_tokens(-1)
self.assertEqual(str(context.exception), "Token count cannot be negative")