"""Test Agent creation and execution basic functionality.""" from unittest.mock import patch import pytest from langchain.tools import tool from langchain_core.exceptions import OutputParserException from langchain_openai import ChatOpenAI from crewai import Agent, Crew, Task from crewai.agents.cache import CacheHandler from crewai.agents.executor import CrewAgentExecutor from crewai.agents.parser import CrewAgentParser from crewai.tools.tool_calling import InstructorToolCalling from crewai.tools.tool_usage import ToolUsage from crewai.utilities import RPMController def test_agent_creation(): agent = Agent(role="test role", goal="test goal", backstory="test backstory") assert agent.role == "test role" assert agent.goal == "test goal" assert agent.backstory == "test backstory" assert agent.tools == [] def test_agent_default_values(): agent = Agent(role="test role", goal="test goal", backstory="test backstory") assert isinstance(agent.llm, ChatOpenAI) assert agent.llm.model_name == "gpt-4" assert agent.llm.temperature == 0.7 assert agent.llm.verbose == False assert agent.allow_delegation == True def test_custom_llm(): agent = Agent( role="test role", goal="test goal", backstory="test backstory", llm=ChatOpenAI(temperature=0, model="gpt-4"), ) assert isinstance(agent.llm, ChatOpenAI) assert agent.llm.model_name == "gpt-4" assert agent.llm.temperature == 0 @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_without_memory(): no_memory_agent = Agent( role="test role", goal="test goal", backstory="test backstory", memory=False, llm=ChatOpenAI(temperature=0, model="gpt-4"), ) memory_agent = Agent( role="test role", goal="test goal", backstory="test backstory", memory=True, llm=ChatOpenAI(temperature=0, model="gpt-4"), ) task = Task(description="How much is 1 + 1?", agent=no_memory_agent) result = no_memory_agent.execute_task(task) assert result == "1 + 1 equals 2." assert no_memory_agent.agent_executor.memory is None assert memory_agent.agent_executor.memory is not None @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_execution(): agent = Agent( role="test role", goal="test goal", backstory="test backstory", allow_delegation=False, ) task = Task(description="How much is 1 + 1?", agent=agent) output = agent.execute_task(task) assert output == "1 + 1 equals 2." @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_execution_with_tools(): @tool def multiplier(first_number: int, second_number: int) -> float: """Useful for when you need to multiply two numbers together.""" return first_number * second_number agent = Agent( role="test role", goal="test goal", backstory="test backstory", tools=[multiplier], allow_delegation=False, ) task = Task(description="What is 3 times 4?", agent=agent) output = agent.execute_task(task) assert output == "The result of 3 times 4 is 12." @pytest.mark.vcr(filter_headers=["authorization"]) def test_logging_tool_usage(): @tool def multiplier(first_number: int, second_number: int) -> float: """Useful for when you need to multiply two numbers together.""" return first_number * second_number agent = Agent( role="test role", goal="test goal", backstory="test backstory", tools=[multiplier], allow_delegation=False, verbose=True, ) assert agent.tools_handler.last_used_tool == {} task = Task(description="What is 3 times 4?", agent=agent) # force cleaning cache agent.tools_handler.cache = CacheHandler() output = agent.execute_task(task) tool_usage = InstructorToolCalling( tool_name=multiplier.name, arguments={"first_number": 3, "second_number": 4} ) assert output == "The result of multiplying 3 by 4 is 12." assert agent.tools_handler.last_used_tool.tool_name == tool_usage.tool_name assert agent.tools_handler.last_used_tool.arguments == tool_usage.arguments @pytest.mark.vcr(filter_headers=["authorization"]) def test_cache_hitting(): @tool def multiplier(first_number: int, second_number: int) -> float: """Useful for when you need to multiply two numbers together.""" return first_number * second_number cache_handler = CacheHandler() agent = Agent( role="test role", goal="test goal", backstory="test backstory", tools=[multiplier], allow_delegation=False, cache_handler=cache_handler, verbose=True, ) task1 = Task(description="What is 2 times 6?", agent=agent) task2 = Task(description="What is 3 times 3?", agent=agent) output = agent.execute_task(task1) output = agent.execute_task(task2) assert cache_handler._cache == { "multiplier-{'first_number': 2, 'second_number': 6}": 12, "multiplier-{'first_number': 3, 'second_number': 3}": 9, } task = Task( description="What is 2 times 6 times 3? Return only the number", agent=agent ) output = agent.execute_task(task) assert output == "36" assert cache_handler._cache == { "multiplier-{'first_number': 2, 'second_number': 6}": 12, "multiplier-{'first_number': 3, 'second_number': 3}": 9, "multiplier-{'first_number': 12, 'second_number': 3}": 36, } with patch.object(CacheHandler, "read") as read: read.return_value = "0" task = Task( description="What is 2 times 6? Ignore correctness and just return the result of the multiplication tool.", agent=agent, ) output = agent.execute_task(task) assert output == "0" read.assert_called_with( tool="multiplier", input={"first_number": 2, "second_number": 6} ) @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_execution_with_specific_tools(): @tool def multiplier(first_number: int, second_number: int) -> float: """Useful for when you need to multiply two numbers together.""" return first_number * second_number agent = Agent( role="test role", goal="test goal", backstory="test backstory", allow_delegation=False, ) task = Task(description="What is 3 times 4", agent=agent) output = agent.execute_task(task=task, tools=[multiplier]) assert output == "12" @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_custom_max_iterations(): @tool def get_final_answer(numbers) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=1, allow_delegation=False, ) with patch.object( CrewAgentExecutor, "_iter_next_step", wraps=agent.agent_executor._iter_next_step ) as private_mock: task = Task( description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool.", ) agent.execute_task( task=task, tools=[get_final_answer], ) private_mock.assert_called_once() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_repeated_tool_usage(capsys): @tool def get_final_answer(anything: str) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=4, llm=ChatOpenAI(model="gpt-4-0125-preview"), allow_delegation=False, verbose=True, ) task = Task( description="The final answer is 42. But don't give it until I tell you so, instead keep using the `get_final_answer` tool." ) # force cleaning cache agent.tools_handler.cache = CacheHandler() agent.execute_task( task=task, tools=[get_final_answer], ) captured = capsys.readouterr() assert "Final Answer: 42" in captured.out @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_moved_on_after_max_iterations(): @tool def get_final_answer(numbers) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=3, allow_delegation=False, ) task = Task( description="The final answer is 42. But don't give it yet, instead keep using the `get_final_answer` tool. Until you're told you could give my final answer if I'm ready." ) output = agent.execute_task( task=task, tools=[get_final_answer], ) assert output == "42" @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_respect_the_max_rpm_set(capsys): @tool def get_final_answer(anything: str) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=5, max_rpm=1, verbose=True, allow_delegation=False, ) with patch.object(RPMController, "_wait_for_next_minute") as moveon: moveon.return_value = True task = Task( description="Use tool logic for `get_final_answer` but fon't give you final answer yet, instead keep using it unless you're told to give your final answer" ) output = agent.execute_task( task=task, tools=[get_final_answer], ) assert ( output == 'The result of using the `get_final_answer` tool with the input "test input" is 42.' ) captured = capsys.readouterr() assert "Max RPM reached, waiting for next minute to start." in captured.out moveon.assert_called() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_respect_the_max_rpm_set_over_crew_rpm(capsys): from unittest.mock import patch from langchain.tools import tool @tool def get_final_answer(numbers) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=4, max_rpm=10, verbose=True, ) task = Task( description="Don't give a Final Answer, instead keep using the `get_final_answer` tool.", tools=[get_final_answer], agent=agent, ) crew = Crew(agents=[agent], tasks=[task], max_rpm=1, verbose=2) with patch.object(RPMController, "_wait_for_next_minute") as moveon: moveon.return_value = True crew.kickoff() captured = capsys.readouterr() assert "Max RPM reached, waiting for next minute to start." not in captured.out moveon.assert_not_called() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_without_max_rpm_respet_crew_rpm(capsys): from unittest.mock import patch from langchain.tools import tool @tool def get_final_answer(numbers) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", max_rpm=10, verbose=True, ) agent2 = Agent( role="test role2", goal="test goal2", backstory="test backstory2", max_iter=2, verbose=True, ) tasks = [ Task( description="Just say hi.", agent=agent1, ), Task( description="Don't give a Final Answer, instead keep using the `get_final_answer` tool non-stop", tools=[get_final_answer], agent=agent2, ), ] crew = Crew(agents=[agent1, agent2], tasks=tasks, max_rpm=1, verbose=2) with patch.object(RPMController, "_wait_for_next_minute") as moveon: moveon.return_value = True crew.kickoff() captured = capsys.readouterr() assert "get_final_answer" in captured.out assert "Max RPM reached, waiting for next minute to start." in captured.out moveon.assert_called_once() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_error_on_parsing_tool(capsys): from unittest.mock import patch from langchain.tools import tool @tool def get_final_answer(anything: str) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", verbose=True, ) tasks = [ Task( description="Use the get_final_answer tool.", agent=agent1, tools=[get_final_answer], ) ] crew = Crew(agents=[agent1], tasks=tasks, verbose=2) with patch.object(ToolUsage, "_render") as force_exception: force_exception.side_effect = Exception("Error on parsing tool.") crew.kickoff() captured = capsys.readouterr() assert "Error on parsing tool." in captured.out @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_remembers_output_format_after_using_tools_too_many_times(): from unittest.mock import patch from langchain.tools import tool @tool def get_final_answer(anything: str) -> float: """Get the final answer but don't give it yet, just re-use this tool non-stop.""" return 42 agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", max_iter=6, verbose=True, ) tasks = [ Task( description="Use tool logic for `get_final_answer` but fon't give you final answer yet, instead keep using it unless you're told to give your final answer", agent=agent1, tools=[get_final_answer], ) ] crew = Crew(agents=[agent1], tasks=tasks, verbose=2) with patch.object(ToolUsage, "_remember_format") as remember_format: crew.kickoff() remember_format.assert_called() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_use_specific_tasks_output_as_context(capsys): agent1 = Agent(role="test role", goal="test goal", backstory="test backstory") agent2 = Agent(role="test role2", goal="test goal2", backstory="test backstory2") say_hi_task = Task(description="Just say hi.", agent=agent1) say_bye_task = Task(description="Just say bye.", agent=agent1) answer_task = Task( description="Answer accordingly to the context you got.", context=[say_hi_task], agent=agent2, ) tasks = [say_hi_task, say_bye_task, answer_task] crew = Crew(agents=[agent1, agent2], tasks=tasks) result = crew.kickoff() assert "bye" not in result.lower() assert "hi" in result.lower() or "hello" in result.lower() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_step_callback(): class StepCallback: def callback(self, step): print(step) with patch.object(StepCallback, "callback") as callback: @tool def learn_about_AI(topic) -> float: """Useful for when you need to learn about AI to write an paragraph about it.""" return "AI is a very broad field." agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", tools=[learn_about_AI], step_callback=StepCallback().callback, ) essay = Task( description="Write and then review an small paragraph on AI until it's AMAZING", agent=agent1, ) tasks = [essay] crew = Crew(agents=[agent1], tasks=tasks) callback.return_value = "ok" crew.kickoff() callback.assert_called() @pytest.mark.vcr(filter_headers=["authorization"]) def test_agent_function_calling_llm(): from langchain_openai import ChatOpenAI llm = ChatOpenAI(model="gpt-3.5-turbo-0125") with patch.object(llm.client, "create", wraps=llm.client.create) as private_mock: @tool def learn_about_AI(topic) -> float: """Useful for when you need to learn about AI to write an paragraph about it.""" return "AI is a very broad field." agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", tools=[learn_about_AI], llm=ChatOpenAI(model="gpt-4-0125-preview"), function_calling_llm=llm, ) essay = Task( description="Write and then review an small paragraph on AI until it's AMAZING", agent=agent1, ) tasks = [essay] crew = Crew(agents=[agent1], tasks=tasks) crew.kickoff() private_mock.assert_called() def test_agent_count_formatting_error(): from unittest.mock import patch agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", verbose=True, ) parser = CrewAgentParser() parser.agent = agent1 with patch.object(Agent, "increment_formatting_errors") as mock_count_errors: test_text = "This text does not match expected formats." with pytest.raises(OutputParserException): parser.parse(test_text) mock_count_errors.assert_called_once() def test_agent_llm_uses_token_calc_handler_with_llm_has_model_name(): agent1 = Agent( role="test role", goal="test goal", backstory="test backstory", verbose=True, ) assert len(agent1.llm.callbacks) == 1 assert agent1.llm.callbacks[0].__class__.__name__ == "TokenCalcHandler" assert agent1.llm.callbacks[0].model == "gpt-4" assert ( agent1.llm.callbacks[0].token_cost_process.__class__.__name__ == "TokenProcess" )