import pytest from pydantic import BaseModel from typing import List from crewai import Agent, Task from crewai.lite_agent import LiteAgent from crewai.utilities.prompts import Prompts from crewai.utilities import I18N from crewai.tools import BaseTool class ResearchOutput(BaseModel): summary: str key_findings: List[str] confidence_score: float class CodeReviewOutput(BaseModel): issues: List[str] recommendations: List[str] class MockTool(BaseTool): name: str = "mock_tool" description: str = "A mock tool for testing" def _run(self, query: str) -> str: return f"Mock result for: {query}" class TestPromptCustomizationDocs: """Test suite for prompt customization documentation examples.""" @pytest.fixture(autouse=True) def setup(self): """Set up test fixtures for isolation.""" self.i18n = I18N() def test_custom_system_and_prompt_templates(self): """Test basic custom template functionality. Validates: - Custom system template assignment - Custom prompt template assignment - Custom response template assignment - System prompt flag configuration """ system_template = """{{ .System }} Additional context: You are working in a production environment. Always prioritize accuracy and provide detailed explanations.""" prompt_template = """{{ .Prompt }} Remember to validate your approach before proceeding.""" response_template = """Please format your response as follows: {{ .Response }} End of response.""" agent = Agent( role="Data Analyst", goal="Analyze data with precision and accuracy", backstory="You are an experienced data analyst with expertise in statistical analysis.", system_template=system_template, prompt_template=prompt_template, response_template=response_template, use_system_prompt=True, llm="gpt-4o-mini" ) assert agent.system_template == system_template assert agent.prompt_template == prompt_template assert agent.response_template == response_template assert agent.use_system_prompt is True def test_system_user_prompt_split(self): """Test system/user prompt separation.""" agent = Agent( role="Research Assistant", goal="Conduct thorough research on given topics", backstory="You are a meticulous researcher with access to various information sources.", use_system_prompt=True, llm="gpt-4o-mini" ) prompts = Prompts( i18n=I18N(), has_tools=False, use_system_prompt=True, agent=agent ) prompt_dict = prompts.task_execution() assert "system" in prompt_dict assert "user" in prompt_dict assert "You are Research Assistant" in prompt_dict["system"] assert agent.goal in prompt_dict["system"] def test_structured_output_with_pydantic(self): """Test structured output using Pydantic models.""" class ResearchOutput(BaseModel): summary: str key_findings: List[str] confidence_score: float agent = Agent( role="Research Assistant", goal="Conduct thorough research", backstory="You are a meticulous researcher.", llm="gpt-4o-mini" ) task = Task( description="Research the latest trends in AI development", expected_output="A structured research report", output_pydantic=ResearchOutput, agent=agent ) assert task.output_pydantic == ResearchOutput assert task.expected_output == "A structured research report" def test_custom_format_instructions(self): """Test custom output format instructions using output_json. Validates: - Task can be configured with structured JSON output using Pydantic models - Output format is properly stored and accessible via _get_output_format method """ class SalesAnalysisOutput(BaseModel): total_sales: float growth_rate: str top_products: List[str] recommendations: str task = Task( description="Analyze the quarterly sales data", expected_output="Analysis in JSON format with specific fields", output_json=SalesAnalysisOutput, agent=Agent( role="Sales Analyst", goal="Analyze sales data", backstory="You are a sales data expert.", llm="gpt-4o-mini" ) ) assert task.output_json == SalesAnalysisOutput assert task._get_output_format().value == "json" def test_stop_words_configuration(self): """Test stop words configuration through response template. Validates: - Response template is properly stored on agent - Agent has create_agent_executor method for setting up execution - Stop words are configured based on response template content """ response_template = """Provide your analysis: {{ .Response }} ---END---""" agent = Agent( role="Analyst", goal="Perform detailed analysis", backstory="You are an expert analyst.", response_template=response_template, llm="gpt-4o-mini" ) assert agent.response_template == response_template assert hasattr(agent, 'create_agent_executor') assert callable(getattr(agent, 'create_agent_executor')) agent.create_agent_executor() assert agent.agent_executor is not None def test_lite_agent_prompt_customization(self): """Test LiteAgent prompt customization.""" class CodeReviewOutput(BaseModel): issues_found: List[str] severity: str recommendations: List[str] lite_agent = LiteAgent( role="Code Reviewer", goal="Review code for quality and security", backstory="You are an experienced software engineer specializing in code review.", response_format=CodeReviewOutput, llm="gpt-4o-mini" ) system_prompt = lite_agent._get_default_system_prompt() assert "Code Reviewer" in system_prompt assert "Review code for quality and security" in system_prompt assert "experienced software engineer" in system_prompt def test_multi_language_support(self): """Test custom templates for different languages.""" spanish_system_template = """{{ .System }} Instrucciones adicionales: Responde siempre en español y proporciona explicaciones detalladas.""" agent = Agent( role="Asistente de Investigación", goal="Realizar investigación exhaustiva en español", backstory="Eres un investigador experimentado que trabaja en español.", system_template=spanish_system_template, use_system_prompt=True, llm="gpt-4o-mini" ) assert agent.system_template == spanish_system_template assert "español" in agent.system_template def test_domain_specific_formatting(self): """Test domain-specific response formatting. Validates: - Domain-specific templates can be applied - Response templates support specialized formatting """ medical_response_template = """MEDICAL ANALYSIS REPORT {{ .Response }} DISCLAIMER: This analysis is for informational purposes only.""" medical_agent = Agent( role="Medical Data Analyst", goal="Analyze medical data with clinical precision", backstory="You are a certified medical data analyst with 10 years of experience.", response_template=medical_response_template, use_system_prompt=True, llm="gpt-4o-mini" ) assert "MEDICAL ANALYSIS REPORT" in medical_agent.response_template assert "DISCLAIMER" in medical_agent.response_template def test_prompt_components_assembly(self): """Test how prompt components are assembled.""" agent = Agent( role="Test Agent", goal="Test goal", backstory="Test backstory", llm="gpt-4o-mini" ) prompts = Prompts( i18n=I18N(), has_tools=False, agent=agent ) prompt_dict = prompts.task_execution() assert "prompt" in prompt_dict assert "Test Agent" in prompt_dict["prompt"] assert "Test goal" in prompt_dict["prompt"] assert "Test backstory" in prompt_dict["prompt"] def test_tools_vs_no_tools_prompts(self): """Test prompt generation differences between agents with and without tools. Validates: - Agents without tools use 'no_tools' template slice - Agents with tools use 'tools' template slice - Prompt generation differs based on tool availability """ agent_no_tools = Agent( role="Analyst", goal="Analyze data", backstory="Expert analyst", llm="gpt-4o-mini" ) mock_tool = MockTool() agent_with_tools = Agent( role="Analyst", goal="Analyze data", backstory="Expert analyst", tools=[mock_tool], llm="gpt-4o-mini" ) assert len(agent_with_tools.tools) == 1 assert agent_with_tools.tools[0].name == "mock_tool" prompts_no_tools = Prompts(i18n=I18N(), has_tools=False, agent=agent_no_tools) prompts_with_tools = Prompts(i18n=I18N(), has_tools=True, agent=agent_with_tools) prompt_dict_no_tools = prompts_no_tools.task_execution() prompt_dict_with_tools = prompts_with_tools.task_execution() assert isinstance(prompt_dict_no_tools, dict) assert isinstance(prompt_dict_with_tools, dict) assert prompt_dict_no_tools != prompt_dict_with_tools def test_template_placeholder_replacement(self): """Test that template placeholders are properly replaced.""" system_template = "SYSTEM: {{ .System }} - Custom addition" prompt_template = "PROMPT: {{ .Prompt }} - Custom addition" response_template = "RESPONSE: {{ .Response }} - Custom addition" agent = Agent( role="Template Tester", goal="Test template replacement", backstory="You test templates.", system_template=system_template, prompt_template=prompt_template, response_template=response_template, llm="gpt-4o-mini" ) prompts = Prompts( i18n=I18N(), has_tools=False, system_template=system_template, prompt_template=prompt_template, response_template=response_template, agent=agent ) prompt_dict = prompts.task_execution() assert "SYSTEM:" in prompt_dict["prompt"] assert "PROMPT:" in prompt_dict["prompt"] assert "RESPONSE:" in prompt_dict["prompt"] assert "Custom addition" in prompt_dict["prompt"] def test_verbose_mode_configuration(self): """Test verbose mode for debugging prompts. Validates: - verbose=True parameter can be set on agents - Verbose mode configuration is properly stored """ agent = Agent( role="Debug Agent", goal="Help debug prompt issues", backstory="You are a debugging specialist.", verbose=True, llm="gpt-4o-mini" ) assert agent.verbose is True def test_i18n_slice_access(self): """Test internationalization slice access. Validates: - I18N class provides access to template slices - Template slices contain expected prompt components """ i18n = I18N() role_playing_slice = i18n.slice("role_playing") observation_slice = i18n.slice("observation") tools_slice = i18n.slice("tools") no_tools_slice = i18n.slice("no_tools") assert "You are {role}" in role_playing_slice assert "Your personal goal is: {goal}" in role_playing_slice assert "\nObservation:" == observation_slice assert "Action:" in tools_slice assert "Final Answer:" in no_tools_slice @pytest.mark.parametrize("role,goal,backstory", [ ("Analyst", "Analyze data", "Expert analyst"), ("Researcher", "Find facts", "Experienced researcher"), ("Writer", "Create content", "Professional writer"), ]) def test_agent_initialization_parametrized(self, role, goal, backstory): """Test agent initialization with different role combinations. Validates: - Agents can be created with various role/goal/backstory combinations - All parameters are properly stored """ agent = Agent(role=role, goal=goal, backstory=backstory, llm="gpt-4o-mini") assert agent.role == role assert agent.goal == goal assert agent.backstory == backstory def test_default_template_behavior(self): """Test behavior when no custom templates are provided. Validates: - Agents work correctly with default templates - Default templates from translations/en.json are used """ agent = Agent( role="Default Agent", goal="Test default behavior", backstory="Testing default templates", llm="gpt-4o-mini" ) assert agent.system_template is None assert agent.prompt_template is None assert agent.response_template is None prompts = Prompts(i18n=self.i18n, has_tools=False, agent=agent) prompt_dict = prompts.task_execution() assert isinstance(prompt_dict, dict) def test_incomplete_template_definitions(self): """Test behavior with incomplete template definitions. Validates: - Agents handle partial template customization gracefully - Missing templates fall back to defaults """ agent_partial = Agent( role="Partial Agent", goal="Test partial templates", backstory="Testing incomplete templates", system_template="{{ .System }} - Custom system only", llm="gpt-4o-mini" ) assert agent_partial.system_template is not None assert agent_partial.prompt_template is None assert agent_partial.response_template is None prompts = Prompts(i18n=self.i18n, has_tools=False, agent=agent_partial) prompt_dict = prompts.task_execution() assert isinstance(prompt_dict, dict) def test_lite_agent_with_and_without_tools(self): """Test LiteAgent behavior with and without tools. Validates: - LiteAgent can be created with and without tools - Tool configuration is properly stored """ lite_agent_no_tools = LiteAgent( role="Reviewer", goal="Review content", backstory="Expert reviewer", llm="gpt-4o-mini" ) mock_tool = MockTool() lite_agent_with_tools = LiteAgent( role="Reviewer", goal="Review content", backstory="Expert reviewer", tools=[mock_tool], llm="gpt-4o-mini" ) assert lite_agent_no_tools.role == "Reviewer" assert lite_agent_with_tools.role == "Reviewer" assert len(lite_agent_with_tools.tools) == 1 assert lite_agent_with_tools.tools[0].name == "mock_tool" def test_malformed_template_handling(self): """Test handling of malformed or incomplete templates gracefully. Validates: - Agent creation succeeds with malformed templates - Missing placeholders don't cause failures - System handles edge cases gracefully """ incomplete_template = "This is a template without placeholders" agent = Agent( role="Test Agent", goal="Test incomplete templates", backstory="Agent for testing edge cases.", system_template=incomplete_template, llm="gpt-4o-mini" ) assert agent.system_template == incomplete_template assert agent.role == "Test Agent" agent_empty = Agent( role="Empty Template Agent", goal="Test empty templates", backstory="Agent with empty template.", response_template="", llm="gpt-4o-mini" ) assert agent_empty.response_template == "" def test_agent_creation_with_missing_required_parameters(self): """Test agent creation behavior when required parameters are missing. Validates: - Agent creation requires role, goal, and backstory - Appropriate errors are raised for missing parameters """ with pytest.raises((TypeError, ValueError)): Agent(llm="gpt-4o-mini") # Missing required parameters with pytest.raises((TypeError, ValueError)): Agent(role="Test", llm="gpt-4o-mini") # Missing goal and backstory