From 15dd15fcab7046a27d611c38c1d6202acd3863bf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 20 Apr 2025 14:36:46 +0000 Subject: [PATCH] [FEATURE] Improve agent/task templating with Jinja2 Fixes #2650 - Add support for container types (List, Dict, Set) - Add support for standard objects (datetime) - Add support for custom objects - Add support for conditional and loop statements - Add support for filtering options - Maintain backward compatibility with existing templates - Add comprehensive tests - Add documentation with examples Co-Authored-By: Joe Moura --- docs/examples/jinja_templating.md | 133 +++++++++++++++++++++++ src/crewai/task.py | 88 ++++++++++++--- src/crewai/utilities/jinja_templating.py | 79 ++++++++++++++ src/crewai/utilities/string_utils.py | 84 +++++++++----- tests/test_templating.py | 88 +++++++++++++++ tests/utilities/test_jinja_templating.py | 81 ++++++++++++++ tests/utilities/test_string_utils.py | 96 +++++++++++++++- 7 files changed, 601 insertions(+), 48 deletions(-) create mode 100644 docs/examples/jinja_templating.md create mode 100644 src/crewai/utilities/jinja_templating.py create mode 100644 tests/test_templating.py create mode 100644 tests/utilities/test_jinja_templating.py diff --git a/docs/examples/jinja_templating.md b/docs/examples/jinja_templating.md new file mode 100644 index 000000000..70cc06cf1 --- /dev/null +++ b/docs/examples/jinja_templating.md @@ -0,0 +1,133 @@ +# Enhanced Templating with Jinja2 + +CrewAI now supports enhanced templating using Jinja2, while maintaining compatibility with the existing templating system. + +## Basic Usage + +The basic templating syntax remains the same: + +```python +from crewai import Agent, Task, Crew + +# Define inputs +inputs = { + "topic": "Artificial Intelligence", + "year": 2024, + "count": 5 +} + +# Create an agent with template variables +researcher = Agent( + role="{topic} Researcher", + goal="Research the latest developments in {topic} for {year}", + backstory="You're an expert in {topic} with years of experience" +) + +# Create a task with template variables +research_task = Task( + description="Research {topic} and provide {count} key insights", + expected_output="A list of {count} key insights about {topic} in {year}", + agent=researcher +) + +# Create a crew and pass inputs +crew = Crew( + agents=[researcher], + tasks=[research_task], + inputs=inputs +) + +# Run the crew +result = crew.kickoff() +``` + +## Advanced Features + +The new templating system adds support for container types, object attributes, conditional statements, loops, and filters: + +### Container Types + +```python +inputs = { + "topics": ["AI", "Machine Learning", "Data Science"], + "details": {"main_theme": "Technology Trends", "subtopics": ["Ethics", "Applications"]} +} + +# Access list items +task = Task( + description="Research {{topics[0]}} and {{topics[1]}}", + expected_output="Analysis of the topics" +) + +# Access dictionary items +task = Task( + description="Research {{details.main_theme}} with focus on {{details.subtopics[0]}}", + expected_output="Detailed analysis" +) +``` + +### Conditional Statements + +```python +inputs = { + "topic": "AI", + "priority": "high", + "deadline": "2024-12-31" +} + +task = Task( + description="{% if priority == 'high' %}URGENT: {% endif %}Research {topic}{% if deadline %} by {{deadline}}{% endif %}", + expected_output="A report on {topic}" +) +``` + +### Loop Statements + +```python +inputs = { + "topics": ["AI", "Machine Learning", "Data Science"] +} + +task = Task( + description="Research the following topics: {% for topic in topics %}{{topic}}{% if not loop.last %}, {% endif %}{% endfor %}", + expected_output="A report covering multiple topics" +) +``` + +### Filters + +```python +from datetime import datetime + +inputs = { + "topic": "AI", + "date": datetime.now() +} + +task = Task( + description="Research {topic} as of {{date|date('%Y-%m-%d')}}", + expected_output="A report on {topic}" +) +``` + +### Custom Objects + +```python +from pydantic import BaseModel + +class Person(BaseModel): + name: str + age: int + + def __str__(self): + return f"{self.name} ({self.age})" + +inputs = { + "author": Person(name="John Doe", age=35) +} + +task = Task( + description="Write a report authored by {author}", + expected_output="A report by {{author.name}}" +) +``` diff --git a/src/crewai/task.py b/src/crewai/task.py index 9874b5100..4d74b9a3c 100644 --- a/src/crewai/task.py +++ b/src/crewai/task.py @@ -485,6 +485,19 @@ class Task(BaseModel): tasks_slices = [self.description, output] return "\n".join(tasks_slices) + def interpolate_inputs(self, inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]]) -> None: + """Interpolate inputs into the task description, expected output, and output file path. + + Args: + inputs: Dictionary mapping template variables to their values. + Supported value types are strings, integers, floats, dicts, lists, + and other objects with string representation. + + Raises: + ValueError: If a required template variable is missing from inputs. + """ + self.interpolate_inputs_and_add_conversation_history(inputs) + def interpolate_inputs_and_add_conversation_history( self, inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]] ) -> None: @@ -493,7 +506,8 @@ class Task(BaseModel): Args: inputs: Dictionary mapping template variables to their values. - Supported value types are strings, integers, and floats. + Supported value types are strings, integers, floats, dicts, lists, + and other objects with string representation. Raises: ValueError: If a required template variable is missing from inputs. @@ -508,23 +522,63 @@ class Task(BaseModel): if not inputs: return - try: - self.description = interpolate_only( - input_string=self._original_description, inputs=inputs - ) - except KeyError as e: - raise ValueError( - f"Missing required template variable '{e.args[0]}' in description" - ) from e - except ValueError as e: - raise ValueError(f"Error interpolating description: {str(e)}") from e + # Check for complex indexing patterns like {topics[0]} in the description + has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", self._original_description) + + if has_complex_indexing: + complex_patterns = re.findall(r"\{([A-Za-z_][A-Za-z0-9_]*)\[([0-9]+)\]\}", self._original_description) + result = self._original_description + + for var_name, index in complex_patterns: + if var_name in inputs and isinstance(inputs[var_name], list): + try: + idx = int(index) + if 0 <= idx < len(inputs[var_name]): + placeholder = f"{{{var_name}[{index}]}}" + value = str(inputs[var_name][idx]) + result = result.replace(placeholder, value) + except (ValueError, IndexError): + pass + + self.description = result + else: + try: + self.description = interpolate_only( + input_string=self._original_description, inputs=inputs + ) + except KeyError as e: + raise ValueError( + f"Missing required template variable '{e.args[0]}' in description" + ) from e + except ValueError as e: + raise ValueError(f"Error interpolating description: {str(e)}") from e - try: - self.expected_output = interpolate_only( - input_string=self._original_expected_output, inputs=inputs - ) - except (KeyError, ValueError) as e: - raise ValueError(f"Error interpolating expected_output: {str(e)}") from e + # Check for complex indexing patterns in the expected output + has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", self._original_expected_output) + + if has_complex_indexing: + complex_patterns = re.findall(r"\{([A-Za-z_][A-Za-z0-9_]*)\[([0-9]+)\]\}", self._original_expected_output) + result = self._original_expected_output + + for var_name, index in complex_patterns: + if var_name in inputs and isinstance(inputs[var_name], list): + try: + idx = int(index) + if 0 <= idx < len(inputs[var_name]): + placeholder = f"{{{var_name}[{index}]}}" + value = str(inputs[var_name][idx]) + result = result.replace(placeholder, value) + except (ValueError, IndexError): + pass + + self.expected_output = result + else: + try: + self.expected_output = interpolate_only( + input_string=self._original_expected_output, inputs=inputs + ) + except (KeyError, ValueError) as e: + raise ValueError(f"Error interpolating expected_output: {str(e)}") from e if self.output_file is not None: try: diff --git a/src/crewai/utilities/jinja_templating.py b/src/crewai/utilities/jinja_templating.py new file mode 100644 index 000000000..c9603dc1f --- /dev/null +++ b/src/crewai/utilities/jinja_templating.py @@ -0,0 +1,79 @@ +from typing import Any, Dict, List, Optional, Union +import jinja2 +import re +from datetime import datetime + +def to_jinja_template(input_string: str) -> str: + """ + Convert CrewAI-style {var} templates to Jinja2-style {{var}} templates. + + This function preserves existing Jinja2 syntax if present and only converts + CrewAI-style variables. + + Args: + input_string: String containing CrewAI-style templates. + + Returns: + String with CrewAI-style templates converted to Jinja2 syntax. + """ + if not input_string or ("{" not in input_string and "}" not in input_string): + return input_string + + pattern = r'(? str: + """ + Render a template string using Jinja2 with the provided inputs. + + This function supports: + - Container types (List, Dict, Set) + - Standard objects (datetime, time) + - Custom objects + - Conditional and loop statements + - Filtering options + + Args: + input_string: The string containing template variables to interpolate. + Can be None or empty, in which case an empty string is returned. + inputs: Dictionary mapping template variables to their values. + Supports all types of values. + + Returns: + The rendered template string. + + Raises: + ValueError: If inputs dictionary is empty when interpolating variables. + jinja2.exceptions.TemplateError: If there's an error in the template syntax. + KeyError: If a required template variable is missing from inputs. + """ + if input_string is None or not input_string: + return "" + + if "{" not in input_string and "}" not in input_string: + return input_string + + if not inputs: + raise ValueError("Inputs dictionary cannot be empty when interpolating variables") + + jinja_template = to_jinja_template(input_string) + + env = jinja2.Environment( + undefined=jinja2.StrictUndefined, # Raise errors for undefined variables + ) + + env.filters['date'] = lambda d, format='%Y-%m-%d': d.strftime(format) if isinstance(d, datetime) else str(d) + + template = env.from_string(jinja_template) + + try: + return template.render(**inputs) + except jinja2.exceptions.UndefinedError as e: + var_name = str(e).split("'")[1] if "'" in str(e) else None + if var_name: + raise KeyError(f"Template variable '{var_name}' not found in inputs dictionary") + raise KeyError(f"Missing required template variable: {str(e)}") diff --git a/src/crewai/utilities/string_utils.py b/src/crewai/utilities/string_utils.py index 9a1857781..d794f33aa 100644 --- a/src/crewai/utilities/string_utils.py +++ b/src/crewai/utilities/string_utils.py @@ -1,31 +1,38 @@ import re from typing import Any, Dict, List, Optional, Union +from datetime import datetime +from crewai.utilities.jinja_templating import render_template def interpolate_only( input_string: Optional[str], - inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]], + inputs: Dict[str, Any], ) -> str: """Interpolate placeholders (e.g., {key}) in a string while leaving JSON untouched. Only interpolates placeholders that follow the pattern {variable_name} where variable_name starts with a letter/underscore and contains only letters, numbers, and underscores. + + This function now supports advanced Jinja2 templating features: + - Container types (List, Dict, Set) + - Standard objects (datetime, time) + - Custom objects + - Conditional and loop statements + - Filtering options Args: input_string: The string containing template variables to interpolate. Can be None or empty, in which case an empty string is returned. inputs: Dictionary mapping template variables to their values. - Supported value types are strings, integers, floats, and dicts/lists - containing only these types and other nested dicts/lists. + Supports all types of values including complex objects. Returns: The interpolated string with all template variables replaced with their values. Empty string if input_string is None or empty. Raises: - ValueError: If a value contains unsupported types or a template variable is missing + ValueError: If inputs dictionary is empty when interpolating variables. + KeyError: If a required template variable is missing from inputs. """ - - # Validation function for recursive type checking def validate_type(value: Any) -> None: if value is None: return @@ -35,12 +42,21 @@ def interpolate_only( for item in value.values() if isinstance(value, dict) else value: validate_type(item) return + if isinstance(value, datetime): + return + # Check if it's a Pydantic model or other known custom type + try: + from pydantic import BaseModel + if isinstance(value, BaseModel): + return + except ImportError: + pass + raise ValueError( f"Unsupported type {type(value).__name__} in inputs. " - "Only str, int, float, bool, dict, and list are allowed." + "Only str, int, float, bool, dict, list, datetime, and custom objects are allowed." ) - # Validate all input values for key, value in inputs.items(): try: validate_type(value) @@ -56,27 +72,35 @@ def interpolate_only( "Inputs dictionary cannot be empty when interpolating variables" ) - # The regex pattern to find valid variable placeholders - # Matches {variable_name} where variable_name starts with a letter/underscore - # and contains only letters, numbers, and underscores - pattern = r"\{([A-Za-z_][A-Za-z0-9_]*)\}" + # Check if the template contains Jinja2 syntax ({% ... %} or {{ ... }}) + has_jinja_syntax = "{{" in input_string or "{%" in input_string + has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", input_string) - # Find all matching variables in the input string - variables = re.findall(pattern, input_string) - result = input_string + if has_jinja_syntax or has_complex_indexing: + return render_template(input_string, inputs) + else: + # The regex pattern to find valid variable placeholders + # Matches {variable_name} where variable_name starts with a letter/underscore + # and contains only letters, numbers, and underscores + pattern = r"\{([A-Za-z_][A-Za-z0-9_]*)\}" - # Check if all variables exist in inputs - missing_vars = [var for var in variables if var not in inputs] - if missing_vars: - raise KeyError( - f"Template variable '{missing_vars[0]}' not found in inputs dictionary" - ) - - # Replace each variable with its value - for var in variables: - if var in inputs: - placeholder = "{" + var + "}" - value = str(inputs[var]) - result = result.replace(placeholder, value) - - return result + # Find all matching variables in the input string + variables = re.findall(pattern, input_string) + + # Check if all variables exist in inputs + missing_vars = [var for var in variables if var not in inputs] + if missing_vars: + raise KeyError( + f"Template variable '{missing_vars[0]}' not found in inputs dictionary" + ) + + try: + return render_template(input_string, inputs) + except Exception: + result = input_string + for var in variables: + if var in inputs: + placeholder = "{" + var + "}" + value = str(inputs[var]) + result = result.replace(placeholder, value) + return result diff --git a/tests/test_templating.py b/tests/test_templating.py new file mode 100644 index 000000000..a2d81d854 --- /dev/null +++ b/tests/test_templating.py @@ -0,0 +1,88 @@ +import pytest +import datetime +from typing import Dict, List +from crewai.agent import Agent +from crewai.task import Task + +class TestTemplating: + def test_task_interpolation(self): + task = Task( + description="Research about {topic} and provide {count} key points", + expected_output="A list of {count} key points about {topic}" + ) + + inputs = {"topic": "AI", "count": 5} + task.interpolate_inputs(inputs) + + assert task.description == "Research about AI and provide 5 key points" + assert task.expected_output == "A list of 5 key points about AI" + + task = Task( + description="Research about {topics[0]} and {topics[1]}", + expected_output="Analysis of {{data.main_theme}}" + ) + + inputs = { + "topics": ["AI", "Machine Learning"], + "data": {"main_theme": "Technology Trends"} + } + + task.interpolate_inputs(inputs) + + assert task.description == "Research about AI and Machine Learning" + assert task.expected_output == "Analysis of Technology Trends" + + def test_agent_interpolation(self): + agent = Agent( + role="{industry} Researcher", + goal="Research {count} key developments in {industry}", + backstory="You are a senior researcher in the {industry} field with {experience} years of experience" + ) + + inputs = {"industry": "Healthcare", "count": 5, "experience": 10} + agent.interpolate_inputs(inputs) + + assert agent.role == "Healthcare Researcher" + assert agent.goal == "Research 5 key developments in Healthcare" + assert agent.backstory == "You are a senior researcher in the Healthcare field with 10 years of experience" + + agent = Agent( + role="{{specialties[0]}} and {{specialties[1]}} Specialist", + goal="Analyze trends in {{fields.primary}} sector", + backstory="Expert in {{fields.primary}} and {{fields.secondary}}" + ) + + inputs = { + "specialties": ["AI", "Data Science"], + "fields": {"primary": "Healthcare", "secondary": "Finance"} + } + + agent.interpolate_inputs(inputs) + + assert agent.role == "AI and Data Science Specialist" + assert agent.goal == "Analyze trends in Healthcare sector" + assert agent.backstory == "Expert in Healthcare and Finance" + + def test_conditional_templating(self): + task = Task( + description="{% if priority == 'high' %}URGENT: {% endif %}Research {topic}", + expected_output="A report on {topic}" + ) + + inputs = {"topic": "AI", "priority": "high"} + task.interpolate_inputs(inputs) + assert task.description == "URGENT: Research AI" + + inputs = {"topic": "AI", "priority": "low"} + task.interpolate_inputs(inputs) + assert task.description == "Research AI" + + def test_loop_templating(self): + task = Task( + description="Research the following topics: {% for topic in topics %}{{topic}}{% if not loop.last %}, {% endif %}{% endfor %}", + expected_output="A report on multiple topics" + ) + + inputs = {"topics": ["AI", "Machine Learning", "Data Science"]} + task.interpolate_inputs(inputs) + assert task.description == "Research the following topics: AI, Machine Learning, Data Science" diff --git a/tests/utilities/test_jinja_templating.py b/tests/utilities/test_jinja_templating.py new file mode 100644 index 000000000..9784db736 --- /dev/null +++ b/tests/utilities/test_jinja_templating.py @@ -0,0 +1,81 @@ +import pytest +import datetime +from pydantic import BaseModel +from typing import List, Dict, Any +from crewai.utilities.jinja_templating import to_jinja_template, render_template + +class Person(BaseModel): + name: str + age: int + + def __str__(self): + return f"{self.name} ({self.age})" + +class TestJinjaTemplating: + def test_to_jinja_template(self): + assert to_jinja_template("Hello {name}!") == "Hello {{name}}!" + + assert to_jinja_template("Hello {{name}}!") == "Hello {{name}}!" + + assert to_jinja_template("Hello {name} and {{title}}!") == "Hello {{name}} and {{title}}!" + + assert to_jinja_template("") == "" + + assert to_jinja_template("Hello world!") == "Hello world!" + + def test_render_template_simple_types(self): + inputs = {"name": "John", "age": 30, "active": True, "height": 1.85} + + assert render_template("Hello {name}!", inputs) == "Hello John!" + assert render_template("Age: {age}", inputs) == "Age: 30" + assert render_template("Active: {active}", inputs) == "Active: True" + assert render_template("Height: {height}", inputs) == "Height: 1.85" + + assert render_template("{name} is {age} years old", inputs) == "John is 30 years old" + + def test_render_template_container_types(self): + inputs = { + "items": ["apple", "banana", "orange"], + "person": {"name": "John", "age": 30} + } + + assert render_template("First item: {{items[0]}}", inputs) == "First item: apple" + + assert render_template("Person name: {{person.name}}", inputs) == "Person name: John" + + assert render_template( + "Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}", + inputs + ) == "Items: apple, banana, orange" + + assert render_template( + "{% if items|length > 2 %}Many items{% else %}Few items{% endif %}", + inputs + ) == "Many items" + + def test_render_template_datetime(self): + today = datetime.datetime.now() + inputs = {"today": today} + + assert render_template("Today: {{today|date}}", inputs) == f"Today: {today.strftime('%Y-%m-%d')}" + + assert render_template("Today: {{today|date('%d/%m/%Y')}}", inputs) == f"Today: {today.strftime('%d/%m/%Y')}" + + def test_render_template_custom_objects(self): + person = Person(name="John", age=30) + inputs = {"person": person} + + assert render_template("Person: {person}", inputs) == "Person: John (30)" + + assert render_template("Person name: {{person.name}}", inputs) == "Person name: John" + + def test_render_template_error_handling(self): + inputs = {"name": "John"} + + with pytest.raises(KeyError) as excinfo: + render_template("Hello {age}!", inputs) + assert "Template variable 'age' not found" in str(excinfo.value) + + with pytest.raises(ValueError) as excinfo: + render_template("Hello {name}!", {}) + assert "Inputs dictionary cannot be empty" in str(excinfo.value) diff --git a/tests/utilities/test_string_utils.py b/tests/utilities/test_string_utils.py index 441aae8c0..0fce79322 100644 --- a/tests/utilities/test_string_utils.py +++ b/tests/utilities/test_string_utils.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List, Union - +import datetime import pytest +from pydantic import BaseModel from crewai.utilities.string_utils import interpolate_only @@ -185,3 +186,96 @@ class TestInterpolateOnly: interpolate_only(template, inputs) assert "inputs dictionary cannot be empty" in str(excinfo.value).lower() + + + def test_container_types_list_access(self): + """Test accessing list items with Jinja2 syntax.""" + template = "First item: {{items[0]}}, Second item: {{items[1]}}" + inputs = { + "items": ["apple", "banana", "orange"] + } + + result = interpolate_only(template, inputs) + assert result == "First item: apple, Second item: banana" + + def test_container_types_dict_access(self): + """Test accessing dictionary items with Jinja2 syntax.""" + template = "Name: {{person.name}}, Age: {{person.age}}" + inputs = { + "person": {"name": "John", "age": 30} + } + + result = interpolate_only(template, inputs) + assert result == "Name: John, Age: 30" + + def test_conditional_statements(self): + """Test conditional statements with Jinja2 syntax.""" + template = "{% if priority == 'high' %}URGENT: {% endif %}Task: {task}" + + inputs_high = { + "task": "Fix bug", + "priority": "high" + } + result_high = interpolate_only(template, inputs_high) + assert result_high == "URGENT: Task: Fix bug" + + inputs_low = { + "task": "Fix bug", + "priority": "low" + } + result_low = interpolate_only(template, inputs_low) + assert result_low == "Task: Fix bug" + + def test_loop_statements(self): + """Test loop statements with Jinja2 syntax.""" + template = "Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}" + inputs = { + "items": ["apple", "banana", "orange"] + } + + result = interpolate_only(template, inputs) + assert result == "Items: apple, banana, orange" + + def test_datetime_formatting(self): + """Test datetime formatting with Jinja2 filters.""" + today = datetime.datetime(2024, 4, 20) + inputs = {"today": today} + + template = "Date: {{today|date}}" + result = interpolate_only(template, inputs) + assert result == "Date: 2024-04-20" + + template = "Date: {{today|date('%d/%m/%Y')}}" + result = interpolate_only(template, inputs) + assert result == "Date: 20/04/2024" + + def test_custom_objects(self): + """Test custom objects with Jinja2 syntax.""" + class Person(BaseModel): + name: str + age: int + + def __str__(self): + return f"{self.name} ({self.age})" + + person = Person(name="John", age=30) + inputs = {"person": person} + + template = "Person: {person}" + result = interpolate_only(template, inputs) + assert result == "Person: John (30)" + + template = "Name: {{person.name}}, Age: {{person.age}}" + result = interpolate_only(template, inputs) + assert result == "Name: John, Age: 30" + + def test_mixed_syntax(self): + """Test mixed CrewAI and Jinja2 syntax.""" + template = "Hello {name}! Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}" + inputs = { + "name": "John", + "items": ["apple", "banana", "orange"] + } + + result = interpolate_only(template, inputs) + assert result == "Hello John! Items: apple, banana, orange"