Fix Jinja2 templating for loop variables and mixed syntax

Co-Authored-By: Joe Moura <joao@crewai.com>

docs/examples/jinja_templating.md

@@ -0,0 +1,133 @@
+# Enhanced Templating with Jinja2
+
+CrewAI now supports enhanced templating using Jinja2, while maintaining compatibility with the existing templating system.
+
+## Basic Usage
+
+The basic templating syntax remains the same:
+
+```python
+from crewai import Agent, Task, Crew
+
+# Define inputs
+inputs = {
+    "topic": "Artificial Intelligence",
+    "year": 2024,
+    "count": 5
+}
+
+# Create an agent with template variables
+researcher = Agent(
+    role="{topic} Researcher",
+    goal="Research the latest developments in {topic} for {year}",
+    backstory="You're an expert in {topic} with years of experience"
+)
+
+# Create a task with template variables
+research_task = Task(
+    description="Research {topic} and provide {count} key insights",
+    expected_output="A list of {count} key insights about {topic} in {year}",
+    agent=researcher
+)
+
+# Create a crew and pass inputs
+crew = Crew(
+    agents=[researcher],
+    tasks=[research_task],
+    inputs=inputs
+)
+
+# Run the crew
+result = crew.kickoff()
+```
+
+## Advanced Features
+
+The new templating system adds support for container types, object attributes, conditional statements, loops, and filters:
+
+### Container Types
+
+```python
+inputs = {
+    "topics": ["AI", "Machine Learning", "Data Science"],
+    "details": {"main_theme": "Technology Trends", "subtopics": ["Ethics", "Applications"]}
+}
+
+# Access list items
+task = Task(
+    description="Research {{topics[0]}} and {{topics[1]}}",
+    expected_output="Analysis of the topics"
+)
+
+# Access dictionary items
+task = Task(
+    description="Research {{details.main_theme}} with focus on {{details.subtopics[0]}}",
+    expected_output="Detailed analysis"
+)
+```
+
+### Conditional Statements
+
+```python
+inputs = {
+    "topic": "AI",
+    "priority": "high",
+    "deadline": "2024-12-31"
+}
+
+task = Task(
+    description="{% if priority == 'high' %}URGENT: {% endif %}Research {topic}{% if deadline %} by {{deadline}}{% endif %}",
+    expected_output="A report on {topic}"
+)
+```
+
+### Loop Statements
+
+```python
+inputs = {
+    "topics": ["AI", "Machine Learning", "Data Science"]
+}
+
+task = Task(
+    description="Research the following topics: {% for topic in topics %}{{topic}}{% if not loop.last %}, {% endif %}{% endfor %}",
+    expected_output="A report covering multiple topics"
+)
+```
+
+### Filters
+
+```python
+from datetime import datetime
+
+inputs = {
+    "topic": "AI",
+    "date": datetime.now()
+}
+
+task = Task(
+    description="Research {topic} as of {{date|date('%Y-%m-%d')}}",
+    expected_output="A report on {topic}"
+)
+```
+
+### Custom Objects
+
+```python
+from pydantic import BaseModel
+
+class Person(BaseModel):
+    name: str
+    age: int
+    
+    def __str__(self):
+        return f"{self.name} ({self.age})"
+
+inputs = {
+    "author": Person(name="John Doe", age=35)
+}
+
+task = Task(
+    description="Write a report authored by {author}",
+    expected_output="A report by {{author.name}}"
+)
+```

src/crewai/task.py

@@ -485,6 +485,19 @@ class Task(BaseModel):
         tasks_slices = [self.description, output]
         return "\n".join(tasks_slices)
 
+    def interpolate_inputs(self, inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]]) -> None:
+        """Interpolate inputs into the task description, expected output, and output file path.
+
+        Args:
+            inputs: Dictionary mapping template variables to their values.
+               Supported value types are strings, integers, floats, dicts, lists,
+               and other objects with string representation.
+
+        Raises:
+            ValueError: If a required template variable is missing from inputs.
+        """
+        self.interpolate_inputs_and_add_conversation_history(inputs)
+        
     def interpolate_inputs_and_add_conversation_history(
         self, inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]]
     ) -> None:
@@ -493,7 +506,8 @@ class Task(BaseModel):
 
         Args:
             inputs: Dictionary mapping template variables to their values.
-                   Supported value types are strings, integers, and floats.
+                   Supported value types are strings, integers, floats, dicts, lists,
+                   and other objects with string representation.
 
         Raises:
             ValueError: If a required template variable is missing from inputs.
@@ -508,23 +522,65 @@ class Task(BaseModel):
         if not inputs:
             return
 
-        try:
-            self.description = interpolate_only(
-                input_string=self._original_description, inputs=inputs
-            )
-        except KeyError as e:
-            raise ValueError(
-                f"Missing required template variable '{e.args[0]}' in description"
-            ) from e
-        except ValueError as e:
-            raise ValueError(f"Error interpolating description: {str(e)}") from e
+        # Check for complex indexing patterns like {topics[0]} in the description
+        has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", self._original_description)
+        
+        if has_complex_indexing:
+            complex_patterns = re.findall(r"\{([A-Za-z_][A-Za-z0-9_]*)\[([0-9]+)\]\}", self._original_description)
+            result = self._original_description
+            
+            for var_name, index in complex_patterns:
+                if var_name in inputs and isinstance(inputs[var_name], list):
+                    try:
+                        idx = int(index)
+                        list_value = inputs[var_name]
+                        if isinstance(list_value, list) and 0 <= idx < len(list_value):
+                            placeholder = f"{{{var_name}[{index}]}}"
+                            value = str(list_value[idx])
+                            result = result.replace(placeholder, value)
+                    except (ValueError, IndexError):
+                        pass
+            
+            self.description = result
+        else:
+            try:
+                self.description = interpolate_only(
+                    input_string=self._original_description, inputs=inputs
+                )
+            except KeyError as e:
+                raise ValueError(
+                    f"Missing required template variable '{e.args[0]}' in description"
+                ) from e
+            except ValueError as e:
+                raise ValueError(f"Error interpolating description: {str(e)}") from e
 
-        try:
-            self.expected_output = interpolate_only(
-                input_string=self._original_expected_output, inputs=inputs
-            )
-        except (KeyError, ValueError) as e:
-            raise ValueError(f"Error interpolating expected_output: {str(e)}") from e
+        # Check for complex indexing patterns in the expected output
+        has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", self._original_expected_output)
+        
+        if has_complex_indexing:
+            complex_patterns = re.findall(r"\{([A-Za-z_][A-Za-z0-9_]*)\[([0-9]+)\]\}", self._original_expected_output)
+            result = self._original_expected_output
+            
+            for var_name, index in complex_patterns:
+                if var_name in inputs and isinstance(inputs[var_name], list):
+                    try:
+                        idx = int(index)
+                        list_value = inputs[var_name]
+                        if isinstance(list_value, list) and 0 <= idx < len(list_value):
+                            placeholder = f"{{{var_name}[{index}]}}"
+                            value = str(list_value[idx])
+                            result = result.replace(placeholder, value)
+                    except (ValueError, IndexError):
+                        pass
+            
+            self.expected_output = result
+        else:
+            try:
+                self.expected_output = interpolate_only(
+                    input_string=self._original_expected_output, inputs=inputs
+                )
+            except (KeyError, ValueError) as e:
+                raise ValueError(f"Error interpolating expected_output: {str(e)}") from e
 
         if self.output_file is not None:
             try:

src/crewai/utilities/jinja_templating.py

@@ -0,0 +1,98 @@
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+import jinja2
+
+
+def to_jinja_template(input_string: str) -> str:
+    """
+    Convert CrewAI-style {var} templates to Jinja2-style {{var}} templates.
+    
+    This function preserves existing Jinja2 syntax if present and only converts
+    CrewAI-style variables.
+    
+    Args:
+        input_string: String containing CrewAI-style templates.
+        
+    Returns:
+        String with CrewAI-style templates converted to Jinja2 syntax.
+    """
+    if not input_string or ("{" not in input_string and "}" not in input_string):
+        return input_string
+        
+    pattern = r'(?<!\{)\{([A-Za-z_][A-Za-z0-9_]*)\}(?!\})'
+    
+    return re.sub(pattern, r'{{\1}}', input_string)
+
+def render_template(
+    input_string: Optional[str],
+    inputs: Dict[str, Any],
+) -> str:
+    """
+    Render a template string using Jinja2 with the provided inputs.
+    
+    This function supports:
+    - Container types (List, Dict, Set)
+    - Standard objects (datetime, time)
+    - Custom objects
+    - Conditional and loop statements
+    - Filtering options
+    
+    Args:
+        input_string: The string containing template variables to interpolate.
+                     Can be None or empty, in which case an empty string is returned.
+        inputs: Dictionary mapping template variables to their values.
+               Supports all types of values.
+               
+    Returns:
+        The rendered template string.
+        
+    Raises:
+        ValueError: If inputs dictionary is empty when interpolating variables.
+        jinja2.exceptions.TemplateError: If there's an error in the template syntax.
+        KeyError: If a required template variable is missing from inputs.
+    """
+    if input_string is None or not input_string:
+        return ""
+        
+    if "{" not in input_string and "}" not in input_string:
+        return input_string
+        
+    if not inputs:
+        raise ValueError("Inputs dictionary cannot be empty when interpolating variables")
+        
+    jinja_template = to_jinja_template(input_string)
+    
+    # Create a custom undefined class that allows loop variables
+    class LoopUndefined(jinja2.StrictUndefined):
+        """Custom undefined class that allows loop variables."""
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            
+        def __str__(self):
+            if self._undefined_name in ('loop', 'item', 'topic'):
+                return ''
+            return super().__str__()
+            
+        def __getattr__(self, name):
+            if self._undefined_name in ('loop', 'item', 'topic'):
+                return self
+            return super().__getattr__(name)
+    
+    env = jinja2.Environment(
+        undefined=LoopUndefined,  # Use custom undefined class for loop variables
+        autoescape=True  # Enable autoescaping for security
+    )
+    
+    env.filters['date'] = lambda d, format='%Y-%m-%d': d.strftime(format) if isinstance(d, datetime) else str(d)
+    
+    template = env.from_string(jinja_template)
+    
+    try:
+        return template.render(**inputs)
+    except jinja2.exceptions.UndefinedError as e:
+        var_name = str(e).split("'")[1] if "'" in str(e) else None
+        if var_name:
+            raise KeyError(f"Template variable '{var_name}' not found in inputs dictionary")
+        raise KeyError(f"Missing required template variable: {str(e)}")

src/crewai/utilities/string_utils.py

@@ -1,31 +1,39 @@
 import re
+from datetime import datetime
 from typing import Any, Dict, List, Optional, Union
 
+from crewai.utilities.jinja_templating import render_template
+
 
 def interpolate_only(
     input_string: Optional[str],
-    inputs: Dict[str, Union[str, int, float, Dict[str, Any], List[Any]]],
+    inputs: Dict[str, Any],
 ) -> str:
     """Interpolate placeholders (e.g., {key}) in a string while leaving JSON untouched.
     Only interpolates placeholders that follow the pattern {variable_name} where
     variable_name starts with a letter/underscore and contains only letters, numbers, and underscores.
+    
+    This function now supports advanced Jinja2 templating features:
+    - Container types (List, Dict, Set)
+    - Standard objects (datetime, time)
+    - Custom objects
+    - Conditional and loop statements
+    - Filtering options
 
     Args:
         input_string: The string containing template variables to interpolate.
                      Can be None or empty, in which case an empty string is returned.
         inputs: Dictionary mapping template variables to their values.
-               Supported value types are strings, integers, floats, and dicts/lists
-               containing only these types and other nested dicts/lists.
+               Supports all types of values including complex objects.
 
     Returns:
         The interpolated string with all template variables replaced with their values.
         Empty string if input_string is None or empty.
 
     Raises:
-        ValueError: If a value contains unsupported types or a template variable is missing
+        ValueError: If inputs dictionary is empty when interpolating variables.
+        KeyError: If a required template variable is missing from inputs.
     """
-
-    # Validation function for recursive type checking
     def validate_type(value: Any) -> None:
         if value is None:
             return
@@ -35,12 +43,21 @@ def interpolate_only(
             for item in value.values() if isinstance(value, dict) else value:
                 validate_type(item)
             return
+        if isinstance(value, datetime):
+            return
+        # Check if it's a Pydantic model or other known custom type
+        try:
+            from pydantic import BaseModel
+            if isinstance(value, BaseModel):
+                return
+        except ImportError:
+            pass
+            
         raise ValueError(
             f"Unsupported type {type(value).__name__} in inputs. "
-            "Only str, int, float, bool, dict, and list are allowed."
+            "Only str, int, float, bool, dict, list, datetime, and custom objects are allowed."
         )
 
-    # Validate all input values
     for key, value in inputs.items():
         try:
             validate_type(value)
@@ -56,6 +73,13 @@ def interpolate_only(
             "Inputs dictionary cannot be empty when interpolating variables"
         )
 
+    # Check if the template contains Jinja2 syntax ({% ... %} or {{ ... }})
+    has_jinja_syntax = "{{" in input_string or "{%" in input_string
+    has_complex_indexing = re.search(r"\{([A-Za-z_][A-Za-z0-9_]*)\[[0-9]+\]\}", input_string)
+    
+    if has_jinja_syntax or has_complex_indexing:
+        return render_template(input_string, inputs)
+    
     # The regex pattern to find valid variable placeholders
     # Matches {variable_name} where variable_name starts with a letter/underscore
     # and contains only letters, numbers, and underscores
@@ -63,8 +87,7 @@ def interpolate_only(
 
     # Find all matching variables in the input string
     variables = re.findall(pattern, input_string)
-    result = input_string
-
+    
     # Check if all variables exist in inputs
     missing_vars = [var for var in variables if var not in inputs]
     if missing_vars:
@@ -72,11 +95,10 @@ def interpolate_only(
             f"Template variable '{missing_vars[0]}' not found in inputs dictionary"
         )
 
-    # Replace each variable with its value
+    result = input_string
     for var in variables:
         if var in inputs:
             placeholder = "{" + var + "}"
             value = str(inputs[var])
             result = result.replace(placeholder, value)
-
     return result

tests/test_templating.py

@@ -0,0 +1,91 @@
+import datetime
+from typing import Dict, List
+
+import pytest
+
+from crewai.agent import Agent
+from crewai.task import Task
+
+
+class TestTemplating:
+    def test_task_interpolation(self):
+        task = Task(
+            description="Research about {topic} and provide {count} key points",
+            expected_output="A list of {count} key points about {topic}"
+        )
+        
+        inputs = {"topic": "AI", "count": 5}
+        task.interpolate_inputs(inputs)
+        
+        assert task.description == "Research about AI and provide 5 key points"
+        assert task.expected_output == "A list of 5 key points about AI"
+        
+        task = Task(
+            description="Research about {topics[0]} and {topics[1]}",
+            expected_output="Analysis of {{data.main_theme}}"
+        )
+        
+        inputs = {
+            "topics": ["AI", "Machine Learning"],
+            "data": {"main_theme": "Technology Trends"}
+        }
+        
+        task.interpolate_inputs(inputs)
+        
+        assert task.description == "Research about AI and Machine Learning"
+        assert task.expected_output == "Analysis of Technology Trends"
+    
+    def test_agent_interpolation(self):
+        agent = Agent(
+            role="{industry} Researcher",
+            goal="Research {count} key developments in {industry}",
+            backstory="You are a senior researcher in the {industry} field with {experience} years of experience"
+        )
+        
+        inputs = {"industry": "Healthcare", "count": 5, "experience": 10}
+        agent.interpolate_inputs(inputs)
+        
+        assert agent.role == "Healthcare Researcher"
+        assert agent.goal == "Research 5 key developments in Healthcare"
+        assert agent.backstory == "You are a senior researcher in the Healthcare field with 10 years of experience"
+        
+        agent = Agent(
+            role="{{specialties[0]}} and {{specialties[1]}} Specialist",
+            goal="Analyze trends in {{fields.primary}} sector",
+            backstory="Expert in {{fields.primary}} and {{fields.secondary}}"
+        )
+        
+        inputs = {
+            "specialties": ["AI", "Data Science"],
+            "fields": {"primary": "Healthcare", "secondary": "Finance"}
+        }
+        
+        agent.interpolate_inputs(inputs)
+        
+        assert agent.role == "AI and Data Science Specialist"
+        assert agent.goal == "Analyze trends in Healthcare sector"
+        assert agent.backstory == "Expert in Healthcare and Finance"
+    
+    def test_conditional_templating(self):
+        task = Task(
+            description="{% if priority == 'high' %}URGENT: {% endif %}Research {topic}",
+            expected_output="A report on {topic}"
+        )
+        
+        inputs = {"topic": "AI", "priority": "high"}
+        task.interpolate_inputs(inputs)
+        assert task.description == "URGENT: Research AI"
+        
+        inputs = {"topic": "AI", "priority": "low"}
+        task.interpolate_inputs(inputs)
+        assert task.description == "Research AI"
+    
+    def test_loop_templating(self):
+        task = Task(
+            description="Research the following topics: {% for topic in topics %}{{topic}}{% if not loop.last %}, {% endif %}{% endfor %}",
+            expected_output="A report on multiple topics"
+        )
+        
+        inputs = {"topics": ["AI", "Machine Learning", "Data Science"]}
+        task.interpolate_inputs(inputs)
+        assert task.description == "Research the following topics: AI, Machine Learning, Data Science"

tests/utilities/test_jinja_templating.py

@@ -0,0 +1,84 @@
+import datetime
+from typing import Any, Dict, List
+
+import pytest
+from pydantic import BaseModel
+
+from crewai.utilities.jinja_templating import render_template, to_jinja_template
+
+
+class Person(BaseModel):
+    name: str
+    age: int
+    
+    def __str__(self):
+        return f"{self.name} ({self.age})"
+
+class TestJinjaTemplating:
+    def test_to_jinja_template(self):
+        assert to_jinja_template("Hello {name}!") == "Hello {{name}}!"
+        
+        assert to_jinja_template("Hello {{name}}!") == "Hello {{name}}!"
+        
+        assert to_jinja_template("Hello {name} and {{title}}!") == "Hello {{name}} and {{title}}!"
+        
+        assert to_jinja_template("") == ""
+        
+        assert to_jinja_template("Hello world!") == "Hello world!"
+    
+    def test_render_template_simple_types(self):
+        inputs = {"name": "John", "age": 30, "active": True, "height": 1.85}
+        
+        assert render_template("Hello {name}!", inputs) == "Hello John!"
+        assert render_template("Age: {age}", inputs) == "Age: 30"
+        assert render_template("Active: {active}", inputs) == "Active: True"
+        assert render_template("Height: {height}", inputs) == "Height: 1.85"
+        
+        assert render_template("{name} is {age} years old", inputs) == "John is 30 years old"
+    
+    def test_render_template_container_types(self):
+        inputs = {
+            "items": ["apple", "banana", "orange"],
+            "person": {"name": "John", "age": 30}
+        }
+        
+        assert render_template("First item: {{items[0]}}", inputs) == "First item: apple"
+        
+        assert render_template("Person name: {{person.name}}", inputs) == "Person name: John"
+        
+        assert render_template(
+            "Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}",
+            inputs
+        ) == "Items: apple, banana, orange"
+        
+        assert render_template(
+            "{% if items|length > 2 %}Many items{% else %}Few items{% endif %}",
+            inputs
+        ) == "Many items"
+    
+    def test_render_template_datetime(self):
+        today = datetime.datetime.now()
+        inputs = {"today": today}
+        
+        assert render_template("Today: {{today|date}}", inputs) == f"Today: {today.strftime('%Y-%m-%d')}"
+        
+        assert render_template("Today: {{today|date('%d/%m/%Y')}}", inputs) == f"Today: {today.strftime('%d/%m/%Y')}"
+    
+    def test_render_template_custom_objects(self):
+        person = Person(name="John", age=30)
+        inputs = {"person": person}
+        
+        assert render_template("Person: {person}", inputs) == "Person: John (30)"
+        
+        assert render_template("Person name: {{person.name}}", inputs) == "Person name: John"
+    
+    def test_render_template_error_handling(self):
+        inputs = {"name": "John"}
+        
+        with pytest.raises(KeyError) as excinfo:
+            render_template("Hello {age}!", inputs)
+        assert "Template variable 'age' not found" in str(excinfo.value)
+        
+        with pytest.raises(ValueError) as excinfo:
+            render_template("Hello {name}!", {})
+        assert "Inputs dictionary cannot be empty" in str(excinfo.value)

tests/utilities/test_string_utils.py

@@ -1,6 +1,8 @@
+import datetime
 from typing import Any, Dict, List, Union
 
 import pytest
+from pydantic import BaseModel
 
 from crewai.utilities.string_utils import interpolate_only
 
@@ -185,3 +187,96 @@ class TestInterpolateOnly:
             interpolate_only(template, inputs)
 
         assert "inputs dictionary cannot be empty" in str(excinfo.value).lower()
+        
+    
+    def test_container_types_list_access(self):
+        """Test accessing list items with Jinja2 syntax."""
+        template = "First item: {{items[0]}}, Second item: {{items[1]}}"
+        inputs = {
+            "items": ["apple", "banana", "orange"]
+        }
+        
+        result = interpolate_only(template, inputs)
+        assert result == "First item: apple, Second item: banana"
+    
+    def test_container_types_dict_access(self):
+        """Test accessing dictionary items with Jinja2 syntax."""
+        template = "Name: {{person.name}}, Age: {{person.age}}"
+        inputs = {
+            "person": {"name": "John", "age": 30}
+        }
+        
+        result = interpolate_only(template, inputs)
+        assert result == "Name: John, Age: 30"
+    
+    def test_conditional_statements(self):
+        """Test conditional statements with Jinja2 syntax."""
+        template = "{% if priority == 'high' %}URGENT: {% endif %}Task: {task}"
+        
+        inputs_high = {
+            "task": "Fix bug",
+            "priority": "high"
+        }
+        result_high = interpolate_only(template, inputs_high)
+        assert result_high == "URGENT: Task: Fix bug"
+        
+        inputs_low = {
+            "task": "Fix bug",
+            "priority": "low"
+        }
+        result_low = interpolate_only(template, inputs_low)
+        assert result_low == "Task: Fix bug"
+    
+    def test_loop_statements(self):
+        """Test loop statements with Jinja2 syntax."""
+        template = "Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}"
+        inputs = {
+            "items": ["apple", "banana", "orange"]
+        }
+        
+        result = interpolate_only(template, inputs)
+        assert result == "Items: apple, banana, orange"
+    
+    def test_datetime_formatting(self):
+        """Test datetime formatting with Jinja2 filters."""
+        today = datetime.datetime(2024, 4, 20)
+        inputs = {"today": today}
+        
+        template = "Date: {{today|date}}"
+        result = interpolate_only(template, inputs)
+        assert result == "Date: 2024-04-20"
+        
+        template = "Date: {{today|date('%d/%m/%Y')}}"
+        result = interpolate_only(template, inputs)
+        assert result == "Date: 20/04/2024"
+    
+    def test_custom_objects(self):
+        """Test custom objects with Jinja2 syntax."""
+        class Person(BaseModel):
+            name: str
+            age: int
+            
+            def __str__(self):
+                return f"{self.name} ({self.age})"
+        
+        person = Person(name="John", age=30)
+        inputs = {"person": person}
+        
+        template = "Person: {person}"
+        result = interpolate_only(template, inputs)
+        assert result == "Person: John (30)"
+        
+        template = "Name: {{person.name}}, Age: {{person.age}}"
+        result = interpolate_only(template, inputs)
+        assert result == "Name: John, Age: 30"
+    
+    def test_mixed_syntax(self):
+        """Test mixed CrewAI and Jinja2 syntax."""
+        template = "Hello {name}! Items: {% for item in items %}{{item}}{% if not loop.last %}, {% endif %}{% endfor %}"
+        inputs = {
+            "name": "John",
+            "items": ["apple", "banana", "orange"]
+        }
+        
+        result = interpolate_only(template, inputs)
+        assert result == "Hello John! Items: apple, banana, orange"