From ef39974bd88175d16a8b5e371b4964c2b6639991 Mon Sep 17 00:00:00 2001
From: Joao Moura <joaomdmoura@gmail.com>
Date: Thu, 14 May 2026 00:28:44 -0400
Subject: [PATCH] feat: add configurable case timeout for benchmarking and
 testing

- Introduced a case_timeout parameter in the benchmark and test functions to allow dynamic timeout settings.
- Updated the project configuration template to include a default case_timeout value of 90 seconds.
- Enhanced the handling of timeouts in benchmark results to reflect the configured case_timeout.
---
 lib/cli/src/crewai_cli/benchmark.py    |  7 +++++--
 lib/cli/src/crewai_cli/cli.py          | 10 ++++++++++
 lib/cli/src/crewai_cli/create_agent.py |  6 +++++-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/lib/cli/src/crewai_cli/benchmark.py b/lib/cli/src/crewai_cli/benchmark.py
index 5e77463dd..5288cd62b 100644
--- a/lib/cli/src/crewai_cli/benchmark.py
+++ b/lib/cli/src/crewai_cli/benchmark.py
@@ -246,6 +246,7 @@ async def _run_model_benchmark(
     emit: Callable[[dict[str, Any]], None],
     agents_dir: Path | None = None,
     verbose: bool = False,
+    case_timeout: int = _CASE_TIMEOUT_SECONDS,
 ) -> list[BenchmarkResult]:
     """Run all benchmark cases for a single model, parallelising up to _MAX_CASES_PARALLEL."""
     total = len(cases)
@@ -303,7 +304,7 @@ async def _run_model_benchmark(
             try:
                 response = await asyncio.wait_for(
                     agent.amessage(case.input),
-                    timeout=_CASE_TIMEOUT_SECONDS,
+                    timeout=case_timeout,
                 )
                 elapsed_ms = _current_time_ms() - start_ms
                 actual = response.content
@@ -328,7 +329,7 @@ async def _run_model_benchmark(
                     case_index=i,
                     input=case.input,
                     expected=case.expected,
-                    actual=f"[Timeout after {_CASE_TIMEOUT_SECONDS}s]",
+                    actual=f"[Timeout after {case_timeout}s]",
                     model=model,
                     passed=False,
                     score=0.0,
@@ -449,6 +450,7 @@ async def run_benchmark(
     judge_model: str = "openai/gpt-4o-mini",
     on_progress: Callable[[dict[str, Any]], None] | None = None,
     verbose: bool = False,
+    case_timeout: int = _CASE_TIMEOUT_SECONDS,
 ) -> dict[str, list[BenchmarkResult]]:
     """Run benchmark cases against an agent definition across models in parallel.
 
@@ -489,6 +491,7 @@ async def run_benchmark(
             _emit,
             agents_dir=agents_dir,
             verbose=verbose,
+            case_timeout=case_timeout,
         )
         for model in models
     ]
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index 0997c75e4..f638a29f1 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -594,6 +594,9 @@ def test(
             else (float(config_threshold) if config_threshold is not None else 0.7)
         )
 
+        config_timeout = _read_config("test", "case_timeout")
+        effective_timeout = int(config_timeout) if config_timeout is not None else 90
+
         _test_new_agents(
             agent_files,
             effective_iterations,
@@ -601,6 +604,7 @@ def test(
             effective_threshold,
             effective_judge,
             verbose=verbose,
+            case_timeout=effective_timeout,
         )
     else:
         legacy_iterations = n_iterations if n_iterations is not None else 3
@@ -859,6 +863,7 @@ def _test_new_agents(
     threshold: float,
     judge_model: str,
     verbose: bool = False,
+    case_timeout: int = 90,
 ) -> None:
     """Run NewAgent test cases with pass/fail threshold (all agents in parallel)."""
     import asyncio
@@ -934,6 +939,7 @@ def _test_new_agents(
                     if verbose
                     else _make_progress_cb(job["agent_name"]),
                     verbose=verbose,
+                    case_timeout=case_timeout,
                 )
             )
         return await asyncio.gather(*tasks, return_exceptions=True)
@@ -1885,6 +1891,8 @@ def benchmark(
     judge_model = (
         judge_model or _read_config("test", "judge_model") or "openai/gpt-4o-mini"
     )
+    config_timeout = _read_config("test", "case_timeout")
+    effective_timeout = int(config_timeout) if config_timeout is not None else 90
 
     if _needs_uv_relaunch():
         uv_args = ["benchmark", agent_path, cases_path, "--judge-model", judge_model]
@@ -1948,6 +1956,7 @@ def benchmark(
                             judge_model=judge_model,
                             on_progress=progress.on_progress if progress else None,
                             verbose=verbose,
+                            case_timeout=effective_timeout,
                         )
                     )
             else:
@@ -1960,6 +1969,7 @@ def benchmark(
                             judge_model=judge_model,
                             on_progress=progress.on_progress if progress else None,
                             verbose=verbose,
+                            case_timeout=effective_timeout,
                         )
                     )
     except Exception as e:
diff --git a/lib/cli/src/crewai_cli/create_agent.py b/lib/cli/src/crewai_cli/create_agent.py
index 432c61b07..e40e8c0c8 100644
--- a/lib/cli/src/crewai_cli/create_agent.py
+++ b/lib/cli/src/crewai_cli/create_agent.py
@@ -134,7 +134,11 @@ PROJECT_CONFIG_TEMPLATE = """\
 
     // LLM used to judge test responses (provider/model format).
     // Override with: crewai test --judge-model openai/gpt-4o
-    "judge_model": "openai/gpt-4o-mini"
+    "judge_model": "openai/gpt-4o-mini",
+
+    // Max seconds to wait for an agent response per test case.
+    // Increase for agents that use tools or multi-step reasoning.
+    "case_timeout": 90
   },
 
   // Rooms define how agents collaborate in the TUI (`crewai run`)