mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-07-08 00:15:12 +00:00
feat: add interactive agent creation and TUI for multi-agent interaction
- Introduced a new `create_agent` command for interactive agent definition. - Added `agent_tui.py` for a conversational TUI supporting multi-agent interactions. - Updated CLI to support agent creation and training workflows. - Enhanced `.gitignore` to exclude demo files and configuration artifacts. - Implemented a benchmark runner for testing agent performance against defined cases. This commit lays the groundwork for a more interactive and user-friendly experience in managing agents within the CrewAI framework.
This commit is contained in:
Joao Moura
1411
lib/cli/src/crewai_cli/agent_tui.py
Normal file
1411
lib/cli/src/crewai_cli/agent_tui.py
Normal file
File diff suppressed because it is too large
Load Diff
380
lib/cli/src/crewai_cli/benchmark.py
Normal file
380
lib/cli/src/crewai_cli/benchmark.py
Normal file
@@ -0,0 +1,380 @@
|
||||
"""Benchmark runner for NewAgent — run agents against test cases and report results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class BenchmarkCase(BaseModel):
|
||||
"""A single benchmark test case."""
|
||||
|
||||
input: str
|
||||
expected: str | None = None
|
||||
criteria: str | None = None
|
||||
|
||||
|
||||
class BenchmarkResult(BaseModel):
|
||||
"""Result of running a single benchmark case."""
|
||||
|
||||
case_index: int
|
||||
input: str
|
||||
expected: str | None = None
|
||||
actual: str = ""
|
||||
model: str = ""
|
||||
passed: bool = False
|
||||
score: float = 0.0
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
response_time_ms: int = 0
|
||||
cost: float | None = None
|
||||
|
||||
|
||||
def load_benchmark_cases(path: str | Path) -> list[BenchmarkCase]:
|
||||
"""Load benchmark cases from a JSON or JSONC file.
|
||||
|
||||
Args:
|
||||
path: Path to a JSON/JSONC file containing an array of test cases.
|
||||
|
||||
Returns:
|
||||
List of BenchmarkCase instances.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file does not exist.
|
||||
ValueError: If the file content is not a valid JSON array of cases.
|
||||
"""
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"Benchmark cases file not found: {path}")
|
||||
|
||||
raw = p.read_text(encoding="utf-8")
|
||||
|
||||
# Strip JSONC comments
|
||||
clean = _strip_jsonc_comments(raw)
|
||||
|
||||
try:
|
||||
data = json.loads(clean)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Invalid JSON in benchmark cases file: {e}") from e
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise ValueError("Benchmark cases file must contain a JSON array")
|
||||
|
||||
cases: list[BenchmarkCase] = []
|
||||
for i, item in enumerate(data):
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError(f"Benchmark case at index {i} must be a JSON object")
|
||||
if "input" not in item:
|
||||
raise ValueError(f"Benchmark case at index {i} missing required 'input' field")
|
||||
cases.append(BenchmarkCase(**item))
|
||||
|
||||
return cases
|
||||
|
||||
|
||||
def _strip_jsonc_comments(text: str) -> str:
|
||||
"""Strip // and /* */ comments from JSONC text."""
|
||||
result = re.sub(r"(?<!:)//.*?$", "", text, flags=re.MULTILINE)
|
||||
result = re.sub(r"/\*.*?\*/", "", result, flags=re.DOTALL)
|
||||
return result
|
||||
|
||||
|
||||
def _check_expected(expected: str, actual: str) -> tuple[bool, float]:
|
||||
"""Check if expected output is found in actual (case-insensitive substring match).
|
||||
|
||||
Returns:
|
||||
Tuple of (passed, score).
|
||||
"""
|
||||
if expected.lower() in actual.lower():
|
||||
return True, 1.0
|
||||
return False, 0.0
|
||||
|
||||
|
||||
async def _judge_with_llm(
|
||||
criteria: str,
|
||||
input_text: str,
|
||||
actual: str,
|
||||
judge_model: str,
|
||||
) -> tuple[bool, float]:
|
||||
"""Use an LLM judge to evaluate a response against criteria.
|
||||
|
||||
Returns:
|
||||
Tuple of (passed, score).
|
||||
"""
|
||||
from crewai.utilities.llm_utils import create_llm
|
||||
|
||||
judge_llm = create_llm(judge_model)
|
||||
|
||||
prompt = (
|
||||
"You are an evaluation judge. Score the following response on a scale of 0.0 to 1.0.\n\n"
|
||||
f"Input: {input_text}\n\n"
|
||||
f"Response: {actual}\n\n"
|
||||
f"Evaluation criteria: {criteria}\n\n"
|
||||
"Respond with ONLY a JSON object in this exact format:\n"
|
||||
'{"score": <float between 0.0 and 1.0>, "passed": <true or false>}\n'
|
||||
"A score >= 0.7 should be considered passed."
|
||||
)
|
||||
|
||||
try:
|
||||
response = judge_llm.call(messages=[{"role": "user", "content": prompt}])
|
||||
text = str(response) if not isinstance(response, str) else response
|
||||
# Extract JSON from response
|
||||
match = re.search(r"\{[^}]+\}", text)
|
||||
if match:
|
||||
result = json.loads(match.group())
|
||||
score = float(result.get("score", 0.0))
|
||||
score = max(0.0, min(1.0, score))
|
||||
passed = bool(result.get("passed", score >= 0.7))
|
||||
return passed, score
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False, 0.0
|
||||
|
||||
|
||||
def _parse_definition(source: Any) -> dict[str, Any]:
|
||||
"""Parse an agent definition — delegates to crewai's parser."""
|
||||
from crewai.new_agent.definition_parser import parse_agent_definition
|
||||
return parse_agent_definition(source)
|
||||
|
||||
|
||||
def _load_agent(source: Any) -> Any:
|
||||
"""Load a NewAgent from a definition — delegates to crewai's loader."""
|
||||
from crewai.new_agent.definition_parser import load_agent_from_definition
|
||||
return load_agent_from_definition(source)
|
||||
|
||||
|
||||
async def run_benchmark(
|
||||
agent_def: dict[str, Any] | str | Path,
|
||||
cases: list[BenchmarkCase],
|
||||
models: list[str] | None = None,
|
||||
judge_model: str = "openai/gpt-4o-mini",
|
||||
) -> dict[str, list[BenchmarkResult]]:
|
||||
"""Run benchmark cases against an agent definition, optionally across multiple models.
|
||||
|
||||
Args:
|
||||
agent_def: Agent definition dict, JSON string, or file path.
|
||||
cases: List of benchmark cases to run.
|
||||
models: Optional list of model identifiers to compare. If None, uses agent's default.
|
||||
judge_model: Model to use for LLM judge evaluation.
|
||||
|
||||
Returns:
|
||||
Dict mapping model name to list of BenchmarkResult.
|
||||
"""
|
||||
defn = _parse_definition(agent_def)
|
||||
|
||||
if models is None or len(models) == 0:
|
||||
models = [defn.get("llm", "default")]
|
||||
|
||||
results_by_model: dict[str, list[BenchmarkResult]] = {}
|
||||
|
||||
for model in models:
|
||||
model_results: list[BenchmarkResult] = []
|
||||
|
||||
for i, case in enumerate(cases):
|
||||
# Override the model and disable memory for benchmark runs
|
||||
bench_defn = dict(defn)
|
||||
if model != "default":
|
||||
bench_defn["llm"] = model
|
||||
bench_defn.setdefault("settings", {})
|
||||
bench_defn["settings"]["memory_read_only"] = True
|
||||
|
||||
try:
|
||||
agent = _load_agent(bench_defn)
|
||||
except Exception as e:
|
||||
model_results.append(
|
||||
BenchmarkResult(
|
||||
case_index=i,
|
||||
input=case.input,
|
||||
expected=case.expected,
|
||||
actual=f"[Agent creation error: {e}]",
|
||||
model=model,
|
||||
passed=False,
|
||||
score=0.0,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
start_ms = _current_time_ms()
|
||||
try:
|
||||
response = await agent.amessage(case.input)
|
||||
elapsed_ms = _current_time_ms() - start_ms
|
||||
|
||||
actual = response.content
|
||||
input_tokens = response.input_tokens or 0
|
||||
output_tokens = response.output_tokens or 0
|
||||
cost = response.cost
|
||||
|
||||
except Exception as e:
|
||||
elapsed_ms = _current_time_ms() - start_ms
|
||||
model_results.append(
|
||||
BenchmarkResult(
|
||||
case_index=i,
|
||||
input=case.input,
|
||||
expected=case.expected,
|
||||
actual=f"[Error: {e}]",
|
||||
model=model,
|
||||
passed=False,
|
||||
score=0.0,
|
||||
response_time_ms=elapsed_ms,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Evaluate
|
||||
passed = False
|
||||
score = 0.0
|
||||
|
||||
if case.expected is not None:
|
||||
passed, score = _check_expected(case.expected, actual)
|
||||
if case.criteria is not None:
|
||||
criteria_passed, criteria_score = await _judge_with_llm(
|
||||
case.criteria, case.input, actual, judge_model
|
||||
)
|
||||
if case.expected is not None:
|
||||
# Combine: both must pass, average scores
|
||||
passed = passed and criteria_passed
|
||||
score = (score + criteria_score) / 2.0
|
||||
else:
|
||||
passed = criteria_passed
|
||||
score = criteria_score
|
||||
|
||||
model_results.append(
|
||||
BenchmarkResult(
|
||||
case_index=i,
|
||||
input=case.input,
|
||||
expected=case.expected,
|
||||
actual=actual,
|
||||
model=model,
|
||||
passed=passed,
|
||||
score=score,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
response_time_ms=elapsed_ms,
|
||||
cost=cost,
|
||||
)
|
||||
)
|
||||
|
||||
results_by_model[model] = model_results
|
||||
|
||||
return results_by_model
|
||||
|
||||
|
||||
def _current_time_ms() -> int:
|
||||
"""Return current time in milliseconds."""
|
||||
return int(time.monotonic() * 1000)
|
||||
|
||||
|
||||
def format_results_table(results: list[BenchmarkResult]) -> str:
|
||||
"""Format benchmark results as a readable table.
|
||||
|
||||
Args:
|
||||
results: List of BenchmarkResult for a single model.
|
||||
|
||||
Returns:
|
||||
Formatted string table.
|
||||
"""
|
||||
if not results:
|
||||
return "No results to display."
|
||||
|
||||
model = results[0].model
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append(f"Benchmark Results — Model: {model}")
|
||||
lines.append("=" * 80)
|
||||
|
||||
header = f"{'#':<4} {'Pass':<6} {'Score':<7} {'Tokens':<12} {'Time (ms)':<10} {'Input (truncated)'}"
|
||||
lines.append(header)
|
||||
lines.append("-" * 80)
|
||||
|
||||
total_passed = 0
|
||||
total_score = 0.0
|
||||
total_input_tokens = 0
|
||||
total_output_tokens = 0
|
||||
total_time_ms = 0
|
||||
|
||||
for r in results:
|
||||
status = "PASS" if r.passed else "FAIL"
|
||||
tokens = f"{r.input_tokens}/{r.output_tokens}"
|
||||
input_trunc = r.input[:40] + "..." if len(r.input) > 40 else r.input
|
||||
line = f"{r.case_index:<4} {status:<6} {r.score:<7.2f} {tokens:<12} {r.response_time_ms:<10} {input_trunc}"
|
||||
lines.append(line)
|
||||
|
||||
if r.passed:
|
||||
total_passed += 1
|
||||
total_score += r.score
|
||||
total_input_tokens += r.input_tokens
|
||||
total_output_tokens += r.output_tokens
|
||||
total_time_ms += r.response_time_ms
|
||||
|
||||
lines.append("-" * 80)
|
||||
n = len(results)
|
||||
avg_score = total_score / n if n > 0 else 0.0
|
||||
lines.append(f"Total: {total_passed}/{n} passed | Avg score: {avg_score:.2f} | "
|
||||
f"Tokens: {total_input_tokens}/{total_output_tokens} | "
|
||||
f"Total time: {total_time_ms}ms")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_comparison_table(results_by_model: dict[str, list[BenchmarkResult]]) -> str:
|
||||
"""Format a comparison table across multiple models.
|
||||
|
||||
Args:
|
||||
results_by_model: Dict mapping model name to list of BenchmarkResult.
|
||||
|
||||
Returns:
|
||||
Formatted comparison string.
|
||||
"""
|
||||
if not results_by_model:
|
||||
return "No results to compare."
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("Model Comparison")
|
||||
lines.append("=" * 90)
|
||||
|
||||
header = f"{'Model':<30} {'Passed':<10} {'Avg Score':<12} {'In Tokens':<12} {'Out Tokens':<12} {'Time (ms)'}"
|
||||
lines.append(header)
|
||||
lines.append("-" * 90)
|
||||
|
||||
for model, results in results_by_model.items():
|
||||
n = len(results)
|
||||
passed = sum(1 for r in results if r.passed)
|
||||
avg_score = sum(r.score for r in results) / n if n > 0 else 0.0
|
||||
total_in = sum(r.input_tokens for r in results)
|
||||
total_out = sum(r.output_tokens for r in results)
|
||||
total_time = sum(r.response_time_ms for r in results)
|
||||
|
||||
model_trunc = model[:28] if len(model) > 28 else model
|
||||
line = (
|
||||
f"{model_trunc:<30} {passed}/{n:<8} {avg_score:<12.2f} "
|
||||
f"{total_in:<12} {total_out:<12} {total_time}"
|
||||
)
|
||||
lines.append(line)
|
||||
|
||||
lines.append("-" * 90)
|
||||
|
||||
# Determine best model by average score
|
||||
if results_by_model:
|
||||
best_model = max(
|
||||
results_by_model.keys(),
|
||||
key=lambda m: (
|
||||
sum(r.score for r in results_by_model[m]) / len(results_by_model[m])
|
||||
if results_by_model[m]
|
||||
else 0.0
|
||||
),
|
||||
)
|
||||
best_score = (
|
||||
sum(r.score for r in results_by_model[best_model])
|
||||
/ len(results_by_model[best_model])
|
||||
if results_by_model[best_model]
|
||||
else 0.0
|
||||
)
|
||||
lines.append(f"Best model: {best_model} (avg score: {best_score:.2f})")
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -11,6 +11,7 @@ from crewai_core.token_manager import TokenManager
|
||||
from crewai_cli.add_crew_to_flow import add_crew_to_flow
|
||||
from crewai_cli.authentication.main import AuthenticationCommand
|
||||
from crewai_cli.config import Settings
|
||||
from crewai_cli.create_agent import create_agent
|
||||
from crewai_cli.create_crew import create_crew
|
||||
from crewai_cli.create_flow import create_flow
|
||||
from crewai_cli.crew_chat import run_chat
|
||||
@@ -91,20 +92,31 @@ def uv(uv_args: tuple[str, ...]) -> None:
|
||||
|
||||
|
||||
@crewai.command()
|
||||
@click.argument("type", type=click.Choice(["crew", "flow"]))
|
||||
@click.argument("name")
|
||||
@click.argument("type", type=click.Choice(["crew", "flow", "agent"]))
|
||||
@click.argument("name", required=False, default=None)
|
||||
@click.option("--provider", type=str, help="The provider to use for the crew")
|
||||
@click.option("--skip_provider", is_flag=True, help="Skip provider validation")
|
||||
def create(
|
||||
type: str, name: str, provider: str | None, skip_provider: bool = False
|
||||
type: str, name: str | None, provider: str | None, skip_provider: bool = False
|
||||
) -> None:
|
||||
"""Create a new crew, or flow."""
|
||||
"""Create a new crew, flow, or agent.
|
||||
|
||||
For agents, NAME is optional — omit it to enter interactive mode.
|
||||
"""
|
||||
if type == "crew":
|
||||
if name is None:
|
||||
click.secho("Error: name is required for crew creation.", fg="red")
|
||||
raise SystemExit(1)
|
||||
create_crew(name, provider, skip_provider)
|
||||
elif type == "flow":
|
||||
if name is None:
|
||||
click.secho("Error: name is required for flow creation.", fg="red")
|
||||
raise SystemExit(1)
|
||||
create_flow(name)
|
||||
elif type == "agent":
|
||||
create_agent(name)
|
||||
else:
|
||||
click.secho("Error: Invalid type. Must be 'crew' or 'flow'.", fg="red")
|
||||
click.secho("Error: Invalid type. Must be 'crew', 'flow', or 'agent'.", fg="red")
|
||||
|
||||
|
||||
@crewai.command()
|
||||
@@ -133,19 +145,115 @@ def version(tools: bool) -> None:
|
||||
"--n_iterations",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Number of iterations to train the crew",
|
||||
help="Number of iterations to run training feedback.",
|
||||
)
|
||||
@click.option(
|
||||
"-f",
|
||||
"--filename",
|
||||
type=str,
|
||||
default="trained_agents_data.pkl",
|
||||
help="Path to a custom file for training",
|
||||
help="Path to a trained-agents pickle (Crew projects only).",
|
||||
)
|
||||
def train(n_iterations: int, filename: str) -> None:
|
||||
"""Train the crew."""
|
||||
click.echo(f"Training the Crew for {n_iterations} iterations")
|
||||
train_crew(n_iterations, filename)
|
||||
"""Train the crew or agents.
|
||||
|
||||
Auto-detects project type: if agents/ directory exists, runs interactive
|
||||
NewAgent training (feedback → canonical memories). Otherwise falls back to
|
||||
legacy Crew training.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from crewai_cli.run_crew import _needs_uv_relaunch, _relaunch_via_uv
|
||||
|
||||
agents_dir = Path("agents")
|
||||
agent_files = (
|
||||
sorted(agents_dir.glob("*.json")) + sorted(agents_dir.glob("*.jsonc"))
|
||||
if agents_dir.is_dir()
|
||||
else []
|
||||
)
|
||||
|
||||
if agent_files:
|
||||
if _needs_uv_relaunch():
|
||||
_relaunch_via_uv(["train", "-n", str(n_iterations), "-f", filename])
|
||||
_train_new_agents(agent_files, n_iterations)
|
||||
else:
|
||||
click.echo(f"Training the Crew for {n_iterations} iterations")
|
||||
train_crew(n_iterations, filename)
|
||||
|
||||
|
||||
def _train_new_agents(agent_files: list, n_iterations: int) -> None:
|
||||
"""Run interactive training for NewAgent agents.
|
||||
|
||||
For each agent, loads benchmark cases, runs them, shows the response,
|
||||
and asks the user for feedback. Feedback is saved as canonical memories.
|
||||
"""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from crewai_cli.benchmark import load_benchmark_cases
|
||||
|
||||
benchmarks_dir = Path("benchmarks")
|
||||
agents_trained = 0
|
||||
|
||||
for agent_path in agent_files:
|
||||
agent_name = agent_path.stem
|
||||
cases_path = benchmarks_dir / f"{agent_name}_cases.json"
|
||||
|
||||
if not cases_path.exists():
|
||||
click.secho(f" Skipping {agent_name} — no {cases_path}", fg="yellow")
|
||||
continue
|
||||
|
||||
try:
|
||||
cases = load_benchmark_cases(cases_path)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
click.secho(f" Error loading cases for {agent_name}: {e}", fg="red")
|
||||
continue
|
||||
|
||||
click.echo()
|
||||
click.secho(f"Training {agent_name} ({len(cases)} cases, {n_iterations} iterations)", fg="cyan", bold=True)
|
||||
|
||||
try:
|
||||
from crewai.new_agent.definition_parser import load_agent_definition
|
||||
agent = load_agent_definition(str(agent_path))
|
||||
except Exception as e:
|
||||
click.secho(f" Error loading agent {agent_name}: {e}", fg="red")
|
||||
continue
|
||||
|
||||
for iteration in range(n_iterations):
|
||||
click.secho(f"\n Iteration {iteration + 1}/{n_iterations}", fg="cyan")
|
||||
for case in cases:
|
||||
user_input = case.input
|
||||
click.echo(f"\n Input: {user_input}")
|
||||
|
||||
try:
|
||||
response = asyncio.run(agent.amessage(user_input))
|
||||
click.echo(f" Response: {response.content[:500]}")
|
||||
except Exception as e:
|
||||
click.secho(f" Error: {e}", fg="red")
|
||||
continue
|
||||
|
||||
if case.criteria:
|
||||
click.echo(f" Criteria: {case.criteria}")
|
||||
|
||||
feedback = click.prompt(
|
||||
" Feedback (Enter to skip, or type feedback)",
|
||||
default="",
|
||||
show_default=False,
|
||||
)
|
||||
if feedback.strip():
|
||||
agent.train(
|
||||
feedback=feedback.strip(),
|
||||
task_context=f"Input: {user_input}\nResponse: {response.content[:300]}",
|
||||
)
|
||||
click.secho(" ✓ Feedback saved as canonical memory", fg="green")
|
||||
|
||||
agents_trained += 1
|
||||
|
||||
click.echo()
|
||||
if agents_trained == 0:
|
||||
click.secho("No agents with matching benchmark cases found.", fg="yellow")
|
||||
else:
|
||||
click.secho(f"Training complete ({agents_trained} agent(s)).", fg="green", bold=True)
|
||||
|
||||
|
||||
@crewai.command()
|
||||
@@ -346,14 +454,14 @@ def memory(
|
||||
"--n_iterations",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of iterations to Test the crew",
|
||||
help="Number of iterations to run (Crew) or repetitions per case (NewAgent).",
|
||||
)
|
||||
@click.option(
|
||||
"-m",
|
||||
"--model",
|
||||
type=str,
|
||||
default="gpt-4o-mini",
|
||||
help="LLM Model to run the tests on the Crew. For now only accepting only OpenAI models.",
|
||||
default=None,
|
||||
help="LLM model to test with. For NewAgent, defaults to each agent's configured model.",
|
||||
)
|
||||
@click.option(
|
||||
"-f",
|
||||
@@ -361,17 +469,136 @@ def memory(
|
||||
"trained_agents_file",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to a trained-agents pickle (produced by `crewai train -f`). "
|
||||
"When set, agents load suggestions from this file instead of the "
|
||||
"default trained_agents_data.pkl. Equivalent to setting "
|
||||
"CREWAI_TRAINED_AGENTS_FILE."
|
||||
),
|
||||
help="Path to a trained-agents pickle (Crew projects only).",
|
||||
)
|
||||
def test(n_iterations: int, model: str, trained_agents_file: str | None) -> None:
|
||||
"""Test the crew and evaluate the results."""
|
||||
click.echo(f"Testing the crew for {n_iterations} iterations with model {model}")
|
||||
evaluate_crew(n_iterations, model, trained_agents_file=trained_agents_file)
|
||||
@click.option(
|
||||
"--threshold",
|
||||
type=float,
|
||||
default=0.7,
|
||||
help="Minimum score to pass a test case (NewAgent only, 0.0-1.0).",
|
||||
)
|
||||
@click.option(
|
||||
"--judge-model",
|
||||
type=str,
|
||||
default="openai/gpt-4o-mini",
|
||||
help="LLM model for evaluation judging (NewAgent only).",
|
||||
)
|
||||
def test(
|
||||
n_iterations: int,
|
||||
model: str | None,
|
||||
trained_agents_file: str | None,
|
||||
threshold: float,
|
||||
judge_model: str,
|
||||
) -> None:
|
||||
"""Test the crew or agents and evaluate the results.
|
||||
|
||||
Auto-detects project type: if agents/ directory exists with .json/.jsonc
|
||||
files, runs NewAgent benchmarks. Otherwise falls back to legacy Crew testing.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from crewai_cli.run_crew import _needs_uv_relaunch, _relaunch_via_uv
|
||||
|
||||
agents_dir = Path("agents")
|
||||
agent_files = sorted(agents_dir.glob("*.json")) + sorted(agents_dir.glob("*.jsonc")) if agents_dir.is_dir() else []
|
||||
|
||||
if agent_files:
|
||||
if _needs_uv_relaunch():
|
||||
uv_args = ["test", "-n", str(n_iterations), "--threshold", str(threshold), "--judge-model", judge_model]
|
||||
if model:
|
||||
uv_args.extend(["-m", model])
|
||||
if trained_agents_file:
|
||||
uv_args.extend(["-f", trained_agents_file])
|
||||
_relaunch_via_uv(uv_args)
|
||||
_test_new_agents(agent_files, n_iterations, model, threshold, judge_model)
|
||||
else:
|
||||
crew_model = model or "gpt-4o-mini"
|
||||
click.echo(f"Testing the crew for {n_iterations} iterations with model {crew_model}")
|
||||
evaluate_crew(n_iterations, crew_model, trained_agents_file=trained_agents_file)
|
||||
|
||||
|
||||
def _test_new_agents(
|
||||
agent_files: list,
|
||||
n_iterations: int,
|
||||
model: str | None,
|
||||
threshold: float,
|
||||
judge_model: str,
|
||||
) -> None:
|
||||
"""Run NewAgent test cases with pass/fail threshold."""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from crewai_cli.benchmark import (
|
||||
format_results_table,
|
||||
load_benchmark_cases,
|
||||
run_benchmark,
|
||||
)
|
||||
|
||||
benchmarks_dir = Path("benchmarks")
|
||||
all_passed = True
|
||||
agents_tested = 0
|
||||
|
||||
for agent_path in agent_files:
|
||||
agent_name = agent_path.stem
|
||||
cases_path = benchmarks_dir / f"{agent_name}_cases.json"
|
||||
|
||||
if not cases_path.exists():
|
||||
click.secho(f" Skipping {agent_name} — no {cases_path} found", fg="yellow")
|
||||
continue
|
||||
|
||||
try:
|
||||
cases = load_benchmark_cases(cases_path)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
click.secho(f" Error loading cases for {agent_name}: {e}", fg="red")
|
||||
all_passed = False
|
||||
continue
|
||||
|
||||
model_list = [model] if model else None
|
||||
|
||||
click.echo()
|
||||
click.secho(f"Testing {agent_name} ({len(cases)} cases)", fg="cyan", bold=True)
|
||||
|
||||
try:
|
||||
results_by_model = asyncio.run(
|
||||
run_benchmark(
|
||||
agent_def=str(agent_path),
|
||||
cases=cases,
|
||||
models=model_list,
|
||||
judge_model=judge_model,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
click.secho(f" Error running tests for {agent_name}: {e}", fg="red")
|
||||
all_passed = False
|
||||
continue
|
||||
|
||||
agents_tested += 1
|
||||
|
||||
for model_name, results in results_by_model.items():
|
||||
click.echo(format_results_table(results))
|
||||
|
||||
failed = [r for r in results if r.score < threshold]
|
||||
if failed:
|
||||
all_passed = False
|
||||
click.secho(
|
||||
f" FAILED: {len(failed)}/{len(results)} cases below threshold ({threshold})",
|
||||
fg="red",
|
||||
)
|
||||
else:
|
||||
click.secho(
|
||||
f" PASSED: all {len(results)} cases >= {threshold}",
|
||||
fg="green",
|
||||
)
|
||||
|
||||
click.echo()
|
||||
if agents_tested == 0:
|
||||
click.secho("No agents with matching benchmark cases found.", fg="yellow")
|
||||
raise SystemExit(1)
|
||||
elif all_passed:
|
||||
click.secho(f"All tests passed ({agents_tested} agent(s)).", fg="green", bold=True)
|
||||
else:
|
||||
click.secho("Some tests failed.", fg="red", bold=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
@crewai.command(
|
||||
@@ -600,6 +827,145 @@ def flow_add_crew(crew_name: str) -> None:
|
||||
add_crew_to_flow(crew_name)
|
||||
|
||||
|
||||
@crewai.group()
|
||||
def agent() -> None:
|
||||
"""Agent management commands."""
|
||||
|
||||
|
||||
@agent.command(name="reset-history")
|
||||
@click.argument("name")
|
||||
@click.option(
|
||||
"--keep-provenance",
|
||||
is_flag=True,
|
||||
help="Keep the provenance (decision audit trail) when clearing history.",
|
||||
)
|
||||
def agent_reset_history(name: str, keep_provenance: bool) -> None:
|
||||
"""Clear conversation history for the named agent."""
|
||||
from pathlib import Path
|
||||
|
||||
conversations_dir = Path.cwd() / ".crewai" / "conversations"
|
||||
history_path = conversations_dir / f"{name}.json"
|
||||
provenance_path = conversations_dir / f"{name}_provenance.json"
|
||||
|
||||
cleared: list[str] = []
|
||||
|
||||
if history_path.exists():
|
||||
history_path.unlink()
|
||||
cleared.append("conversation history")
|
||||
|
||||
if not keep_provenance and provenance_path.exists():
|
||||
provenance_path.unlink()
|
||||
cleared.append("provenance log")
|
||||
|
||||
if cleared:
|
||||
click.secho(
|
||||
f"Cleared {' and '.join(cleared)} for agent '{name}'.",
|
||||
fg="green",
|
||||
)
|
||||
else:
|
||||
click.secho(
|
||||
f"No conversation history found for agent '{name}'.",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
|
||||
@agent.command(name="memory")
|
||||
@click.argument("name")
|
||||
@click.option("--search", "-s", default=None, help="Search memories by keyword")
|
||||
@click.option("--clear", is_flag=True, help="Clear all memories")
|
||||
@click.option("--limit", "-n", "limit_", default=10, help="Number of memories to show")
|
||||
def agent_memory(name: str, search: str | None, clear: bool, limit_: int) -> None:
|
||||
"""Inspect or manage agent memories."""
|
||||
from pathlib import Path
|
||||
|
||||
agents_dir = Path.cwd() / "agents"
|
||||
agent_path = None
|
||||
for ext in (".json", ".jsonc"):
|
||||
p = agents_dir / f"{name}{ext}"
|
||||
if p.exists():
|
||||
agent_path = p
|
||||
break
|
||||
|
||||
if not agent_path:
|
||||
click.echo(f"Agent '{name}' not found in agents/ directory.")
|
||||
return
|
||||
|
||||
try:
|
||||
from crewai.new_agent.definition_parser import load_agent_from_definition
|
||||
|
||||
agent_instance = load_agent_from_definition(agent_path, agents_dir)
|
||||
except Exception as e:
|
||||
click.echo(f"Failed to load agent '{name}': {e}")
|
||||
return
|
||||
|
||||
if agent_instance is None:
|
||||
click.echo(f"Could not create agent '{name}'.")
|
||||
return
|
||||
|
||||
if clear:
|
||||
if click.confirm(f"Clear all memories for '{name}'?"):
|
||||
if hasattr(agent_instance, "_memory_instance") and agent_instance._memory_instance:
|
||||
try:
|
||||
agent_instance._memory_instance.reset()
|
||||
click.echo(f"Memories cleared for '{name}'.")
|
||||
except Exception as e:
|
||||
click.echo(f"Failed to clear memories: {e}")
|
||||
else:
|
||||
click.echo(f"No memory configured for '{name}'.")
|
||||
return
|
||||
|
||||
if not hasattr(agent_instance, "_memory_instance") or not agent_instance._memory_instance:
|
||||
click.echo(f"No memory configured for '{name}'.")
|
||||
return
|
||||
|
||||
# GAP-93: Rich formatted output for agent memory inspection
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
except ImportError:
|
||||
# Fall back to plain text if rich is not available
|
||||
Console = None # type: ignore[assignment,misc]
|
||||
|
||||
try:
|
||||
if search:
|
||||
results = agent_instance._memory_instance.recall(search, limit=limit_, depth="shallow")
|
||||
else:
|
||||
results = agent_instance._memory_instance.list_records(limit=limit_)
|
||||
|
||||
if not results:
|
||||
msg = f"No memories matching '{search}'" if search else f"No memories stored for '{name}'."
|
||||
click.echo(msg)
|
||||
return
|
||||
|
||||
if Console is not None:
|
||||
console = Console()
|
||||
title = f"Memories matching '{search}' — {name}" if search else f"Memories — {name}"
|
||||
table = Table(title=title, show_lines=True)
|
||||
table.add_column("#", style="dim", width=4)
|
||||
table.add_column("Content", min_width=40)
|
||||
table.add_column("Type", width=10)
|
||||
table.add_column("Scope", width=10)
|
||||
|
||||
for i, mem in enumerate(results, 1):
|
||||
record = getattr(mem, "record", mem)
|
||||
content = getattr(record, "content", "") or str(mem)
|
||||
if len(content) > 200:
|
||||
content = content[:200] + "..."
|
||||
meta = getattr(record, "metadata", {}) or {}
|
||||
mem_type = meta.get("type", "raw")
|
||||
scope = getattr(record, "scope", meta.get("scope", "—"))
|
||||
table.add_row(str(i), content, mem_type, scope)
|
||||
|
||||
console.print(table)
|
||||
else:
|
||||
heading = f"Memories matching '{search}':" if search else f"Recent memories for '{name}':"
|
||||
click.echo(heading)
|
||||
for i, r in enumerate(results, 1):
|
||||
click.echo(f" {i}. {str(r)[:100]}")
|
||||
except Exception as e:
|
||||
click.echo(f"Memory operation failed: {e}")
|
||||
|
||||
|
||||
@crewai.group()
|
||||
def triggers() -> None:
|
||||
"""Trigger related commands. Use 'crewai triggers list' to see available triggers, or 'crewai triggers run app_slug/trigger_slug' to execute."""
|
||||
@@ -956,5 +1322,73 @@ def checkpoint_prune(
|
||||
prune_checkpoints(ctx.obj["location"], keep, older_than, dry_run)
|
||||
|
||||
|
||||
@crewai.command()
|
||||
@click.argument("agent_path", type=click.Path(exists=True))
|
||||
@click.argument("cases_path", type=click.Path(exists=True))
|
||||
@click.option(
|
||||
"--models",
|
||||
"-m",
|
||||
multiple=True,
|
||||
help="Models to compare (e.g., openai/gpt-4o openai/gpt-4o-mini)",
|
||||
)
|
||||
@click.option(
|
||||
"--judge-model",
|
||||
default="openai/gpt-4o-mini",
|
||||
help="Model for LLM judge evaluation",
|
||||
)
|
||||
def benchmark(
|
||||
agent_path: str,
|
||||
cases_path: str,
|
||||
models: tuple[str, ...],
|
||||
judge_model: str,
|
||||
) -> None:
|
||||
"""Run agent against test cases and report results."""
|
||||
import asyncio
|
||||
|
||||
from crewai_cli.benchmark import (
|
||||
format_comparison_table,
|
||||
format_results_table,
|
||||
load_benchmark_cases,
|
||||
run_benchmark,
|
||||
)
|
||||
|
||||
try:
|
||||
cases = load_benchmark_cases(cases_path)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
click.secho(f"Error loading benchmark cases: {e}", fg="red")
|
||||
raise SystemExit(1) from e
|
||||
|
||||
click.echo(f"Loaded {len(cases)} benchmark case(s) from {cases_path}")
|
||||
click.echo(f"Agent definition: {agent_path}")
|
||||
|
||||
model_list = list(models) if models else None
|
||||
if model_list:
|
||||
click.echo(f"Models to compare: {', '.join(model_list)}")
|
||||
click.echo(f"Judge model: {judge_model}")
|
||||
click.echo()
|
||||
|
||||
try:
|
||||
results_by_model = asyncio.run(
|
||||
run_benchmark(
|
||||
agent_def=agent_path,
|
||||
cases=cases,
|
||||
models=model_list,
|
||||
judge_model=judge_model,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
click.secho(f"Error running benchmark: {e}", fg="red")
|
||||
raise SystemExit(1) from e
|
||||
|
||||
# Print results for each model
|
||||
for model, results in results_by_model.items():
|
||||
click.echo(format_results_table(results))
|
||||
click.echo()
|
||||
|
||||
# Print comparison if multiple models
|
||||
if len(results_by_model) > 1:
|
||||
click.echo(format_comparison_table(results_by_model))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
crewai()
|
||||
|
||||
754
lib/cli/src/crewai_cli/create_agent.py
Normal file
754
lib/cli/src/crewai_cli/create_agent.py
Normal file
@@ -0,0 +1,754 @@
|
||||
"""Create agent definitions via interactive prompts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
|
||||
from crewai_cli.constants import ENV_VARS, MODELS
|
||||
from crewai_cli.utils import load_env_vars, write_env_file
|
||||
|
||||
|
||||
AGENT_TEMPLATE = """\
|
||||
{{
|
||||
// Agent identity — defines the agent's persona and expertise
|
||||
// These three fields shape how the agent thinks and communicates
|
||||
"name": "{name}",
|
||||
|
||||
// What this agent does (any role you want)
|
||||
"role": "{role}",
|
||||
|
||||
// The agent's primary objective
|
||||
"goal": "{goal}",
|
||||
|
||||
// Background context that shapes personality and approach
|
||||
"backstory": "{backstory}",
|
||||
|
||||
// Which LLM powers this agent
|
||||
// Format: "provider/model" — e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-20250514"
|
||||
"llm": "{llm}",
|
||||
|
||||
// Separate LLM for tool/function calls (optional, defaults to main LLM)
|
||||
// Useful for using a cheaper model for tool routing
|
||||
// "function_calling_llm": "openai/gpt-4o-mini",
|
||||
|
||||
// Tools this agent can use — referenced by name from the crewai-tools package
|
||||
// See: https://docs.crewai.com/tools for available tools
|
||||
// Use "custom:tool_name" for custom tools defined in your tools/ directory
|
||||
"tools": [],
|
||||
|
||||
// MCP servers — external tool servers following the Model Context Protocol
|
||||
// Can be URLs ("https://mcp.example.com") or platform slugs ("notion")
|
||||
"mcps": [],
|
||||
|
||||
// Platform app integrations — managed by CrewAI Platform
|
||||
// App name ("gmail") or app/action ("gmail/send_email")
|
||||
"apps": [],
|
||||
|
||||
// Coworkers — other agents this agent can delegate work to
|
||||
// {{"ref": "name"}} for local agents in agents/ directory
|
||||
// {{"amp": "handle"}} for agents from the CrewAI AMP repository (your org)
|
||||
// {{"amp": "handle", "llm": "..."}} for AMP agents with LLM override
|
||||
// {{"a2a": "url"}} for remote agents via A2A protocol
|
||||
"coworkers": [],
|
||||
|
||||
// Knowledge sources — files/directories the agent can search for context
|
||||
// Supports: PDF, CSV, JSON, TXT, Excel, and directories
|
||||
"knowledge_sources": [],
|
||||
|
||||
// Output guardrail — validates agent responses before sending to user
|
||||
// "type": "llm" uses an LLM to check the response against instructions
|
||||
// Remove this block to disable guardrails
|
||||
// "guardrail": {{
|
||||
// "type": "llm",
|
||||
// "instructions": "Never reveal internal pricing information.",
|
||||
// "llm": "openai/gpt-4o-mini"
|
||||
// }},
|
||||
|
||||
// Settings — all have sensible defaults, only override what you need
|
||||
"settings": {{
|
||||
// Agent remembers across conversations
|
||||
"memory": true,
|
||||
|
||||
// Enable extended thinking / chain-of-thought
|
||||
"reasoning": true,
|
||||
|
||||
// Dreaming: consolidate memories over time into canonical insights
|
||||
"self_improving": true,
|
||||
|
||||
// Agent plans before complex tasks
|
||||
"planning": true,
|
||||
|
||||
// Agent decides at runtime whether to plan (default: true)
|
||||
// "auto_plan": true,
|
||||
|
||||
// Allow agent to spawn parallel copies for subtasks (default: true)
|
||||
// "can_spawn_copies": true,
|
||||
|
||||
// How deep spawned copies can nest (default: 1)
|
||||
// "max_spawn_depth": 1,
|
||||
|
||||
// Max parallel copies running at once (default: 4)
|
||||
// "max_concurrent_spawns": 4,
|
||||
|
||||
// Messages sent to LLM per turn, null = all (default: null)
|
||||
// "max_history_messages": null,
|
||||
|
||||
// Detect claimed-but-not-done actions (default: false)
|
||||
// "narration_guard": false,
|
||||
|
||||
// Hours between dreaming cycles (default: 24)
|
||||
// "dreaming_interval_hours": 24,
|
||||
|
||||
// New memories before dreaming triggers (default: 10)
|
||||
// "dreaming_trigger_threshold": 10,
|
||||
|
||||
// Separate LLM for dreaming (default: uses agent's LLM)
|
||||
// "dreaming_llm": "openai/gpt-4o-mini",
|
||||
|
||||
// Provenance detail level: "minimal", "standard", or "detailed"
|
||||
// "provenance_detail": "standard"
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
PROJECT_CONFIG_TEMPLATE = """\
|
||||
{
|
||||
// Project configuration for crewai agents
|
||||
// Rooms define how agents collaborate in the TUI
|
||||
|
||||
"rooms": {
|
||||
"common": {
|
||||
// Which agents participate in this room
|
||||
"agents": [],
|
||||
|
||||
// Engagement mode:
|
||||
// "dm" — chat with one agent at a time (default)
|
||||
// "tagged" — @mention to direct messages
|
||||
// "organic" — all agents see messages, respond if relevant
|
||||
"engagement": "dm"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
_STARTER_CASES = """\
|
||||
[
|
||||
{
|
||||
"input": "Hello, what can you help me with?",
|
||||
"criteria": "The agent should clearly describe its role and capabilities."
|
||||
}
|
||||
]
|
||||
"""
|
||||
|
||||
|
||||
_PROVIDER_TO_EXTRA: dict[str, str] = {
|
||||
# Native providers with dedicated SDK extras
|
||||
"anthropic": "anthropic",
|
||||
"gemini": "google-genai",
|
||||
"google": "google-genai",
|
||||
"azure": "azure-ai-inference",
|
||||
"azure_openai": "azure-ai-inference",
|
||||
"bedrock": "bedrock",
|
||||
"aws": "aws",
|
||||
# Providers that route through litellm
|
||||
"watsonx": "litellm",
|
||||
"groq": "litellm",
|
||||
"nvidia_nim": "litellm",
|
||||
"huggingface": "litellm",
|
||||
"sambanova": "litellm",
|
||||
# OpenAI-compatible providers — no extra needed:
|
||||
# openai, ollama, cerebras, deepseek, openrouter, hosted_vllm, dashscope
|
||||
}
|
||||
|
||||
_PROVIDER_BONUS_EXTRAS: dict[str, list[str]] = {
|
||||
"watsonx": ["watson"],
|
||||
}
|
||||
|
||||
|
||||
_GITIGNORE_TEMPLATE = """\
|
||||
.env
|
||||
__pycache__/
|
||||
.DS_Store
|
||||
.crewai/
|
||||
"""
|
||||
|
||||
|
||||
def _build_pyproject(project_name: str, crewai_version: str, llm_provider: str) -> str:
|
||||
"""Build pyproject.toml content with the right LLM provider extra."""
|
||||
extras = ["tools"]
|
||||
provider_extra = _PROVIDER_TO_EXTRA.get(llm_provider, "")
|
||||
if provider_extra and provider_extra not in extras:
|
||||
extras.append(provider_extra)
|
||||
for bonus in _PROVIDER_BONUS_EXTRAS.get(llm_provider, []):
|
||||
if bonus not in extras:
|
||||
extras.append(bonus)
|
||||
|
||||
extras_str = ",".join(extras)
|
||||
|
||||
lines = [
|
||||
"[project]",
|
||||
f'name = "{project_name}"',
|
||||
'version = "0.1.0"',
|
||||
'description = "CrewAI agent project"',
|
||||
'requires-python = ">=3.10,<3.14"',
|
||||
"dependencies = [",
|
||||
f' "crewai[{extras_str}]>={crewai_version}",',
|
||||
f' "crewai-cli>={crewai_version}",',
|
||||
"]",
|
||||
"",
|
||||
"[tool.uv]",
|
||||
'prerelease = "allow"',
|
||||
"constraint-dependencies = [",
|
||||
' "onnxruntime<=1.25.1",',
|
||||
"]",
|
||||
"",
|
||||
"[tool.crewai]",
|
||||
'type = "agent"',
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _bootstrap_project(base: Path, llm_model: str = "") -> None:
|
||||
"""Create project structure if it doesn't exist yet."""
|
||||
agents_dir = base / "agents"
|
||||
agents_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tools_dir = base / "tools"
|
||||
tools_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
benchmarks_dir = base / "benchmarks"
|
||||
benchmarks_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
config_path = base / "config.json"
|
||||
if not config_path.exists():
|
||||
config_path.write_text(PROJECT_CONFIG_TEMPLATE, encoding="utf-8")
|
||||
|
||||
provider = llm_model.split("/")[0].lower() if "/" in llm_model else ""
|
||||
pyproject_path = base / "pyproject.toml"
|
||||
if not pyproject_path.exists():
|
||||
crewai_version = _get_crewai_version()
|
||||
pyproject_path.write_text(
|
||||
_build_pyproject(base.name, crewai_version, provider),
|
||||
encoding="utf-8",
|
||||
)
|
||||
else:
|
||||
_maybe_add_provider_extra(pyproject_path, provider)
|
||||
|
||||
gitignore_path = base / ".gitignore"
|
||||
if not gitignore_path.exists():
|
||||
gitignore_path.write_text(_GITIGNORE_TEMPLATE, encoding="utf-8")
|
||||
|
||||
|
||||
def _maybe_add_provider_extra(pyproject_path: Path, provider: str) -> None:
|
||||
"""If the pyproject.toml exists but doesn't include the provider extra, add it."""
|
||||
all_extras = []
|
||||
primary = _PROVIDER_TO_EXTRA.get(provider, "")
|
||||
if primary:
|
||||
all_extras.append(primary)
|
||||
all_extras.extend(_PROVIDER_BONUS_EXTRAS.get(provider, []))
|
||||
if not all_extras:
|
||||
return
|
||||
try:
|
||||
content = pyproject_path.read_text(encoding="utf-8")
|
||||
missing = [
|
||||
e for e in all_extras
|
||||
if f"[{e}]" not in content and f",{e}]" not in content and f",{e}," not in content
|
||||
]
|
||||
if not missing:
|
||||
return
|
||||
import re as _re
|
||||
suffix = "," + ",".join(missing)
|
||||
def _add_extras(m: _re.Match) -> str:
|
||||
bracket = m.group(0)
|
||||
return bracket[:-1] + suffix + "]"
|
||||
updated = _re.sub(r'crewai\[[^\]]+\]', _add_extras, content, count=1)
|
||||
if updated != content:
|
||||
pyproject_path.write_text(updated, encoding="utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _get_crewai_version() -> str:
|
||||
"""Get the installed crewai version for the dependency pin."""
|
||||
try:
|
||||
from crewai_cli.version import get_crewai_version
|
||||
return get_crewai_version()
|
||||
except Exception:
|
||||
return "1.14.5"
|
||||
|
||||
|
||||
def _run_uv_sync(base: Path) -> None:
|
||||
"""Run uv sync to install dependencies."""
|
||||
click.echo()
|
||||
click.secho("Installing dependencies...", fg="cyan")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["uv", "sync"],
|
||||
cwd=str(base),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
click.secho("Dependencies installed successfully.", fg="green")
|
||||
else:
|
||||
click.secho("Failed to install dependencies:", fg="red")
|
||||
if result.stderr:
|
||||
click.echo(result.stderr)
|
||||
click.echo("Try running: uv sync")
|
||||
except FileNotFoundError:
|
||||
click.secho(
|
||||
"uv not found. Install it (https://docs.astral.sh/uv/) then run: uv sync",
|
||||
fg="yellow",
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
click.secho("uv sync timed out. Run manually: uv sync", fg="yellow")
|
||||
except Exception as e:
|
||||
click.secho(f"Could not run uv sync: {e}", fg="yellow")
|
||||
click.echo("Run manually: uv sync")
|
||||
|
||||
|
||||
def _create_benchmark_cases(base: Path, agent_name: str) -> None:
|
||||
"""Create a starter benchmark cases file for the agent."""
|
||||
cases_path = base / "benchmarks" / f"{agent_name}_cases.json"
|
||||
if cases_path.exists():
|
||||
return
|
||||
cases_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cases_path.write_text(_STARTER_CASES, encoding="utf-8")
|
||||
|
||||
|
||||
_POPULAR_MODELS: list[tuple[str, str]] = [
|
||||
("openai/gpt-4o", "OpenAI GPT-4o"),
|
||||
("openai/gpt-4o-mini", "OpenAI GPT-4o Mini (cheaper)"),
|
||||
("openai/o3", "OpenAI o3 (reasoning)"),
|
||||
("anthropic/claude-sonnet-4-6", "Anthropic Claude Sonnet 4.6"),
|
||||
("anthropic/claude-haiku-4-5-20251001", "Anthropic Claude Haiku 4.5 (fast)"),
|
||||
("gemini/gemini-2.5-pro-exp-03-25", "Google Gemini 2.5 Pro"),
|
||||
("groq/llama-3.1-70b-versatile", "Groq Llama 3.1 70B (fast)"),
|
||||
("ollama/llama3.1", "Ollama Llama 3.1 (local)"),
|
||||
]
|
||||
|
||||
|
||||
_POPULAR_TOOLS: list[tuple[str, str]] = [
|
||||
("SerperDevTool", "Web search via Serper API"),
|
||||
("ScrapeWebsiteTool", "Scrape and extract content from URLs"),
|
||||
("FileReadTool", "Read local files"),
|
||||
("FileWriterTool", "Write content to local files"),
|
||||
("DirectoryReadTool", "List directory contents"),
|
||||
("CodeInterpreterTool", "Execute Python code in a sandbox"),
|
||||
("CSVSearchTool", "Search within CSV files"),
|
||||
("PDFSearchTool", "Search within PDF documents"),
|
||||
("JSONSearchTool", "Search within JSON files"),
|
||||
("GithubSearchTool", "Search GitHub repositories"),
|
||||
("YoutubeVideoSearchTool", "Search YouTube video transcripts"),
|
||||
("TavilySearchTool", "Web search via Tavily API"),
|
||||
("BraveSearchTool", "Web search via Brave API"),
|
||||
("RagTool", "RAG over custom knowledge sources"),
|
||||
("DallETool", "Generate images with DALL-E"),
|
||||
("VisionTool", "Analyze images with vision models"),
|
||||
]
|
||||
|
||||
|
||||
_AGENT_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
|
||||
|
||||
|
||||
# ── Arrow-key selection helpers ──────────────────────────────────
|
||||
|
||||
|
||||
_CYAN = "\033[36m"
|
||||
_BOLD = "\033[1m"
|
||||
_GREEN = "\033[32m"
|
||||
_DIM = "\033[2m"
|
||||
_RESET = "\033[0m"
|
||||
|
||||
|
||||
def _is_interactive() -> bool:
|
||||
"""Check if stdin/stdout are real terminals (not piped or in tests)."""
|
||||
try:
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _read_key() -> str:
|
||||
"""Read a single keypress. Returns 'up', 'down', 'enter', 'space', or the char."""
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
ch = msvcrt.getwch()
|
||||
if ch in ("\x00", "\xe0"):
|
||||
ch2 = msvcrt.getwch()
|
||||
return {"H": "up", "P": "down"}.get(ch2, "")
|
||||
if ch == "\r":
|
||||
return "enter"
|
||||
if ch == " ":
|
||||
return "space"
|
||||
if ch == "\x03":
|
||||
raise KeyboardInterrupt
|
||||
return ch
|
||||
|
||||
import termios
|
||||
import tty
|
||||
fd = sys.stdin.fileno()
|
||||
old = termios.tcgetattr(fd)
|
||||
try:
|
||||
tty.setcbreak(fd)
|
||||
ch = sys.stdin.read(1)
|
||||
if ch == "\x1b":
|
||||
seq = sys.stdin.read(2)
|
||||
if seq == "[A":
|
||||
return "up"
|
||||
if seq == "[B":
|
||||
return "down"
|
||||
return "esc"
|
||||
if ch in ("\r", "\n"):
|
||||
return "enter"
|
||||
if ch == " ":
|
||||
return "space"
|
||||
if ch == "\x03":
|
||||
raise KeyboardInterrupt
|
||||
return ch
|
||||
finally:
|
||||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
||||
|
||||
|
||||
def _draw_single(labels: list[str], cursor: int, *, clear: bool = False) -> None:
|
||||
"""Draw single-select menu. If clear=True, move cursor up first."""
|
||||
total = len(labels)
|
||||
if clear:
|
||||
sys.stdout.write(f"\033[{total}A")
|
||||
for i, label in enumerate(labels):
|
||||
if i == cursor:
|
||||
sys.stdout.write(f"\033[2K {_CYAN}→{_RESET} {_BOLD}{label}{_RESET}\n")
|
||||
else:
|
||||
sys.stdout.write(f"\033[2K {label}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _draw_multi(labels: list[str], cursor: int, selected: set[int], *, clear: bool = False) -> None:
|
||||
"""Draw multi-select menu with checkboxes."""
|
||||
hint = f" {_DIM}↑↓ navigate, space toggle, enter confirm{_RESET}"
|
||||
total = len(labels) + 1 # +1 for hint line
|
||||
if clear:
|
||||
sys.stdout.write(f"\033[{total}A")
|
||||
sys.stdout.write(f"\033[2K{hint}\n")
|
||||
for i, label in enumerate(labels):
|
||||
check = f"{_CYAN}[×]{_RESET}" if i in selected else "[ ]"
|
||||
arrow = f"{_CYAN}→{_RESET} " if i == cursor else " "
|
||||
bold = f"{_BOLD}{label}{_RESET}" if i == cursor else label
|
||||
sys.stdout.write(f"\033[2K {arrow}{check} {bold}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _clear_lines(n: int) -> None:
|
||||
"""Clear n lines above and position cursor at the top."""
|
||||
sys.stdout.write(f"\033[{n}A")
|
||||
for _ in range(n):
|
||||
sys.stdout.write("\033[2K\n")
|
||||
sys.stdout.write(f"\033[{n}A")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def create_agent(name: str | None = None) -> None:
|
||||
"""Create an agent definition interactively.
|
||||
|
||||
Both paths (with and without a name) ask the same structured
|
||||
questions and produce the same annotated JSONC output.
|
||||
"""
|
||||
click.secho("\nCrewAI Agent Creator\n", fg="cyan", bold=True)
|
||||
|
||||
if name is None:
|
||||
name = _prompt_agent_name()
|
||||
|
||||
base = Path.cwd()
|
||||
# Directories are bootstrapped now, pyproject written after model selection
|
||||
for d in ("agents", "tools", "benchmarks"):
|
||||
(base / d).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
dest = base / "agents" / f"{name}.jsonc"
|
||||
if dest.exists():
|
||||
if not click.confirm(f"File {dest} already exists. Overwrite?"):
|
||||
click.secho("Operation cancelled.", fg="yellow")
|
||||
return
|
||||
|
||||
click.secho(f"Configuring agent: {name}\n", fg="cyan")
|
||||
|
||||
role = click.prompt(" Role (what this agent does)", type=str)
|
||||
goal = click.prompt(" Goal (the agent's objective)", type=str)
|
||||
backstory = click.prompt(
|
||||
" Backstory (context that shapes personality, optional)",
|
||||
type=str, default="", show_default=False,
|
||||
)
|
||||
|
||||
llm = _select_model()
|
||||
|
||||
tools = _select_tools()
|
||||
|
||||
content = AGENT_TEMPLATE.format(
|
||||
name=name,
|
||||
role=role,
|
||||
goal=goal,
|
||||
backstory=backstory,
|
||||
llm=llm,
|
||||
)
|
||||
|
||||
if tools:
|
||||
tools_json = json.dumps(tools)
|
||||
content = content.replace('"tools": []', f'"tools": {tools_json}')
|
||||
|
||||
dest.write_text(content, encoding="utf-8")
|
||||
_bootstrap_project(base, llm)
|
||||
_add_agent_to_config(base, name)
|
||||
_create_benchmark_cases(base, name)
|
||||
_setup_env(base, llm)
|
||||
_run_uv_sync(base)
|
||||
|
||||
click.echo()
|
||||
click.secho(f"Agent created: {dest}", fg="green", bold=True)
|
||||
click.echo("Run: crewai run")
|
||||
|
||||
|
||||
def _select_model() -> str:
|
||||
"""Let the user pick an LLM model from popular options or type a custom one."""
|
||||
labels = [f"{label} ({model_id})" for model_id, label in _POPULAR_MODELS]
|
||||
labels.append("Other (enter manually)")
|
||||
|
||||
click.echo()
|
||||
click.secho(" LLM Model:", fg="cyan")
|
||||
|
||||
if _is_interactive():
|
||||
try:
|
||||
_draw_single(labels, 0)
|
||||
cursor = 0
|
||||
total = len(labels)
|
||||
while True:
|
||||
key = _read_key()
|
||||
if key == "up" and cursor > 0:
|
||||
cursor -= 1
|
||||
_draw_single(labels, cursor, clear=True)
|
||||
elif key == "down" and cursor < total - 1:
|
||||
cursor += 1
|
||||
_draw_single(labels, cursor, clear=True)
|
||||
elif key == "enter":
|
||||
_clear_lines(total)
|
||||
idx = cursor
|
||||
break
|
||||
except Exception:
|
||||
idx = _select_model_fallback(labels)
|
||||
else:
|
||||
idx = _select_model_fallback(labels)
|
||||
|
||||
if idx == len(_POPULAR_MODELS):
|
||||
custom = click.prompt(" Enter model (provider/model)", type=str)
|
||||
return custom.strip()
|
||||
|
||||
selected = _POPULAR_MODELS[idx][0]
|
||||
click.secho(f" → {selected}", fg="green")
|
||||
return selected
|
||||
|
||||
|
||||
def _select_model_fallback(labels: list[str]) -> int:
|
||||
"""Numbered fallback for non-TTY environments."""
|
||||
for idx, label in enumerate(labels, 1):
|
||||
click.echo(f" {idx}. {label}")
|
||||
click.echo()
|
||||
while True:
|
||||
choice = click.prompt(" Select a model", type=str, default="1")
|
||||
try:
|
||||
num = int(choice)
|
||||
if 1 <= num <= len(labels):
|
||||
return num - 1
|
||||
except ValueError:
|
||||
pass
|
||||
click.secho(f" Invalid choice. Enter 1-{len(labels)}.", fg="red")
|
||||
|
||||
|
||||
def _select_tools() -> list[str]:
|
||||
"""Let the user pick tools from popular options and/or add custom ones."""
|
||||
labels = [f"{cls_name:<28s} {desc}" for cls_name, desc in _POPULAR_TOOLS]
|
||||
labels.append("Add custom tool class names")
|
||||
|
||||
click.echo()
|
||||
click.secho(" Tools (press Enter to skip):", fg="cyan")
|
||||
|
||||
if _is_interactive():
|
||||
try:
|
||||
indices = _select_tools_interactive(labels)
|
||||
except Exception:
|
||||
indices = _select_tools_fallback(labels)
|
||||
else:
|
||||
indices = _select_tools_fallback(labels)
|
||||
|
||||
selected: list[str] = []
|
||||
has_custom = False
|
||||
for idx in indices:
|
||||
if idx == len(_POPULAR_TOOLS):
|
||||
has_custom = True
|
||||
elif 0 <= idx < len(_POPULAR_TOOLS):
|
||||
cls_name = _POPULAR_TOOLS[idx][0]
|
||||
if cls_name not in selected:
|
||||
selected.append(cls_name)
|
||||
|
||||
if has_custom:
|
||||
custom = click.prompt(
|
||||
" Custom tool class names (comma-separated)",
|
||||
type=str, default="", show_default=False,
|
||||
)
|
||||
for name in custom.split(","):
|
||||
name = name.strip()
|
||||
if name and name not in selected:
|
||||
selected.append(name)
|
||||
|
||||
if selected:
|
||||
click.secho(f" → {', '.join(selected)}", fg="green")
|
||||
return selected
|
||||
|
||||
|
||||
def _select_tools_interactive(labels: list[str]) -> list[int]:
|
||||
"""Arrow-key multi-select for tools."""
|
||||
cursor = 0
|
||||
chosen: set[int] = set()
|
||||
total_lines = len(labels) + 1 # +1 for hint line
|
||||
|
||||
_draw_multi(labels, cursor, chosen)
|
||||
|
||||
while True:
|
||||
key = _read_key()
|
||||
if key == "up" and cursor > 0:
|
||||
cursor -= 1
|
||||
_draw_multi(labels, cursor, chosen, clear=True)
|
||||
elif key == "down" and cursor < len(labels) - 1:
|
||||
cursor += 1
|
||||
_draw_multi(labels, cursor, chosen, clear=True)
|
||||
elif key == "space":
|
||||
if cursor in chosen:
|
||||
chosen.discard(cursor)
|
||||
else:
|
||||
chosen.add(cursor)
|
||||
_draw_multi(labels, cursor, chosen, clear=True)
|
||||
elif key == "enter":
|
||||
_clear_lines(total_lines)
|
||||
return sorted(chosen)
|
||||
|
||||
|
||||
def _select_tools_fallback(labels: list[str]) -> list[int]:
|
||||
"""Numbered fallback for non-TTY environments."""
|
||||
for idx, label in enumerate(labels, 1):
|
||||
click.echo(f" {idx:2d}. {label}")
|
||||
click.echo()
|
||||
|
||||
raw = click.prompt(
|
||||
" Select tools (e.g. 1 3 5)", type=str, default="", show_default=False,
|
||||
)
|
||||
if not raw.strip():
|
||||
return []
|
||||
|
||||
indices: list[int] = []
|
||||
for token in raw.split():
|
||||
try:
|
||||
num = int(token)
|
||||
if 1 <= num <= len(labels):
|
||||
indices.append(num - 1)
|
||||
except ValueError:
|
||||
pass
|
||||
return indices
|
||||
|
||||
|
||||
def _setup_env(base: Path, llm_model: str) -> None:
|
||||
"""Prompt for API keys based on the selected LLM provider and write .env."""
|
||||
env_vars = load_env_vars(base)
|
||||
|
||||
provider = llm_model.split("/")[0].lower() if "/" in llm_model else ""
|
||||
if not provider:
|
||||
return
|
||||
|
||||
env_vars["MODEL"] = llm_model
|
||||
|
||||
already_set = all(
|
||||
details.get("key_name", "") in env_vars
|
||||
for details in ENV_VARS.get(provider, [])
|
||||
if "key_name" in details
|
||||
)
|
||||
if already_set and env_vars.get("MODEL"):
|
||||
return
|
||||
|
||||
if provider in ENV_VARS:
|
||||
click.echo()
|
||||
for details in ENV_VARS[provider]:
|
||||
key_name = details.get("key_name")
|
||||
if not key_name or key_name in env_vars:
|
||||
continue
|
||||
if details.get("default"):
|
||||
env_vars[key_name] = details.get("API_BASE", "")
|
||||
continue
|
||||
value = click.prompt(
|
||||
f" {details.get('prompt', f'Enter {key_name}')}",
|
||||
default="", show_default=False,
|
||||
)
|
||||
if value.strip():
|
||||
env_vars[key_name] = value.strip()
|
||||
|
||||
if env_vars:
|
||||
write_env_file(base, env_vars)
|
||||
click.secho("API keys saved to .env", fg="green")
|
||||
else:
|
||||
click.secho(
|
||||
"No API keys provided. Create a .env file manually before running.",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
|
||||
def _prompt_agent_name() -> str:
|
||||
"""Prompt for a valid agent identifier."""
|
||||
while True:
|
||||
name = click.prompt(
|
||||
" Agent identifier (lowercase, hyphens/underscores, no spaces)",
|
||||
type=str,
|
||||
)
|
||||
name = name.strip().lower()
|
||||
if _AGENT_NAME_RE.match(name):
|
||||
return name
|
||||
click.secho(
|
||||
" Invalid name — use lowercase letters, numbers, hyphens, or underscores.",
|
||||
fg="red",
|
||||
)
|
||||
|
||||
|
||||
def _strip_comments(text: str) -> str:
|
||||
"""Strip // and /* */ comments from JSONC text, then fix trailing commas."""
|
||||
result = re.sub(r'(?<!:)//.*?$', '', text, flags=re.MULTILINE)
|
||||
result = re.sub(r'/\*.*?\*/', '', result, flags=re.DOTALL)
|
||||
result = re.sub(r',\s*([}\]])', r'\1', result)
|
||||
return result
|
||||
|
||||
|
||||
def _add_agent_to_config(base: Path, agent_name: str) -> None:
|
||||
"""Add the agent to the common room in config.json."""
|
||||
config_path = base / "config.json"
|
||||
if not config_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
raw = config_path.read_text(encoding="utf-8")
|
||||
clean = _strip_comments(raw)
|
||||
config = json.loads(clean)
|
||||
|
||||
rooms = config.get("rooms", {})
|
||||
common = rooms.get("common", {"agents": [], "engagement": "dm"})
|
||||
agents = common.get("agents", [])
|
||||
if agent_name not in agents:
|
||||
agents.append(agent_name)
|
||||
common["agents"] = agents
|
||||
rooms["common"] = common
|
||||
config["rooms"] = rooms
|
||||
config_path.write_text(json.dumps(config, indent=2), encoding="utf-8")
|
||||
except Exception as e:
|
||||
click.echo(f"Warning: Could not update config.json: {e}", err=True)
|
||||
@@ -1,4 +1,5 @@
|
||||
from enum import Enum
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import click
|
||||
@@ -8,18 +9,60 @@ from packaging import version
|
||||
from crewai_cli.utils import build_env_with_all_tool_credentials, read_toml
|
||||
from crewai_cli.version import get_crewai_version
|
||||
|
||||
_UV_CONTEXT_VAR = "_CREWAI_UV"
|
||||
|
||||
|
||||
class CrewType(Enum):
|
||||
STANDARD = "standard"
|
||||
FLOW = "flow"
|
||||
|
||||
|
||||
def run_crew(trained_agents_file: str | None = None) -> None:
|
||||
"""Run the crew or flow by running a command in the UV environment.
|
||||
def _has_agents_dir() -> bool:
|
||||
"""Check if current directory has an agents/ directory with definitions."""
|
||||
from pathlib import Path
|
||||
agents_dir = Path.cwd() / "agents"
|
||||
if not agents_dir.is_dir():
|
||||
return False
|
||||
files = list(agents_dir.glob("*.json")) + list(agents_dir.glob("*.jsonc"))
|
||||
return len(files) > 0
|
||||
|
||||
Starting from version 0.103.0, this command can be used to run both
|
||||
standard crews and flows. For flows, it detects the type from pyproject.toml
|
||||
and automatically runs the appropriate command.
|
||||
|
||||
def _needs_uv_relaunch() -> bool:
|
||||
"""True when we should re-exec through ``uv run`` for the project venv."""
|
||||
if os.environ.get(_UV_CONTEXT_VAR):
|
||||
return False
|
||||
from pathlib import Path
|
||||
pyproject = Path.cwd() / "pyproject.toml"
|
||||
if not pyproject.exists():
|
||||
return False
|
||||
try:
|
||||
return 'type = "agent"' in pyproject.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _relaunch_via_uv(args: list[str]) -> None:
|
||||
"""Re-exec ``uv run crewai <args>`` inside the project venv, then exit."""
|
||||
env = {**os.environ, _UV_CONTEXT_VAR: "1"}
|
||||
cmd = ["uv", "run", "crewai", *args]
|
||||
try:
|
||||
result = subprocess.run(cmd, env=env)
|
||||
raise SystemExit(result.returncode)
|
||||
except FileNotFoundError:
|
||||
click.secho(
|
||||
"uv not found — running without project venv. "
|
||||
"Install uv (https://docs.astral.sh/uv/) for full provider support.",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
|
||||
def run_crew(trained_agents_file: str | None = None) -> None:
|
||||
"""Run the crew, flow, or agent TUI.
|
||||
|
||||
Detects the project type:
|
||||
- If agents/ directory exists with definitions: launch agent TUI
|
||||
- If pyproject.toml type is "flow": run the flow
|
||||
- Otherwise: run the crew
|
||||
|
||||
Args:
|
||||
trained_agents_file: Optional path to a trained-agents pickle produced
|
||||
@@ -27,6 +70,18 @@ def run_crew(trained_agents_file: str | None = None) -> None:
|
||||
``CREWAI_TRAINED_AGENTS_FILE`` so agents load suggestions from this
|
||||
file instead of the default ``trained_agents_data.pkl``.
|
||||
"""
|
||||
# Check for agents/ directory first — agent projects don't need pyproject.toml
|
||||
if _has_agents_dir():
|
||||
if _needs_uv_relaunch():
|
||||
uv_args = ["run"]
|
||||
if trained_agents_file:
|
||||
uv_args.extend(["-f", trained_agents_file])
|
||||
_relaunch_via_uv(uv_args)
|
||||
click.echo("Launching agent TUI...")
|
||||
from crewai_cli.agent_tui import run_agent_tui
|
||||
run_agent_tui()
|
||||
return
|
||||
|
||||
crewai_version = get_crewai_version()
|
||||
min_required_version = "0.71.0"
|
||||
pyproject_data = read_toml()
|
||||
|
||||
Reference in New Issue
Block a user