Update md5 to sha3256

- refactor code to multiple functions
- Added ability for users to select provider and model when uing crewai create command and ave API key to .env

.github/SECURITY.md

@@ -0,0 +1,23 @@
+CrewAI takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organization.
+
+If you believe you have found a security vulnerability in any CrewAI product or service, please report it to us as described below.
+
+## Reporting a Vulnerability
+
+Please do not report security vulnerabilities through public GitHub issues.
+
+To report a vulnerability, please email us at security@crewai.com.
+
+Please include the requested information listed below so that we can triage your report more quickly
+
+- Type of issue (e.g. SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue (please include screenshots if needed)
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+Once we have received your report, we will respond to you at the email address you provide. If the issue is confirmed, we will release a patch as soon as possible depending on the complexity of the issue.
+
+At this time, we are not offering a bug bounty program. Any rewards will be at our discretion.

.gitignore

@@ -17,3 +17,4 @@ rc-tests/*
 temp/*
 .vscode/*
 crew_tasks_output.json
+.dccache

docs/concepts/cli.mdx

@@ -6,7 +6,7 @@ icon: terminal
 
 # CrewAI CLI Documentation
 
-The CrewAI CLI provides a set of commands to interact with CrewAI, allowing you to create, train, run, and manage crews and pipelines.
+The CrewAI CLI provides a set of commands to interact with CrewAI, allowing you to create, train, run, and manage crews & flows.
 
 ## Installation
 
@@ -146,3 +146,34 @@ crewai run
 Make sure to run these commands from the directory where your CrewAI project is set up. 
 Some commands may require additional configuration or setup within your project structure.
 </Note>
+
+
+### 9. API Keys
+
+When running ```crewai create crew``` command, the CLI will first show you the top 5 most common LLM providers and ask you to select one.
+
+Once you've selected an LLM provider, you will be prompted for API keys.
+
+#### Initial API key providers
+
+The CLI will initiallyprompt for API keys for the following services:
+
+* OpenAI
+* Groq
+* Anthropic
+* Google Gemini
+
+When you select a provider, the CLI will prompt you to enter your API key.
+
+#### Other Options
+
+If you select option 6, you will be able to select from a list of LiteLLM supported providers.
+
+When you select a provider, the CLI will prompt you to enter the Key name and the API key.
+
+See the following link for each provider's key name:
+
+* [LiteLLM Providers](https://docs.litellm.ai/docs/providers)
+
+
+

src/crewai/agents/agent_builder/base_agent.py

@@ -1,7 +1,7 @@
 import uuid
 from abc import ABC, abstractmethod
 from copy import copy as shallow_copy
-from hashlib import md5
+from hashlib import sha256
 from typing import Any, Dict, List, Optional, TypeVar
 
 from pydantic import (
@@ -181,7 +181,7 @@ class BaseAgent(ABC, BaseModel):
             self._original_goal or self.goal,
             self._original_backstory or self.backstory,
         ]
-        return md5("|".join(source).encode(), usedforsecurity=False).hexdigest()
+        return sha256("|".join(source).encode()).hexdigest()
 
     @abstractmethod
     def execute_task(

src/crewai/cli/constants.py

@@ -0,0 +1,19 @@
+ENV_VARS = {
+    'openai': ['OPENAI_API_KEY'],
+    'anthropic': ['ANTHROPIC_API_KEY'],
+    'gemini': ['GEMINI_API_KEY'],
+    'groq': ['GROQ_API_KEY'],
+    'ollama': ['FAKE_KEY'],
+}
+
+PROVIDERS = ['openai', 'anthropic', 'gemini', 'groq', 'ollama']
+
+MODELS = {
+    'openai': ['gpt-4', 'gpt-4o', 'gpt-4o-mini', 'o1-mini', 'o1-preview'],
+    'anthropic': ['claude-3-5-sonnet-20240620', 'claude-3-sonnet-20240229', 'claude-3-opus-20240229', 'claude-3-haiku-20240307'],
+    'gemini': ['gemini-1.5-flash', 'gemini-1.5-pro', 'gemini-gemma-2-9b-it', 'gemini-gemma-2-27b-it'],
+    'groq': ['llama-3.1-8b-instant', 'llama-3.1-70b-versatile', 'llama-3.1-405b-reasoning', 'gemma2-9b-it', 'gemma-7b-it'],
+    'ollama': ['llama3.1', 'mixtral'],
+}
+
+JSON_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"

src/crewai/cli/create_crew.py

@@ -1,12 +1,17 @@
 from pathlib import Path
-
 import click
-
-from crewai.cli.utils import copy_template
+from crewai.cli.utils import copy_template, load_env_vars, write_env_file
+from crewai.cli.provider import (
+    get_provider_data,
+    select_provider,
+    select_model,
+    PROVIDERS,
+)
+from crewai.cli.constants import ENV_VARS
+import sys
 
 
-def create_crew(name, parent_folder=None):
-    """Create a new crew."""
+def create_folder_structure(name, parent_folder=None):
     folder_name = name.replace(" ", "_").replace("-", "_").lower()
     class_name = name.replace("_", " ").replace("-", " ").title().replace(" ", "")
 
@@ -15,11 +20,19 @@ def create_crew(name, parent_folder=None):
     else:
         folder_path = Path(folder_name)
 
-    click.secho(
-        f"Creating {'crew' if parent_folder else 'folder'} {folder_name}...",
-        fg="green",
-        bold=True,
-    )
+    if folder_path.exists():
+        if not click.confirm(
+            f"Folder {folder_name} already exists. Do you want to override it?"
+        ):
+            click.secho("Operation cancelled.", fg="yellow")
+            sys.exit(0)
+        click.secho(f"Overriding folder {folder_name}...", fg="green", bold=True)
+    else:
+        click.secho(
+            f"Creating {'crew' if parent_folder else 'folder'} {folder_name}...",
+            fg="green",
+            bold=True,
+        )
 
     if not folder_path.exists():
         folder_path.mkdir(parents=True)
@@ -28,19 +41,119 @@ def create_crew(name, parent_folder=None):
             (folder_path / "src" / folder_name).mkdir(parents=True)
             (folder_path / "src" / folder_name / "tools").mkdir(parents=True)
             (folder_path / "src" / folder_name / "config").mkdir(parents=True)
-            with open(folder_path / ".env", "w") as file:
-                file.write("OPENAI_API_KEY=YOUR_API_KEY")
-    else:
-        click.secho(
-            f"\tFolder {folder_name} already exists. Please choose a different name.",
-            fg="red",
-        )
+
+    return folder_path, folder_name, class_name
+
+
+def copy_template_files(folder_path, name, class_name, parent_folder):
+    package_dir = Path(__file__).parent
+    templates_dir = package_dir / "templates" / "crew"
+
+    root_template_files = (
+        [".gitignore", "pyproject.toml", "README.md"] if not parent_folder else []
+    )
+    tools_template_files = ["tools/custom_tool.py", "tools/__init__.py"]
+    config_template_files = ["config/agents.yaml", "config/tasks.yaml"]
+    src_template_files = (
+        ["__init__.py", "main.py", "crew.py"] if not parent_folder else ["crew.py"]
+    )
+
+    for file_name in root_template_files:
+        src_file = templates_dir / file_name
+        dst_file = folder_path / file_name
+        copy_template(src_file, dst_file, name, class_name, folder_path.name)
+
+    src_folder = (
+        folder_path / "src" / folder_path.name if not parent_folder else folder_path
+    )
+
+    for file_name in src_template_files:
+        src_file = templates_dir / file_name
+        dst_file = src_folder / file_name
+        copy_template(src_file, dst_file, name, class_name, folder_path.name)
+
+    if not parent_folder:
+        for file_name in tools_template_files + config_template_files:
+            src_file = templates_dir / file_name
+            dst_file = src_folder / file_name
+            copy_template(src_file, dst_file, name, class_name, folder_path.name)
+
+
+def create_crew(name, parent_folder=None):
+    folder_path, folder_name, class_name = create_folder_structure(name, parent_folder)
+    env_vars = load_env_vars(folder_path)
+
+    existing_provider = None
+    for provider, env_keys in ENV_VARS.items():
+        if any(key in env_vars for key in env_keys):
+            existing_provider = provider
+            break
+
+    if existing_provider:
+        if not click.confirm(
+            f"Found existing environment variable configuration for {existing_provider.capitalize()}. Do you want to override it?"
+        ):
+            click.secho("Keeping existing provider configuration.", fg="yellow")
+            return
+
+    provider_models = get_provider_data()
+    if not provider_models:
         return
 
+    while True:
+        selected_provider = select_provider(provider_models)
+        if selected_provider is None:  # User typed 'q'
+            click.secho("Exiting...", fg="yellow")
+            sys.exit(0)
+        if selected_provider:  # Valid selection
+            break
+        click.secho(
+            "No provider selected. Please try again or press 'q' to exit.", fg="red"
+        )
+
+    while True:
+        selected_model = select_model(selected_provider, provider_models)
+        if selected_model is None:  # User typed 'q'
+            click.secho("Exiting...", fg="yellow")
+            sys.exit(0)
+        if selected_model:  # Valid selection
+            break
+        click.secho(
+            "No model selected. Please try again or press 'q' to exit.", fg="red"
+        )
+
+    if selected_provider in PROVIDERS:
+        api_key_var = ENV_VARS[selected_provider][0]
+    else:
+        api_key_var = click.prompt(
+            f"Enter the environment variable name for your {selected_provider.capitalize()} API key",
+            type=str,
+            default="",
+        )
+
+    api_key_value = ""
+    click.echo(
+        f"Enter your {selected_provider.capitalize()} API key (press Enter to skip): ",
+        nl=False,
+    )
+    try:
+        api_key_value = input()
+    except (KeyboardInterrupt, EOFError):
+        api_key_value = ""
+
+    if api_key_value.strip():
+        env_vars = {api_key_var: api_key_value}
+        write_env_file(folder_path, env_vars)
+        click.secho("API key saved to .env file", fg="green")
+    else:
+        click.secho("No API key provided. Skipping .env file creation.", fg="yellow")
+
+    env_vars["MODEL"] = selected_model
+    click.secho(f"Selected model: {selected_model}", fg="green")
+
     package_dir = Path(__file__).parent
     templates_dir = package_dir / "templates" / "crew"
 
-    # List of template files to copy
     root_template_files = (
         [".gitignore", "pyproject.toml", "README.md"] if not parent_folder else []
     )

src/crewai/cli/provider.py

@@ -0,0 +1,200 @@
+import json
+import time
+import requests
+from collections import defaultdict
+import click
+from pathlib import Path 
+from crewai.cli.constants import PROVIDERS, MODELS, JSON_URL
+
+
+
+def select_choice(prompt_message, choices):
+    """
+    Presents a list of choices to the user and prompts them to select one.
+    
+    Args:
+    - prompt_message (str): The message to display to the user before presenting the choices.
+    - choices (list): A list of options to present to the user.
+    
+    Returns:
+    - str: The selected choice from the list, or None if the user chooses to quit.
+    """
+
+    provider_models = get_provider_data()
+    if not provider_models:
+        return
+    click.secho(prompt_message, fg="cyan")
+    for idx, choice in enumerate(choices, start=1):
+        click.secho(f"{idx}. {choice}", fg="cyan")
+    click.secho("q. Quit", fg="cyan")
+    
+    while True:
+        choice = click.prompt("Enter the number of your choice or 'q' to quit", type=str)
+        
+        if choice.lower() == 'q':
+            return None
+            
+        try:
+            selected_index = int(choice) - 1
+            if 0 <= selected_index < len(choices):
+                return choices[selected_index]
+        except ValueError:
+            pass
+            
+        click.secho("Invalid selection. Please select a number between 1 and 6 or 'q' to quit.", fg="red")
+
+def select_provider(provider_models):
+    """
+    Presents a list of providers to the user and prompts them to select one.
+    
+    Args:
+    - provider_models (dict): A dictionary of provider models.
+    
+    Returns:
+    - str: The selected provider
+    - None: If user explicitly quits
+    """
+    predefined_providers = [p.lower() for p in PROVIDERS]
+    all_providers = sorted(set(predefined_providers + list(provider_models.keys())))
+
+    provider = select_choice("Select a provider to set up:", predefined_providers + ['other'])
+    if provider is None:  # User typed 'q'
+        return None
+        
+    if provider == 'other':
+        provider = select_choice("Select a provider from the full list:", all_providers)
+        if provider is None:  # User typed 'q'
+            return None
+            
+    return provider.lower() if provider else False
+
+def select_model(provider, provider_models):
+    """
+    Presents a list of models for a given provider to the user and prompts them to select one.
+    
+    Args:
+    - provider (str): The provider for which to select a model.
+    - provider_models (dict): A dictionary of provider models.
+    
+    Returns:
+    - str: The selected model, or None if the operation is aborted or an invalid selection is made.
+    """
+    predefined_providers = [p.lower() for p in PROVIDERS]
+
+    if provider in predefined_providers:
+        available_models = MODELS.get(provider, [])
+    else:
+        available_models = provider_models.get(provider, [])
+
+    if not available_models:
+        click.secho(f"No models available for provider '{provider}'.", fg="red")
+        return None
+
+    selected_model = select_choice(f"Select a model to use for {provider.capitalize()}:", available_models)
+    return selected_model
+
+def load_provider_data(cache_file, cache_expiry):
+    """
+    Loads provider data from a cache file if it exists and is not expired. If the cache is expired or corrupted, it fetches the data from the web.
+    
+    Args:
+    - cache_file (Path): The path to the cache file.
+    - cache_expiry (int): The cache expiry time in seconds.
+    
+    Returns:
+    - dict or None: The loaded provider data or None if the operation fails.
+    """
+    current_time = time.time()
+    if cache_file.exists() and (current_time - cache_file.stat().st_mtime) < cache_expiry:
+        data = read_cache_file(cache_file)
+        if data:
+            return data
+        click.secho("Cache is corrupted. Fetching provider data from the web...", fg="yellow")
+    else:
+        click.secho("Cache expired or not found. Fetching provider data from the web...", fg="cyan")
+    return fetch_provider_data(cache_file)
+
+def read_cache_file(cache_file):
+    """
+    Reads and returns the JSON content from a cache file. Returns None if the file contains invalid JSON.
+    
+    Args:
+    - cache_file (Path): The path to the cache file.
+    
+    Returns:
+    - dict or None: The JSON content of the cache file or None if the JSON is invalid.
+    """
+    try:
+        with open(cache_file, "r") as f:
+            return json.load(f)
+    except json.JSONDecodeError:
+        return None
+
+def fetch_provider_data(cache_file):
+    """
+    Fetches provider data from a specified URL and caches it to a file.
+    
+    Args:
+    - cache_file (Path): The path to the cache file.
+    
+    Returns:
+    - dict or None: The fetched provider data or None if the operation fails.
+    """
+    try:
+        response = requests.get(JSON_URL, stream=True, timeout=10)
+        response.raise_for_status()
+        data = download_data(response)
+        with open(cache_file, "w") as f:
+            json.dump(data, f)
+        return data
+    except requests.RequestException as e:
+        click.secho(f"Error fetching provider data: {e}", fg="red")
+    except json.JSONDecodeError:
+        click.secho("Error parsing provider data. Invalid JSON format.", fg="red")
+    return None
+
+def download_data(response):
+    """
+    Downloads data from a given HTTP response and returns the JSON content.
+    
+    Args:
+    - response (requests.Response): The HTTP response object.
+    
+    Returns:
+    - dict: The JSON content of the response.
+    """
+    total_size = int(response.headers.get('content-length', 0))
+    block_size = 8192
+    data_chunks = []
+    with click.progressbar(length=total_size, label='Downloading', show_pos=True) as progress_bar:
+        for chunk in response.iter_content(block_size):
+            if chunk:
+                data_chunks.append(chunk)
+                progress_bar.update(len(chunk))
+    data_content = b''.join(data_chunks)
+    return json.loads(data_content.decode('utf-8'))
+
+def get_provider_data():
+    """
+    Retrieves provider data from a cache file, filters out models based on provider criteria, and returns a dictionary of providers mapped to their models.
+    
+    Returns:
+    - dict or None: A dictionary of providers mapped to their models or None if the operation fails.
+    """
+    cache_dir = Path.home() / '.crewai'
+    cache_dir.mkdir(exist_ok=True)
+    cache_file = cache_dir / 'provider_cache.json'
+    cache_expiry = 24 * 3600 
+
+    data = load_provider_data(cache_file, cache_expiry)
+    if not data:
+        return None
+
+    provider_models = defaultdict(list)
+    for model_name, properties in data.items():
+        provider = properties.get("litellm_provider", "").strip().lower()
+        if 'http' in provider or provider == 'other':
+            continue
+        if provider:
+            provider_models[provider].append(model_name)
+    return provider_models

src/crewai/cli/utils.py

@@ -9,6 +9,7 @@ import click
 from rich.console import Console
 
 from crewai.cli.authentication.utils import TokenManager
+from crewai.cli.constants import ENV_VARS
 
 if sys.version_info >= (3, 11):
     import tomllib
@@ -200,3 +201,76 @@ def tree_find_and_replace(directory, find, replace):
                 new_dirpath = os.path.join(path, new_dirname)
                 old_dirpath = os.path.join(path, dirname)
                 os.rename(old_dirpath, new_dirpath)
+
+
+def load_env_vars(folder_path):
+    """
+    Loads environment variables from a .env file in the specified folder path.
+
+    Args:
+    - folder_path (Path): The path to the folder containing the .env file.
+
+    Returns:
+    - dict: A dictionary of environment variables.
+    """
+    env_file_path = folder_path / ".env"
+    env_vars = {}
+    if env_file_path.exists():
+        with open(env_file_path, "r") as file:
+            for line in file:
+                key, _, value = line.strip().partition("=")
+                if key and value:
+                    env_vars[key] = value
+    return env_vars
+
+
+def update_env_vars(env_vars, provider, model):
+    """
+    Updates environment variables with the API key for the selected provider and model.
+
+    Args:
+    - env_vars (dict): Environment variables dictionary.
+    - provider (str): Selected provider.
+    - model (str): Selected model.
+
+    Returns:
+    - None
+    """
+    api_key_var = ENV_VARS.get(
+        provider,
+        [
+            click.prompt(
+                f"Enter the environment variable name for your {provider.capitalize()} API key",
+                type=str,
+            )
+        ],
+    )[0]
+
+    if api_key_var not in env_vars:
+        try:
+            env_vars[api_key_var] = click.prompt(
+                f"Enter your {provider.capitalize()} API key", type=str, hide_input=True
+            )
+        except click.exceptions.Abort:
+            click.secho("Operation aborted by the user.", fg="red")
+            return None
+    else:
+        click.secho(f"API key already exists for {provider.capitalize()}.", fg="yellow")
+
+    env_vars["MODEL"] = model
+    click.secho(f"Selected model: {model}", fg="green")
+    return env_vars
+
+
+def write_env_file(folder_path, env_vars):
+    """
+    Writes environment variables to a .env file in the specified folder.
+
+    Args:
+    - folder_path (Path): The path to the folder where the .env file will be written.
+    - env_vars (dict): A dictionary of environment variables to write.
+    """
+    env_file_path = folder_path / ".env"
+    with open(env_file_path, "w") as file:
+        for key, value in env_vars.items():
+            file.write(f"{key}={value}\n")

src/crewai/crew.py

@@ -4,7 +4,7 @@ import os
 import uuid
 import warnings
 from concurrent.futures import Future
-from hashlib import md5
+from hashlib import sha256
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 from pydantic import (
@@ -388,7 +388,7 @@ class Crew(BaseModel):
         source = [agent.key for agent in self.agents] + [
             task.key for task in self.tasks
         ]
-        return md5("|".join(source).encode(), usedforsecurity=False).hexdigest()
+        return sha256("|".join(source).encode()).hexdigest()
 
     def _setup_from_config(self):
         assert self.config is not None, "Config should not be None."

src/crewai/task.py

@@ -5,7 +5,7 @@ import threading
 import uuid
 from concurrent.futures import Future
 from copy import copy
-from hashlib import md5
+from hashlib import sha256
 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
 
 from opentelemetry.trace import Span
@@ -196,7 +196,7 @@ class Task(BaseModel):
         expected_output = self._original_expected_output or self.expected_output
         source = [description, expected_output]
 
-        return md5("|".join(source).encode(), usedforsecurity=False).hexdigest()
+        return sha256("|".join(source).encode()).hexdigest()
 
     def execute_async(
         self,

tests/agents/agent_builder/base_agent_test.py

@@ -1,4 +1,4 @@
-import hashlib
+from hashlib import sha256
 from typing import Any, List, Optional
 
 from crewai.agents.agent_builder.base_agent import BaseAgent
@@ -32,5 +32,5 @@ def test_key():
         goal="test goal",
         backstory="test backstory",
     )
-    hash = hashlib.md5("test role|test goal|test backstory".encode()).hexdigest()
+    hash = sha256("test role|test goal|test backstory".encode()).hexdigest()
     assert agent.key == hash

tests/crew_test.py

@@ -1,6 +1,6 @@
 """Test Agent creation and execution basic functionality."""
 
-import hashlib
+from hashlib import sha256
 import json
 from concurrent.futures import Future
 from unittest import mock
@@ -2328,7 +2328,7 @@ def test_key():
         process=Process.sequential,
         tasks=tasks,
     )
-    hash = hashlib.md5(
+    hash = sha256(
         f"{researcher.key}|{writer.key}|{tasks[0].key}|{tasks[1].key}".encode()
     ).hexdigest()
 
@@ -2368,7 +2368,7 @@ def test_key_with_interpolated_inputs():
         process=Process.sequential,
         tasks=tasks,
     )
-    hash = hashlib.md5(
+    hash = sha256(
         f"{researcher.key}|{writer.key}|{tasks[0].key}|{tasks[1].key}".encode()
     ).hexdigest()
 

tests/task_test.py

@@ -1,6 +1,6 @@
 """Test Agent creation and execution basic functionality."""
 
-import hashlib
+from hashlib import sha256  
 import json
 import os
 from unittest.mock import MagicMock, patch
@@ -819,7 +819,7 @@ def test_key():
         description=original_description,
         expected_output=original_expected_output,
     )
-    hash = hashlib.md5(
+    hash = sha256(
         f"{original_description}|{original_expected_output}".encode()
     ).hexdigest()