From 6708d47d3956bdce3521d346ecb42abc57534150 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 21 Mar 2025 19:44:28 +0000 Subject: [PATCH] Fix CI issues and implement PR feedback improvements Co-Authored-By: Joe Moura --- docs/how-to/open-source-deployment.mdx | 1 + examples/deployment/deployment.yaml | 1 + src/crewai/deployment/cli.py | 8 +- src/crewai/deployment/config.py | 78 +++++++++++++++--- src/crewai/deployment/docker/container.py | 31 ++++--- src/crewai/deployment/docker/exceptions.py | 18 +++++ src/crewai/deployment/main.py | 81 +++++++++++++++---- .../deployment/templates/deployment.yaml | 1 + src/crewai/deployment/templates/server.py | 19 +++-- tests/deployment/test_deployment.py | 4 +- 10 files changed, 198 insertions(+), 44 deletions(-) create mode 100644 src/crewai/deployment/docker/exceptions.py diff --git a/docs/how-to/open-source-deployment.mdx b/docs/how-to/open-source-deployment.mdx index b66e48d0c..b9cde4b75 100644 --- a/docs/how-to/open-source-deployment.mdx +++ b/docs/how-to/open-source-deployment.mdx @@ -18,6 +18,7 @@ Create a file named `deployment.yaml`: # CrewAI Deployment Configuration name: my-crewai-app port: 8000 +host: 127.0.0.1 # Default to localhost for security # Crews configuration crews: diff --git a/examples/deployment/deployment.yaml b/examples/deployment/deployment.yaml index 1ed02a90d..25978fd7e 100644 --- a/examples/deployment/deployment.yaml +++ b/examples/deployment/deployment.yaml @@ -1,6 +1,7 @@ # CrewAI Deployment Configuration name: analysis-app port: 8000 +host: 127.0.0.1 # Default to localhost for security # Flows configuration flows: diff --git a/src/crewai/deployment/cli.py b/src/crewai/deployment/cli.py index 60ed9a1a5..c8ef64d9a 100644 --- a/src/crewai/deployment/cli.py +++ b/src/crewai/deployment/cli.py @@ -21,6 +21,12 @@ def create_deployment(config_path): deployment = Deployment(config_path) deployment.prepare() console.print(f"Deployment prepared at {deployment.deployment_dir}", style="bold green") + console.print(f"Configuration:", style="bold blue") + console.print(f" Name: {deployment.config.name}") + console.print(f" Port: {deployment.config.port}") + console.print(f" Host: {deployment.config.host}") + console.print(f" Crews: {[c.name for c in deployment.config.crews]}") + console.print(f" Flows: {[f.name for f in deployment.config.flows]}") except Exception as e: console.print(f"Error creating deployment: {e}", style="bold red") @@ -57,7 +63,7 @@ def start_deployment(deployment_name): deployment = Deployment(config_path) deployment.start() console.print(f"Deployment {deployment_name} started", style="bold green") - console.print(f"API server running at http://localhost:{deployment.config.port}") + console.print(f"API server running at http://{deployment.config.host}:{deployment.config.port}") except Exception as e: console.print(f"Error starting deployment: {e}", style="bold red") diff --git a/src/crewai/deployment/config.py b/src/crewai/deployment/config.py index f473f629b..cb98e8246 100644 --- a/src/crewai/deployment/config.py +++ b/src/crewai/deployment/config.py @@ -1,41 +1,93 @@ -from pathlib import Path -from typing import Any, Dict, List, Optional - +import os import yaml +from pathlib import Path +from typing import Dict, List, Optional, Union, Any +from pydantic import BaseModel, Field, validator + + +class CrewConfig(BaseModel): + """Configuration for a crew in a deployment.""" + name: str = Field(..., min_length=1) + module_path: str = Field(..., min_length=1) + class_name: str = Field(..., min_length=1) + + +class FlowConfig(BaseModel): + """Configuration for a flow in a deployment.""" + name: str = Field(..., min_length=1) + module_path: str = Field(..., min_length=1) + class_name: str = Field(..., min_length=1) + + +class DeploymentConfig(BaseModel): + """Main configuration for a CrewAI deployment.""" + name: str = Field(..., min_length=1) + port: int = Field(..., gt=0, lt=65536) + host: Optional[str] = Field(default="127.0.0.1") + crews: List[CrewConfig] = Field(default_factory=list) + flows: List[FlowConfig] = Field(default_factory=list) + environment: List[str] = Field(default_factory=list) + + @validator('environment', pre=True) + def parse_environment(cls, v): + if not v: + return [] + return v class Config: """ - Configuration for CrewAI deployments. + Configuration manager for CrewAI deployments. """ def __init__(self, config_path: str): self.config_path = Path(config_path) - self.config_data = self._load_config() + self._config_data = self._load_config() + self.config = self._validate_config() def _load_config(self) -> Dict[str, Any]: """Load configuration from YAML file.""" if not self.config_path.exists(): - raise FileNotFoundError(f"Config file {self.config_path} not found") + raise FileNotFoundError(f"Configuration file not found: {self.config_path}") with open(self.config_path, "r") as f: - return yaml.safe_load(f) + try: + return yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML in configuration file: {e}") + + def _validate_config(self) -> DeploymentConfig: + """Validate configuration using Pydantic.""" + try: + return DeploymentConfig(**self._config_data) + except Exception as e: + raise ValueError(f"Invalid configuration: {e}") @property def name(self) -> str: """Get deployment name.""" - return self.config_data.get("name", "crewai-deployment") + return self.config.name @property def port(self) -> int: """Get server port.""" - return int(self.config_data.get("port", 8000)) + return self.config.port @property - def crews(self) -> List[Dict[str, Any]]: + def host(self) -> str: + """Get host configuration.""" + return self.config.host + + @property + def crews(self) -> List[CrewConfig]: """Get crews configuration.""" - return self.config_data.get("crews", []) + return self.config.crews @property - def flows(self) -> List[Dict[str, Any]]: + def flows(self) -> List[FlowConfig]: """Get flows configuration.""" - return self.config_data.get("flows", []) + return self.config.flows + + @property + def environment(self) -> List[str]: + """Get environment variables configuration.""" + return self.config.environment diff --git a/src/crewai/deployment/docker/container.py b/src/crewai/deployment/docker/container.py index 63a71ede2..729471e36 100644 --- a/src/crewai/deployment/docker/container.py +++ b/src/crewai/deployment/docker/container.py @@ -3,6 +3,7 @@ import shutil import subprocess from pathlib import Path from typing import Dict, List, Optional +from crewai.deployment.docker.exceptions import DockerBuildError, DockerRunError, DockerComposeError class DockerContainer: @@ -15,7 +16,7 @@ class DockerContainer: self.dockerfile_path = self.deployment_dir / "Dockerfile" self.compose_path = self.deployment_dir / "docker-compose.yml" - def generate_dockerfile(self, requirements: List[str] = None): + def generate_dockerfile(self, requirements: Optional[List[str]] = None): """Generate a Dockerfile for the deployment.""" template_dir = Path(__file__).parent / "templates" dockerfile_template = template_dir / "Dockerfile" @@ -46,20 +47,32 @@ class DockerContainer: def build(self): """Build the Docker image.""" - cmd = ["docker", "build", "-t", f"crewai-{self.name}", "."] - subprocess.run(cmd, check=True, cwd=self.deployment_dir) + try: + cmd = ["docker", "build", "-t", f"crewai-{self.name}", "."] + subprocess.run(cmd, check=True, cwd=self.deployment_dir) + except subprocess.CalledProcessError as e: + raise DockerBuildError(f"Failed to build Docker image: {e}") def start(self): """Start the Docker containers using docker-compose.""" - cmd = ["docker-compose", "up", "-d"] - subprocess.run(cmd, check=True, cwd=self.deployment_dir) + try: + cmd = ["docker-compose", "up", "-d"] + subprocess.run(cmd, check=True, cwd=self.deployment_dir) + except subprocess.CalledProcessError as e: + raise DockerRunError(f"Failed to start Docker containers: {e}") def stop(self): """Stop the Docker containers.""" - cmd = ["docker-compose", "down"] - subprocess.run(cmd, check=True, cwd=self.deployment_dir) + try: + cmd = ["docker-compose", "down"] + subprocess.run(cmd, check=True, cwd=self.deployment_dir) + except subprocess.CalledProcessError as e: + raise DockerComposeError(f"Failed to stop Docker containers: {e}") def logs(self): """Get container logs.""" - cmd = ["docker-compose", "logs"] - subprocess.run(cmd, check=True, cwd=self.deployment_dir) + try: + cmd = ["docker-compose", "logs"] + subprocess.run(cmd, check=True, cwd=self.deployment_dir) + except subprocess.CalledProcessError as e: + raise DockerComposeError(f"Failed to get Docker logs: {e}") diff --git a/src/crewai/deployment/docker/exceptions.py b/src/crewai/deployment/docker/exceptions.py new file mode 100644 index 000000000..37c40589d --- /dev/null +++ b/src/crewai/deployment/docker/exceptions.py @@ -0,0 +1,18 @@ +class DockerError(Exception): + """Base exception for Docker-related errors in CrewAI deployments.""" + pass + + +class DockerBuildError(DockerError): + """Exception raised when Docker build fails.""" + pass + + +class DockerRunError(DockerError): + """Exception raised when Docker container fails to run.""" + pass + + +class DockerComposeError(DockerError): + """Exception raised when docker-compose commands fail.""" + pass diff --git a/src/crewai/deployment/main.py b/src/crewai/deployment/main.py index eb0949759..c07e7b073 100644 --- a/src/crewai/deployment/main.py +++ b/src/crewai/deployment/main.py @@ -1,11 +1,20 @@ import json import os import shutil +import logging from pathlib import Path from typing import Any, Dict, List, Optional from crewai.deployment.config import Config from crewai.deployment.docker.container import DockerContainer +from crewai.deployment.docker.exceptions import DockerError + +# Configure structured logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger("crewai.deployment") class Deployment: @@ -13,6 +22,7 @@ class Deployment: Handles the deployment of CrewAI crews and flows. """ def __init__(self, config_path: str): + logger.info(f"Initializing deployment from config: {config_path}") self.config = Config(config_path) self.deployment_dir = Path(f"./deployments/{self.config.name}") self.docker = DockerContainer( @@ -22,6 +32,8 @@ class Deployment: def prepare(self): """Prepare the deployment directory and files.""" + logger.info(f"Preparing deployment: {self.config.name}") + # Create deployment directory os.makedirs(self.deployment_dir, exist_ok=True) @@ -29,22 +41,27 @@ class Deployment: deployment_config = { "name": self.config.name, "port": self.config.port, + "host": self.config.host, "crews": [], "flows": [] } # Process crews for crew_config in self.config.crews: - name = crew_config["name"] - module_path = crew_config["module_path"] - class_name = crew_config["class_name"] + name = crew_config.name + module_path = crew_config.module_path + class_name = crew_config.class_name + + logger.info(f"Processing crew: {name}") # Copy crew module to deployment directory source_path = Path(module_path) dest_path = self.deployment_dir / source_path.name if source_path.exists(): shutil.copy(source_path, dest_path) + logger.debug(f"Copied {source_path} to {dest_path}") else: + logger.warning(f"Crew module not found: {source_path}") # For testing purposes, create an empty file with open(dest_path, 'w') as f: pass @@ -58,16 +75,20 @@ class Deployment: # Process flows for flow_config in self.config.flows: - name = flow_config["name"] - module_path = flow_config["module_path"] - class_name = flow_config["class_name"] + name = flow_config.name + module_path = flow_config.module_path + class_name = flow_config.class_name + + logger.info(f"Processing flow: {name}") # Copy flow module to deployment directory source_path = Path(module_path) dest_path = self.deployment_dir / source_path.name if source_path.exists(): shutil.copy(source_path, dest_path) + logger.debug(f"Copied {source_path} to {dest_path}") else: + logger.warning(f"Flow module not found: {source_path}") # For testing purposes, create an empty file with open(dest_path, 'w') as f: pass @@ -80,29 +101,61 @@ class Deployment: }) # Write deployment config - with open(self.deployment_dir / "deployment_config.json", "w") as f: + config_file = self.deployment_dir / "deployment_config.json" + with open(config_file, "w") as f: json.dump(deployment_config, f, indent=2) + logger.info(f"Created deployment config: {config_file}") # Copy server template server_template = Path(__file__).parent / "templates" / "server.py" - shutil.copy(server_template, self.deployment_dir / "server.py") + server_dest = self.deployment_dir / "server.py" + shutil.copy(server_template, server_dest) + logger.info(f"Copied server template to {server_dest}") # Generate Docker files - self.docker.generate_dockerfile() - self.docker.generate_compose_file(port=self.config.port) + try: + self.docker.generate_dockerfile() + self.docker.generate_compose_file(port=self.config.port) + logger.info("Generated Docker configuration files") + except Exception as e: + logger.error(f"Failed to generate Docker files: {e}") + raise def build(self): """Build the Docker image for the deployment.""" - self.docker.build() + logger.info(f"Building Docker image for {self.config.name}") + try: + self.docker.build() + logger.info("Docker image built successfully") + except DockerError as e: + logger.error(f"Failed to build Docker image: {e}") + raise def start(self): """Start the deployment.""" - self.docker.start() + logger.info(f"Starting deployment {self.config.name}") + try: + self.docker.start() + logger.info(f"Deployment started at http://{self.config.host}:{self.config.port}") + except DockerError as e: + logger.error(f"Failed to start deployment: {e}") + raise def stop(self): """Stop the deployment.""" - self.docker.stop() + logger.info(f"Stopping deployment {self.config.name}") + try: + self.docker.stop() + logger.info("Deployment stopped") + except DockerError as e: + logger.error(f"Failed to stop deployment: {e}") + raise def logs(self): """Get deployment logs.""" - self.docker.logs() + logger.info(f"Fetching logs for {self.config.name}") + try: + self.docker.logs() + except DockerError as e: + logger.error(f"Failed to fetch logs: {e}") + raise diff --git a/src/crewai/deployment/templates/deployment.yaml b/src/crewai/deployment/templates/deployment.yaml index b6422e312..17db9791d 100644 --- a/src/crewai/deployment/templates/deployment.yaml +++ b/src/crewai/deployment/templates/deployment.yaml @@ -1,6 +1,7 @@ # CrewAI Deployment Configuration name: my-crewai-app port: 8000 +host: 127.0.0.1 # Default to localhost for security # Crews configuration crews: diff --git a/src/crewai/deployment/templates/server.py b/src/crewai/deployment/templates/server.py index 483a8ce1d..f40704f8b 100644 --- a/src/crewai/deployment/templates/server.py +++ b/src/crewai/deployment/templates/server.py @@ -58,10 +58,13 @@ for flow_config in config.get("flows", []): def read_root(): return {"status": "running", "crews": list(crews.keys()), "flows": list(flows.keys())} -@app.post("/run/crew/{crew_name}") +@app.post("/run/crew/{crew_name}", response_model=RunResponse) def run_crew(crew_name: str, request: RunRequest): if crew_name not in crews: - raise HTTPException(status_code=404, detail=f"Crew '{crew_name}' not found") + raise HTTPException( + status_code=404, + detail=f"Crew '{crew_name}' not found. Available crews: {list(crews.keys())}" + ) try: crew_instance = crews[crew_name].crew() @@ -70,18 +73,22 @@ def run_crew(crew_name: str, request: RunRequest): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -@app.post("/run/flow/{flow_name}") +@app.post("/run/flow/{flow_name}", response_model=RunResponse) def run_flow(flow_name: str, request: RunRequest): if flow_name not in flows: - raise HTTPException(status_code=404, detail=f"Flow '{flow_name}' not found") + raise HTTPException( + status_code=404, + detail=f"Flow '{flow_name}' not found. Available flows: {list(flows.keys())}" + ) try: flow_instance = flows[flow_name] result = flow_instance.kickoff(inputs=request.inputs) return {"result": {"value": str(result)}} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=f"Error running flow: {str(e)}") if __name__ == "__main__": port = int(os.environ.get("PORT", 8000)) - uvicorn.run(app, host="0.0.0.0", port=port) + host = os.environ.get("HOST", "127.0.0.1") # Default to localhost instead of 0.0.0.0 + uvicorn.run(app, host=host, port=port) diff --git a/tests/deployment/test_deployment.py b/tests/deployment/test_deployment.py index 41a335c68..142e9ac58 100644 --- a/tests/deployment/test_deployment.py +++ b/tests/deployment/test_deployment.py @@ -18,6 +18,7 @@ class TestDeployment(unittest.TestCase): f.write(""" name: test-deployment port: 8000 + host: 127.0.0.1 crews: - name: test_crew @@ -42,8 +43,9 @@ class TestDeployment(unittest.TestCase): config = Config(self.config_path) self.assertEqual(config.name, "test-deployment") self.assertEqual(config.port, 8000) + self.assertEqual(config.host, "127.0.0.1") self.assertEqual(len(config.crews), 1) - self.assertEqual(config.crews[0]["name"], "test_crew") + self.assertEqual(config.crews[0].name, "test_crew") @mock.patch("crewai.deployment.docker.container.DockerContainer.generate_dockerfile") @mock.patch("crewai.deployment.docker.container.DockerContainer.generate_compose_file")