WIP. Working on adding and & or to flows. In the middle of setting up template for flow as well

This commit is contained in:
Brandon Hancock
2024-09-11 16:26:02 -04:00
parent 3a266d6b40
commit a028566bd6
22 changed files with 627 additions and 32 deletions

View File

@@ -4,6 +4,7 @@ import click
import pkg_resources
from crewai.cli.create_crew import create_crew
from crewai.cli.create_flow import create_flow
from crewai.cli.create_pipeline import create_pipeline
from crewai.memory.storage.kickoff_task_outputs_storage import (
KickoffTaskOutputsSQLiteStorage,
@@ -25,19 +26,20 @@ def crewai():
@crewai.command()
@click.argument("type", type=click.Choice(["crew", "pipeline"]))
@click.argument("type", type=click.Choice(["crew", "pipeline", "flow"]))
@click.argument("name")
@click.option(
"--router", is_flag=True, help="Create a pipeline with router functionality"
)
def create(type, name, router):
"""Create a new crew or pipeline."""
def create(type, name):
"""Create a new crew, pipeline, or flow."""
if type == "crew":
create_crew(name)
elif type == "pipeline":
create_pipeline(name, router)
create_pipeline(name)
elif type == "flow":
create_flow(name)
else:
click.secho("Error: Invalid type. Must be 'crew' or 'pipeline'.", fg="red")
click.secho(
"Error: Invalid type. Must be 'crew', 'pipeline', or 'flow'.", fg="red"
)
@crewai.command()

View File

@@ -0,0 +1,88 @@
from pathlib import Path
import click
def create_flow(name):
"""Create a new flow."""
folder_name = name.replace(" ", "_").replace("-", "_").lower()
class_name = name.replace("_", " ").replace("-", " ").title().replace(" ", "")
click.secho(f"Creating flow {folder_name}...", fg="green", bold=True)
project_root = Path(folder_name)
if project_root.exists():
click.secho(f"Error: Folder {folder_name} already exists.", fg="red")
return
# Create directory structure
(project_root / "src" / folder_name).mkdir(parents=True)
(project_root / "src" / folder_name / "crews").mkdir(parents=True)
(project_root / "src" / folder_name / "tools").mkdir(parents=True)
(project_root / "tests").mkdir(exist_ok=True)
# Create .env file
with open(project_root / ".env", "w") as file:
file.write("OPENAI_API_KEY=YOUR_API_KEY")
package_dir = Path(__file__).parent
templates_dir = package_dir / "templates" / "flow"
# List of template files to copy
root_template_files = [".gitignore", "pyproject.toml", "README.md"]
src_template_files = ["__init__.py", "main.py"]
tools_template_files = ["tools/__init__.py", "tools/custom_tool.py"]
crew_folders = [
"research_crew",
"write_linkedin_crew",
"write_x_crew",
]
def process_file(src_file, dst_file):
with open(src_file, "r") as file:
content = file.read()
content = content.replace("{{name}}", name)
content = content.replace("{{flow_name}}", class_name)
content = content.replace("{{folder_name}}", folder_name)
with open(dst_file, "w") as file:
file.write(content)
# Copy and process root template files
for file_name in root_template_files:
src_file = templates_dir / file_name
dst_file = project_root / file_name
process_file(src_file, dst_file)
# Copy and process src template files
for file_name in src_template_files:
src_file = templates_dir / file_name
dst_file = project_root / "src" / folder_name / file_name
process_file(src_file, dst_file)
# Copy tools files
for file_name in tools_template_files:
src_file = templates_dir / file_name
dst_file = project_root / "src" / folder_name / file_name
process_file(src_file, dst_file)
# Copy crew folders
for crew_folder in crew_folders:
src_crew_folder = templates_dir / "crews" / crew_folder
dst_crew_folder = project_root / "src" / folder_name / "crews" / crew_folder
if src_crew_folder.exists():
for src_file in src_crew_folder.rglob("*"):
if src_file.is_file():
relative_path = src_file.relative_to(src_crew_folder)
dst_file = dst_crew_folder / relative_path
dst_file.parent.mkdir(parents=True, exist_ok=True)
process_file(src_file, dst_file)
else:
click.secho(
f"Warning: Crew folder {crew_folder} not found in template.",
fg="yellow",
)
click.secho(f"Flow {name} created successfully!", fg="green", bold=True)

View File

@@ -0,0 +1,2 @@
.env
__pycache__/

View File

@@ -0,0 +1,57 @@
# {{crew_name}} Crew
Welcome to the {{crew_name}} Crew project, powered by [crewAI](https://crewai.com). This template is designed to help you set up a multi-agent AI system with ease, leveraging the powerful and flexible framework provided by crewAI. Our goal is to enable your agents to collaborate effectively on complex tasks, maximizing their collective intelligence and capabilities.
## Installation
Ensure you have Python >=3.10 <=3.13 installed on your system. This project uses [Poetry](https://python-poetry.org/) for dependency management and package handling, offering a seamless setup and execution experience.
First, if you haven't already, install Poetry:
```bash
pip install poetry
```
Next, navigate to your project directory and install the dependencies:
1. First lock the dependencies and then install them:
```bash
crewai install
```
### Customizing
**Add your `OPENAI_API_KEY` into the `.env` file**
- Modify `src/{{folder_name}}/config/agents.yaml` to define your agents
- Modify `src/{{folder_name}}/config/tasks.yaml` to define your tasks
- Modify `src/{{folder_name}}/crew.py` to add your own logic, tools and specific args
- Modify `src/{{folder_name}}/main.py` to add custom inputs for your agents and tasks
## Running the Project
To kickstart your crew of AI agents and begin task execution, run this from the root folder of your project:
```bash
crewai run
```
This command initializes the {{name}} Crew, assembling the agents and assigning them tasks as defined in your configuration.
This example, unmodified, will run the create a `report.md` file with the output of a research on LLMs in the root folder.
## Understanding Your Crew
The {{name}} Crew is composed of multiple AI agents, each with unique roles, goals, and tools. These agents collaborate on a series of tasks, defined in `config/tasks.yaml`, leveraging their collective skills to achieve complex objectives. The `config/agents.yaml` file outlines the capabilities and configurations of each agent in your crew.
## Support
For support, questions, or feedback regarding the {{crew_name}} Crew or crewAI.
- Visit our [documentation](https://docs.crewai.com)
- Reach out to us through our [GitHub repository](https://github.com/joaomdmoura/crewai)
- [Join our Discord](https://discord.com/invite/X4JWnZnxPb)
- [Chat with our docs](https://chatg.pt/DWjSBZn)
Let's create wonders together with the power and simplicity of crewAI.

View File

@@ -0,0 +1,19 @@
researcher:
role: >
{topic} Senior Data Researcher
goal: >
Uncover cutting-edge developments in {topic}
backstory: >
You're a seasoned researcher with a knack for uncovering the latest
developments in {topic}. Known for your ability to find the most relevant
information and present it in a clear and concise manner.
reporting_analyst:
role: >
{topic} Reporting Analyst
goal: >
Create detailed reports based on {topic} data analysis and research findings
backstory: >
You're a meticulous analyst with a keen eye for detail. You're known for
your ability to turn complex data into clear and concise reports, making
it easy for others to understand and act on the information you provide.

View File

@@ -0,0 +1,16 @@
research_task:
description: >
Conduct a thorough research about {topic}
Make sure you find any interesting and relevant information given
the current year is 2024.
expected_output: >
A list with 10 bullet points of the most relevant information about {topic}
agent: researcher
reporting_task:
description: >
Review the context you got and expand each topic into a full section for a report.
Make sure the report is detailed and contains any and all relevant information.
expected_output: >
A fully fledge reports with a title, mains topics, each with a full section of information.
agent: reporting_analyst

View File

@@ -0,0 +1,58 @@
from pydantic import BaseModel
from crewai import Agent, Crew, Process, Task
from crewai.project import CrewBase, agent, crew, task
# Uncomment the following line to use an example of a custom tool
# from demo_pipeline.tools.custom_tool import MyCustomTool
# Check our tools documentations for more information on how to use them
# from crewai_tools import SerperDevTool
class ResearchReport(BaseModel):
"""Research Report"""
title: str
body: str
@CrewBase
class ResearchCrew():
"""Research Crew"""
agents_config = 'config/agents.yaml'
tasks_config = 'config/tasks.yaml'
@agent
def researcher(self) -> Agent:
return Agent(
config=self.agents_config['researcher'],
verbose=True
)
@agent
def reporting_analyst(self) -> Agent:
return Agent(
config=self.agents_config['reporting_analyst'],
verbose=True
)
@task
def research_task(self) -> Task:
return Task(
config=self.tasks_config['research_task'],
)
@task
def reporting_task(self) -> Task:
return Task(
config=self.tasks_config['reporting_task'],
output_pydantic=ResearchReport
)
@crew
def crew(self) -> Crew:
"""Creates the Research Crew"""
return Crew(
agents=self.agents, # Automatically created by the @agent decorator
tasks=self.tasks, # Automatically created by the @task decorator
process=Process.sequential,
verbose=True,
)

View File

@@ -0,0 +1,51 @@
from crewai import Agent, Crew, Process, Task
from crewai.project import CrewBase, agent, crew, task
# Uncomment the following line to use an example of a custom tool
# from {{folder_name}}.tools.custom_tool import MyCustomTool
# Check our tools documentations for more information on how to use them
# from crewai_tools import SerperDevTool
@CrewBase
class WriteLinkedInCrew():
"""Research Crew"""
agents_config = 'config/agents.yaml'
tasks_config = 'config/tasks.yaml'
@agent
def researcher(self) -> Agent:
return Agent(
config=self.agents_config['researcher'],
verbose=True
)
@agent
def reporting_analyst(self) -> Agent:
return Agent(
config=self.agents_config['reporting_analyst'],
verbose=True
)
@task
def research_task(self) -> Task:
return Task(
config=self.tasks_config['research_task'],
)
@task
def reporting_task(self) -> Task:
return Task(
config=self.tasks_config['reporting_task'],
output_file='report.md'
)
@crew
def crew(self) -> Crew:
"""Creates the {{crew_name}} crew"""
return Crew(
agents=self.agents, # Automatically created by the @agent decorator
tasks=self.tasks, # Automatically created by the @task decorator
process=Process.sequential,
verbose=True,
)

View File

@@ -0,0 +1,14 @@
x_writer_agent:
role: >
Expert Social Media Content Creator specializing in short form written content
goal: >
Create viral-worthy, engaging short form posts that distill complex {topic} information
into compelling 280-character messages
backstory: >
You're a social media virtuoso with a particular talent for short form content. Your posts
consistently go viral due to your ability to craft hooks that stop users mid-scroll.
You've studied the techniques of social media masters like Justin Welsh, Dickie Bush,
Nicolas Cole, and Shaan Puri, incorporating their best practices into your own unique style.
Your superpower is taking intricate {topic} concepts and transforming them into
bite-sized, shareable content that resonates with a wide audience. You know exactly
how to structure a post for maximum impact and engagement.

View File

@@ -0,0 +1,22 @@
write_x_task:
description: >
Using the research report provided, create an engaging short form post about {topic}.
Your post should have a great hook, summarize key points, and be structured for easy
consumption on a digital platform. The post must be under 280 characters.
Follow these guidelines:
1. Start with an attention-grabbing hook
2. Condense the main insights from the research
3. Use clear, concise language
4. Include a call-to-action or thought-provoking question if space allows
5. Ensure the post flows well and is easy to read quickly
Here is the title of the research report you will be using
Title: {title}
Research:
{body}
expected_output: >
A compelling X post under 280 characters that effectively summarizes the key findings
about {topic}, starts with a strong hook, and is optimized for engagement on the platform.
agent: x_writer_agent

View File

@@ -0,0 +1,36 @@
from crewai import Agent, Crew, Process, Task
from crewai.project import CrewBase, agent, crew, task
# Uncomment the following line to use an example of a custom tool
# from demo_pipeline.tools.custom_tool import MyCustomTool
# Check our tools documentations for more information on how to use them
# from crewai_tools import SerperDevTool
@CrewBase
class WriteXCrew:
"""Research Crew"""
agents_config = "config/agents.yaml"
tasks_config = "config/tasks.yaml"
@agent
def x_writer_agent(self) -> Agent:
return Agent(config=self.agents_config["x_writer_agent"], verbose=True)
@task
def write_x_task(self) -> Task:
return Task(
config=self.tasks_config["write_x_task"],
)
@crew
def crew(self) -> Crew:
"""Creates the Write X Crew"""
return Crew(
agents=self.agents, # Automatically created by the @agent decorator
tasks=self.tasks, # Automatically created by the @task decorator
process=Process.sequential,
verbose=True,
)

View File

@@ -0,0 +1,39 @@
#!/usr/bin/env python
import asyncio
from pydantic import BaseModel
from crewai.flow.flow import Flow, listen, start
# TODO: THERE SHOULD BE 3 FLOWS IN HERE: SIMPLE, ASYNC, BRANCHING (with router)
class ExampleState(BaseModel):
counter: int = 0
message: str = ""
class ExampleFlow(Flow[ExampleState]):
initial_state = ExampleState
@start()
def start_method(self):
print("Starting the structured flow")
self.state.message = "Hello from structured flow"
@listen(start_method)
def second_method(self, result):
print(f"Second method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
self.state.message += " - updated"
print(f"State after second_method: {self.state}")
return "Second result"
async def run():
"""
Run the flow.
"""
example_flow = ExampleFlow()
await example_flow.run()
if __name__ == "__main__":
asyncio.run(run())

View File

@@ -0,0 +1,17 @@
[tool.poetry]
name = "{{folder_name}}"
version = "0.1.0"
description = "{{name}} using crewAI"
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = ">=3.10,<=3.13"
crewai = { extras = ["tools"], version = ">=0.55.2,<1.0.0" }
asyncio = "*"
[tool.poetry.scripts]
{{folder_name}} = "{{folder_name}}.main:main"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

View File

@@ -0,0 +1,12 @@
from crewai_tools import BaseTool
class MyCustomTool(BaseTool):
name: str = "Name of my tool"
description: str = (
"Clear description for what this tool is useful for, you agent will need this information to use it."
)
def _run(self, argument: str) -> str:
# Implementation goes here
return "this is an example of a tool output, ignore it and move along."

View File

@@ -1,3 +1,5 @@
import asyncio
import inspect
from typing import Any, Callable, Dict, Generic, List, Type, TypeVar, Union
from pydantic import BaseModel
@@ -16,11 +18,34 @@ class FlowMeta(type):
if hasattr(attr_value, "__is_start_method__"):
start_methods.append(attr_name)
if hasattr(attr_value, "__trigger_methods__"):
for trigger in attr_value.__trigger_methods__:
trigger_name = trigger.__name__ if callable(trigger) else trigger
if trigger_name not in listeners:
listeners[trigger_name] = []
listeners[trigger_name].append(attr_name)
condition = attr_value.__trigger_methods__
if callable(condition):
# Single method reference
method_name = condition.__name__
if method_name not in listeners:
listeners[method_name] = []
listeners[method_name].append((attr_name, "SINGLE", [method_name]))
elif isinstance(condition, str):
# Single method name
if condition not in listeners:
listeners[condition] = []
listeners[condition].append((attr_name, "SINGLE", [condition]))
elif isinstance(condition, tuple):
# AND or OR condition
condition_type = (
"AND" if any(item == "and" for item in condition) else "OR"
)
methods = [
m.__name__ if callable(m) else m
for m in condition
if m != "and" and m != "or"
]
for method in methods:
if method not in listeners:
listeners[method] = []
listeners[method].append((attr_name, condition_type, methods))
else:
raise ValueError(f"Invalid listener format for {attr_name}")
setattr(cls, "_start_methods", start_methods)
setattr(cls, "_listeners", listeners)
@@ -38,12 +63,13 @@ class FlowMeta(type):
class Flow(Generic[T], metaclass=FlowMeta):
_start_methods: List[str] = []
_listeners: Dict[str, List[str]] = {}
_listeners: Dict[str, List[tuple[str, str, List[str]]]] = {}
initial_state: Union[Type[T], T, None] = None
def __init__(self):
self._methods: Dict[str, Callable] = {}
self._state = self._create_initial_state()
self._completed_methods: set[str] = set()
for method_name in dir(self):
if callable(getattr(self, method_name)) and not method_name.startswith(
@@ -63,23 +89,55 @@ class Flow(Generic[T], metaclass=FlowMeta):
def state(self) -> T:
return self._state
def run(self):
async def run(self):
if not self._start_methods:
raise ValueError("No start method defined")
for start_method in self._start_methods:
result = self._methods[start_method]()
self._execute_listeners(start_method, result)
result = await self._execute_method(self._methods[start_method])
await self._execute_listeners(start_method, result)
async def _execute_method(self, method: Callable, *args, **kwargs):
if inspect.iscoroutinefunction(method):
return await method(*args, **kwargs)
else:
return method(*args, **kwargs)
async def _execute_listeners(self, trigger_method: str, result: Any):
self._completed_methods.add(trigger_method)
def _execute_listeners(self, trigger_method: str, result: Any):
if trigger_method in self._listeners:
for listener in self._listeners[trigger_method]:
try:
listener_result = self._methods[listener](result)
self._execute_listeners(listener, listener_result)
except Exception as e:
print(f"Error in method {listener}: {str(e)}")
return
listener_tasks = []
for listener, condition_type, methods in self._listeners[trigger_method]:
if condition_type == "OR":
if trigger_method in methods:
listener_tasks.append(
self._execute_single_listener(listener, result)
)
elif condition_type == "AND":
if all(method in self._completed_methods for method in methods):
listener_tasks.append(
self._execute_single_listener(listener, result)
)
elif condition_type == "SINGLE":
listener_tasks.append(
self._execute_single_listener(listener, result)
)
# Run all listener tasks concurrently and wait for them to complete
await asyncio.gather(*listener_tasks)
async def _execute_single_listener(self, listener: str, result: Any):
try:
method = self._methods[listener]
sig = inspect.signature(method)
if len(sig.parameters) > 1: # More than just 'self'
listener_result = await self._execute_method(method, result)
else:
listener_result = await self._execute_method(method)
await self._execute_listeners(listener, listener_result)
except Exception as e:
print(f"Error in method {listener}: {str(e)}")
def start():
@@ -90,9 +148,9 @@ def start():
return decorator
def listen(*trigger_methods):
def listen(condition):
def decorator(func):
func.__trigger_methods__ = trigger_methods
func.__trigger_methods__ = condition
return func
return decorator

View File

@@ -1,3 +1,5 @@
import asyncio
from crewai.flow.flow import Flow, listen, start
from pydantic import BaseModel
@@ -11,7 +13,7 @@ class StructuredExampleFlow(Flow[ExampleState]):
initial_state = ExampleState
@start()
def start_method(self):
async def start_method(self):
print("Starting the structured flow")
print(f"State in start_method: {self.state}")
self.state.message = "Hello from structured flow"
@@ -19,7 +21,7 @@ class StructuredExampleFlow(Flow[ExampleState]):
return "Start result"
@listen(start_method)
def second_method(self, result):
async def second_method(self, result):
print(f"Second method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
@@ -27,7 +29,19 @@ class StructuredExampleFlow(Flow[ExampleState]):
print(f"State after second_method: {self.state}")
return "Second result"
@listen(start_method)
async def third_method(self, result):
print(f"Third method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
self.state.message += " - updated"
print(f"State after third_method: {self.state}")
return "Third result"
# Instantiate and run the flow
structured_flow = StructuredExampleFlow()
structured_flow.run()
async def main():
flow = StructuredExampleFlow()
await flow.run()
asyncio.run(main())

View File

@@ -0,0 +1,47 @@
import asyncio
from crewai.flow.flow import Flow, listen, start
from pydantic import BaseModel
class ExampleState(BaseModel):
counter: int = 0
message: str = ""
class StructuredExampleFlow(Flow[ExampleState]):
initial_state = ExampleState
@start()
async def start_method(self):
print("Starting the structured flow")
print(f"State in start_method: {self.state}")
self.state.message = "Hello from structured flow"
print(f"State after start_method: {self.state}")
return "Start result"
@listen(start_method)
async def second_method(self, result):
print(f"Second method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
self.state.message += " - updated"
print(f"State after second_method: {self.state}")
return "Second result"
@listen(start_method)
async def third_method(self, result):
print(f"Third method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
self.state.message += " - updated"
print(f"State after third_method: {self.state}")
return "Third result"
async def main():
flow = StructuredExampleFlow()
await flow.run()
asyncio.run(main())

View File

@@ -0,0 +1,43 @@
import asyncio
from crewai.flow.flow import Flow, listen, start
from pydantic import BaseModel
class ExampleState(BaseModel):
counter: int = 0
message: str = ""
class StructuredExampleFlow(Flow[ExampleState]):
initial_state = ExampleState
@start()
async def start_method(self):
print("Starting the structured flow")
print(f"State in start_method: {self.state}")
self.state.message = "Hello from structured flow"
print(f"State after start_method: {self.state}")
return "Start result"
@listen(start_method)
async def second_method(self, result):
print(f"Second method, received: {result}")
print(f"State before increment: {self.state}")
self.state.counter += 1
self.state.message += " - updated"
print(f"State after second_method: {self.state}")
return "Second result"
@listen(start_method or second_method)
async def logger(self):
print("OR METHOD RUNNING")
print("CURRENT STATE FROM OR: ", self.state)
async def main():
flow = StructuredExampleFlow()
await flow.run()
asyncio.run(main())