diff --git a/docs/how-to/mlflow-observability.mdx b/docs/how-to/mlflow-observability.mdx new file mode 100644 index 000000000..f0c4a9ec4 --- /dev/null +++ b/docs/how-to/mlflow-observability.mdx @@ -0,0 +1,206 @@ +--- +title: Agent Monitoring with MLflow +description: Quickly start monitoring your Agents with MLflow. +icon: bars-staggered +--- + +# MLflow Overview + +[MLflow](https://mlflow.org/) is an open-source platform to assist machine learning practitioners and teams in handling the complexities of the machine learning process. + +It provides a tracing feature that enhances LLM observability in your Generative AI applications by capturing detailed information about the execution of your application’s services. +Tracing provides a way to record the inputs, outputs, and metadata associated with each intermediate step of a request, enabling you to easily pinpoint the source of bugs and unexpected behaviors. + +![Overview of MLflow crewAI tracing usage](/images/mlflow-tracing.gif) + +### Features + +- **Tracing Dashboard**: Monitor activities of your crewAI agents with detailed dashboards that include inputs, outputs and metadata of spans. +- **Automated Tracing**: A fully automated integration with crewAI, which can be enabled by running `mlflow.crewai.autolog()`. +- **Manual Trace Instrumentation with minor efforts**: Customize trace instrumentation through MLflow's high-level fluent APIs such as decorators, function wrappers and context managers. +- **OpenTelemetry Compatibility**: MLflow Tracing supports exporting traces to an OpenTelemetry Collector, which can then be used to export traces to various backends such as Jaeger, Zipkin, and AWS X-Ray. +- **Package and Deploy Agents**: Package and deploy your crewAI agents to an inference server with a variety of deployment targets. +- **Securely Host LLMs**: Host multiple LLM from various providers in one unified endpoint through MFflow gateway. +- **Evaluation**: Evaluate your crewAI agents with a wide range of metrics using a convenient API `mlflow.evaluate()`. + +## Setup Instructions + + + + ```shell + # The crewAI integration is available in mlflow>=2.19.0 + pip install mlflow + ``` + + + ```shell + # This process is optional, but it is recommended to use MLflow tracking server for better visualization and broader features. + mlflow server + ``` + + + Add the following two lines to your application code: + + ```python + import mlflow + + mlflow.crewai.autolog() + + # Optional: Set a tracking URI and an experiment name if you have a tracking server + mlflow.set_tracking_uri("http://localhost:5000") + mlflow.set_experiment("CrewAI") + ``` + + Example Usage for tracing CrewAI Agents: + + ```python + from crewai import Agent, Crew, Task + from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSource + from crewai_tools import SerperDevTool, WebsiteSearchTool + + from textwrap import dedent + + content = "Users name is John. He is 30 years old and lives in San Francisco." + string_source = StringKnowledgeSource( + content=content, metadata={"preference": "personal"} + ) + + search_tool = WebsiteSearchTool() + + + class TripAgents: + def city_selection_agent(self): + return Agent( + role="City Selection Expert", + goal="Select the best city based on weather, season, and prices", + backstory="An expert in analyzing travel data to pick ideal destinations", + tools=[ + search_tool, + ], + verbose=True, + ) + + def local_expert(self): + return Agent( + role="Local Expert at this city", + goal="Provide the BEST insights about the selected city", + backstory="""A knowledgeable local guide with extensive information + about the city, it's attractions and customs""", + tools=[search_tool], + verbose=True, + ) + + + class TripTasks: + def identify_task(self, agent, origin, cities, interests, range): + return Task( + description=dedent( + f""" + Analyze and select the best city for the trip based + on specific criteria such as weather patterns, seasonal + events, and travel costs. This task involves comparing + multiple cities, considering factors like current weather + conditions, upcoming cultural or seasonal events, and + overall travel expenses. + Your final answer must be a detailed + report on the chosen city, and everything you found out + about it, including the actual flight costs, weather + forecast and attractions. + + Traveling from: {origin} + City Options: {cities} + Trip Date: {range} + Traveler Interests: {interests} + """ + ), + agent=agent, + expected_output="Detailed report on the chosen city including flight costs, weather forecast, and attractions", + ) + + def gather_task(self, agent, origin, interests, range): + return Task( + description=dedent( + f""" + As a local expert on this city you must compile an + in-depth guide for someone traveling there and wanting + to have THE BEST trip ever! + Gather information about key attractions, local customs, + special events, and daily activity recommendations. + Find the best spots to go to, the kind of place only a + local would know. + This guide should provide a thorough overview of what + the city has to offer, including hidden gems, cultural + hotspots, must-visit landmarks, weather forecasts, and + high level costs. + The final answer must be a comprehensive city guide, + rich in cultural insights and practical tips, + tailored to enhance the travel experience. + + Trip Date: {range} + Traveling from: {origin} + Traveler Interests: {interests} + """ + ), + agent=agent, + expected_output="Comprehensive city guide including hidden gems, cultural hotspots, and practical travel tips", + ) + + + class TripCrew: + def __init__(self, origin, cities, date_range, interests): + self.cities = cities + self.origin = origin + self.interests = interests + self.date_range = date_range + + def run(self): + agents = TripAgents() + tasks = TripTasks() + + city_selector_agent = agents.city_selection_agent() + local_expert_agent = agents.local_expert() + + identify_task = tasks.identify_task( + city_selector_agent, + self.origin, + self.cities, + self.interests, + self.date_range, + ) + gather_task = tasks.gather_task( + local_expert_agent, self.origin, self.interests, self.date_range + ) + + crew = Crew( + agents=[city_selector_agent, local_expert_agent], + tasks=[identify_task, gather_task], + verbose=True, + memory=True, + knowledge={ + "sources": [string_source], + "metadata": {"preference": "personal"}, + }, + ) + + result = crew.kickoff() + return result + + + trip_crew = TripCrew("California", "Tokyo", "Dec 12 - Dec 20", "sports") + result = trip_crew.run() + + print(result) + ``` + Refer to [MLflow Tracing Documentation](https://mlflow.org/docs/latest/llms/tracing/index.html) for more configurations and use cases. + + + Now traces for your crewAI agents are captured by MLflow. + Let's visit MLflow tracking server to view the traces and get insights into your Agents. + + Open `127.0.0.1:5000` on your browser to visit MLflow tracking server. + + MLflow tracing example with crewai + + + + diff --git a/docs/images/mlflow-tracing.gif b/docs/images/mlflow-tracing.gif new file mode 100644 index 000000000..e968c0924 Binary files /dev/null and b/docs/images/mlflow-tracing.gif differ diff --git a/docs/images/mlflow1.png b/docs/images/mlflow1.png new file mode 100644 index 000000000..b2c6f3b80 Binary files /dev/null and b/docs/images/mlflow1.png differ diff --git a/docs/mint.json b/docs/mint.json index 585fc0abd..a49f60946 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -101,6 +101,7 @@ "how-to/conditional-tasks", "how-to/agentops-observability", "how-to/langtrace-observability", + "how-to/mlflow-observability", "how-to/openlit-observability", "how-to/portkey-observability" ] diff --git a/mkdocs.yml b/mkdocs.yml index 948744de2..511078fff 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -152,6 +152,7 @@ nav: - Agent Monitoring with AgentOps: 'how-to/AgentOps-Observability.md' - Agent Monitoring with LangTrace: 'how-to/Langtrace-Observability.md' - Agent Monitoring with OpenLIT: 'how-to/openlit-Observability.md' + - Agent Monitoring with MLflow: 'how-to/mlflow-Observability.md' - Tools Docs: - Browserbase Web Loader: 'tools/BrowserbaseLoadTool.md' - Code Docs RAG Search: 'tools/CodeDocsSearchTool.md'