mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-01-13 18:18:29 +00:00
chore: delete all cassettes and update conftest
This commit is contained in:
Greyson LaLonde
18
.github/workflows/tests.yml
vendored
18
.github/workflows/tests.yml
vendored
@@ -5,18 +5,6 @@ on: [pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
OPENAI_API_KEY: fake-api-key
|
||||
PYTHONUNBUFFERED: 1
|
||||
BRAVE_API_KEY: fake-brave-key
|
||||
SNOWFLAKE_USER: fake-snowflake-user
|
||||
SNOWFLAKE_PASSWORD: fake-snowflake-password
|
||||
SNOWFLAKE_ACCOUNT: fake-snowflake-account
|
||||
SNOWFLAKE_WAREHOUSE: fake-snowflake-warehouse
|
||||
SNOWFLAKE_DATABASE: fake-snowflake-database
|
||||
SNOWFLAKE_SCHEMA: fake-snowflake-schema
|
||||
EMBEDCHAIN_DB_URI: sqlite:///test.db
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: tests (${{ matrix.python-version }})
|
||||
@@ -84,26 +72,20 @@ jobs:
|
||||
# fi
|
||||
|
||||
cd lib/crewai && uv run pytest \
|
||||
--block-network \
|
||||
--timeout=30 \
|
||||
-vv \
|
||||
--splits 8 \
|
||||
--group ${{ matrix.group }} \
|
||||
$DURATIONS_ARG \
|
||||
--durations=10 \
|
||||
-n auto \
|
||||
--maxfail=3
|
||||
|
||||
- name: Run tool tests (group ${{ matrix.group }} of 8)
|
||||
run: |
|
||||
cd lib/crewai-tools && uv run pytest \
|
||||
--block-network \
|
||||
--timeout=30 \
|
||||
-vv \
|
||||
--splits 8 \
|
||||
--group ${{ matrix.group }} \
|
||||
--durations=10 \
|
||||
-n auto \
|
||||
--maxfail=3
|
||||
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from crewai.tools.base_tool import BaseTool, EnvVar
|
||||
from crewai_tools import tools
|
||||
from pydantic import BaseModel
|
||||
from pydantic.json_schema import GenerateJsonSchema
|
||||
from pydantic_core import PydanticOmit
|
||||
|
||||
from crewai_tools import tools
|
||||
|
||||
|
||||
class SchemaGenerator(GenerateJsonSchema):
|
||||
def handle_invalid_for_json_schema(self, schema, error_info):
|
||||
@@ -91,7 +91,7 @@ class EmbeddingService:
|
||||
"azure": "AZURE_OPENAI_API_KEY",
|
||||
"amazon-bedrock": "AWS_ACCESS_KEY_ID", # or AWS_PROFILE
|
||||
"cohere": "COHERE_API_KEY",
|
||||
"google-generativeai": "GOOGLE_API_KEY",
|
||||
"google-generativeai": "GEMINI_API_KEY",
|
||||
"google-vertex": "GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"huggingface": "HUGGINGFACE_API_KEY",
|
||||
"jina": "JINA_API_KEY",
|
||||
|
||||
@@ -247,7 +247,7 @@ class StagehandTool(BaseTool):
|
||||
if "claude" in model_str.lower() or "anthropic" in model_str.lower():
|
||||
return self.model_api_key or os.getenv("ANTHROPIC_API_KEY")
|
||||
if "gemini" in model_str.lower():
|
||||
return self.model_api_key or os.getenv("GOOGLE_API_KEY")
|
||||
return self.model_api_key or os.getenv("GEMINI_API_KEY")
|
||||
# Default to trying OpenAI, then Anthropic
|
||||
return (
|
||||
self.model_api_key
|
||||
@@ -313,7 +313,7 @@ class StagehandTool(BaseTool):
|
||||
|
||||
if not model_api_key:
|
||||
raise ValueError(
|
||||
"No appropriate API key found for model. Please set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY"
|
||||
"No appropriate API key found for model. Please set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GEMINI_API_KEY"
|
||||
)
|
||||
|
||||
# Build the StagehandConfig with proper parameter names
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register custom markers."""
|
||||
config.addinivalue_line("markers", "integration: mark test as an integration test")
|
||||
config.addinivalue_line("markers", "asyncio: mark test as an async test")
|
||||
|
||||
# Set the asyncio loop scope through ini configuration
|
||||
config.inicfg["asyncio_mode"] = "auto"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def event_loop():
|
||||
"""Create an instance of the default event loop for each test case."""
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
yield loop
|
||||
loop.close()
|
||||
@@ -2,7 +2,7 @@ import json
|
||||
from unittest import mock
|
||||
|
||||
from crewai.tools.base_tool import BaseTool, EnvVar
|
||||
from generate_tool_specs import ToolSpecExtractor
|
||||
from crewai_tools.generate_tool_specs import ToolSpecExtractor
|
||||
from pydantic import BaseModel, Field
|
||||
import pytest
|
||||
|
||||
@@ -61,8 +61,8 @@ def test_unwrap_schema(extractor):
|
||||
@pytest.fixture
|
||||
def mock_tool_extractor(extractor):
|
||||
with (
|
||||
mock.patch("generate_tool_specs.dir", return_value=["MockTool"]),
|
||||
mock.patch("generate_tool_specs.getattr", return_value=MockTool),
|
||||
mock.patch("crewai_tools.generate_tool_specs.dir", return_value=["MockTool"]),
|
||||
mock.patch("crewai_tools.generate_tool_specs.getattr", return_value=MockTool),
|
||||
):
|
||||
extractor.extract_all_tools()
|
||||
assert len(extractor.tools_spec) == 1
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,289 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"url": "https://firecrawl.dev", "includeTags": [], "excludeTags": [],
|
||||
"onlyMainContent": true, "waitFor": 0, "skipTlsVerification": true, "removeBase64Images":
|
||||
true, "fastMode": false, "blockAds": true, "storeInCache": true, "maxAge": 172800000,
|
||||
"formats": ["markdown"], "headers": {}, "mobile": false, "proxy": "auto", "origin":
|
||||
"python-sdk@4.5.0"}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Accept-Encoding:
|
||||
- gzip, deflate, zstd
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '350'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- python-requests/2.32.5
|
||||
method: POST
|
||||
uri: https://api.firecrawl.dev/v2/scrape
|
||||
response:
|
||||
body:
|
||||
string: "{\"success\":true,\"data\":{\"markdown\":\"We just raised our Series
|
||||
A and shipped Firecrawl /v2 \U0001F389. [Read the blog.](https://www.firecrawl.dev/blog/firecrawl-v2-series-a-announcement)\\n\\n[2
|
||||
Months Free \u2014 Annually](https://www.firecrawl.dev/pricing)\\n\\n# Turn
|
||||
websites into LLM-ready data\\n\\nPower your AI apps with clean web data\\n\\nfrom
|
||||
any website. [It's also open source.](https://github.com/firecrawl/firecrawl)\\n\\nScrape\\n\\nSearch\\nNew\\n\\nMap\\n\\nCrawl\\n\\nScrape\\n\\nLogo\\n\\nNavigation\\n\\nButton\\n\\nH1
|
||||
Title\\n\\nDescription\\n\\nCTA Button\\n\\n\\\\[ .JSON \\\\]\\n\\n```json\\n1[\\\\\\n2
|
||||
\ {\\\\\\n3 \\\"url\\\": \\\"https://example.com\\\",\\\\\\n4 \\\"markdown\\\":
|
||||
\\\"# Getting Started...\\\",\\\\\\n5 \\\"json\\\": { \\\"title\\\": \\\"Guide\\\",
|
||||
\\\"docs\\\": \\\"...\\\" },\\\\\\n6 \\\"screenshot\\\": \\\"https://example.com/hero.png\\\"\\\\\\n7
|
||||
\ }\\\\\\n8]\\n```\\n\\nScrape Completed\\n\\nTrusted by5000+\\n\\ncompaniesof
|
||||
all sizes\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\\\[01/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nMain Features\\n\\n//\\n\\nDeveloper First\\n\\n//\\n\\n##
|
||||
Startscraping today\\n\\nEnhance your apps with industry leading web scraping
|
||||
and crawling capabilities.\\n\\nScrape\\n\\nGet llm-ready data from websites.
|
||||
Markdown, JSON, screenshot, etc.\\n\\nSearch\\n\\nNew\\n\\nSearch the web
|
||||
and get full content from results.\\n\\nCrawl\\n\\nCrawl all the pages on
|
||||
a website and get data for each page.\\n\\nPython\\n\\nNode.js\\n\\nCurl\\n\\nCopy
|
||||
code\\n\\n```python\\n1# pip install firecrawl-py\\n2from firecrawl import
|
||||
Firecrawl\\n3\\n4app = Firecrawl(api_key=\\\"fc-YOUR_API_KEY\\\")\\n5\\n6#
|
||||
Scrape a website:\\n7app.scrape('firecrawl.dev')\\n8\\n9\\n10\\n```\\n\\n\\\\[
|
||||
.MD \\\\]\\n\\n```markdown\\n1# Firecrawl\\n2\\n3Firecrawl is a powerful web
|
||||
scraping\\n4library that makes it easy to extract\\n5data from websites.\\n6\\n7##
|
||||
Installation\\n8\\n9To install Firecrawl, run:\\n10\\n11\\n```\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIntegrations\\n\\n###
|
||||
Use well-known tools\\n\\nAlready fully integrated with the greatest existing
|
||||
tools and workflows.\\n\\n[See all integrations](https://www.firecrawl.dev/app)\\n\\n\\n\\nmendableai/firecrawl\\n\\nPublic\\n\\nStar\\n\\n65.3K\\n\\n\\\\[python-SDK\\\\]
|
||||
improvs/async\\n\\n#1337\\n\\n\xB7\\n\\nApr 18, 2025\\n\\n\xB7\\n\\n\\n\\nrafaelsideguide\\n\\nfeat(extract):
|
||||
cost limit\\n\\n#1473\\n\\n\xB7\\n\\nApr 17, 2025\\n\\n\xB7\\n\\n\\n\\nmogery\\n\\nfeat(scrape):
|
||||
get job result from GCS, avoid Redis\\n\\n#1461\\n\\n\xB7\\n\\nApr 15, 2025\\n\\n\xB7\\n\\n\\n\\nmogery\\n\\nExtract
|
||||
v2/rerank improvs\\n\\n#1437\\n\\n\xB7\\n\\nApr 11, 2025\\n\\n\xB7\\n\\n\\n\\nrafaelsideguide\\n\\n\\n\\n\\n\\n+90\\n\\nOpen
|
||||
Source\\n\\n### Code you can trust\\n\\nDeveloped transparently and collaboratively.
|
||||
Join our community of contributors.\\n\\n[Check out our repo](https://github.com/firecrawl/firecrawl)\\n\\n\\\\[02/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nCore\\n\\n//\\n\\nBuilt to outperform\\n\\n//\\n\\n##
|
||||
Core principles, provenperformance\\n\\nBuilt from the ground up to outperform
|
||||
traditional scrapers.\\n\\nNo proxy headaches\\n\\nReliable.Covers 96% of
|
||||
the web,\\n\\nincluding JS-heavy and protected pages. No proxies, no puppets,
|
||||
just clean data.\\n\\nFirecrawl\\n\\n96%\\n\\n\\n\\nPuppeteer\\n\\n79%\\n\\ncURL\\n\\n75%\\n\\nSpeed
|
||||
that feels invisible\\n\\nBlazingly fast.Delivers results in less than 1 second,
|
||||
fast for real-time agents\\n\\nand dynamic apps.\\n\\nURL\\n\\nCrawl\\n\\nScrape\\n\\nfirecrawl.dev/docs\\n\\n50ms\\n\\n51ms\\n\\nfirecrawl.dev/templates\\n\\n52ms\\n\\n50ms\\n\\nfirecrawl.dev/changelog\\n\\n49ms\\n\\n52ms\\n\\nfirecrawl.dev/about\\n\\n52ms\\n\\n50ms\\n\\nfirecrawl.dev/changelog\\n\\n50ms\\n\\n52ms\\n\\nfirecrawl.dev/playground\\n\\n51ms\\n\\n49ms\\n\\n\\\\[
|
||||
CTA \\\\]\\n\\n\\\\[ CRAWL \\\\]\\n\\n\\\\[ SCRAPE \\\\]\\n\\n\\\\[ CTA \\\\]\\n\\n//\\n\\nGet
|
||||
started\\n\\n//\\n\\nReady to build?\\n\\nStart getting Web Data for free
|
||||
and scale seamlessly as your project expands. No credit card needed.\\n\\n[Start
|
||||
for free](https://www.firecrawl.dev/signin) [See our plans](https://www.firecrawl.dev/pricing)\\n\\n\\\\[03/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nFeatures\\n\\n//\\n\\nZero configuration\\n\\n//\\n\\n##
|
||||
We handle the hard stuff\\n\\nRotating proxies, orchestration, rate limits,
|
||||
js-blocked content and more.\\n\\nDocs to data\\n\\nMedia parsing.Firecrawl
|
||||
can parse and output content from web hosted pdfs, docx, and more.\\n\\nhttps://example.com/docs/report.pdf\\n\\nhttps://example.com/files/brief.docx\\n\\nhttps://example.com/docs/guide.html\\n\\ndocx\\n\\nParsing...\\n\\nKnows
|
||||
the moment\\n\\nSmart wait.Firecrawl intelligently waits for content to load,
|
||||
making scraping faster and more reliable.\\n\\nhttps://example-spa.com\\n\\nRequest
|
||||
Sent\\n\\nScrapes the real thing\\n\\nCached, when you need it.Selective caching,
|
||||
you choose your caching patterns, growing web index.\\n\\n\\n\\nUser\\n\\nFirecrawl\\n\\nCache\\n\\nInvisible
|
||||
access\\n\\nStealth mode.Crawls the web without\\n\\nbeing blocked, mimics
|
||||
real users to access protected or dynamic content.\\n\\nInteractive scraping\\n\\nActions.Click,
|
||||
scroll, write, wait, press and more before extracting content.\\n\\nhttps://example.com\\n\\nNavigate\\n\\nClick\\n\\nType\\n\\nWait\\n\\nScroll\\n\\nPress\\n\\nScreenshot\\n\\nScrape\\n\\n\\\\[04/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nPricing\\n\\n//\\n\\nTransparent\\n\\n//\\n\\n## Flexible
|
||||
pricing\\n\\nExplore transparent pricing built for real-world scraping. Start
|
||||
for free, then scale as you grow.\\n\\n\U0001F1FA\U0001F1F8USD\\n\\nFree Plan\\n\\nA
|
||||
lightweight way to try scraping.\\n\\nNo cost, no card, no hassle.\\n\\n500
|
||||
credits\\n\\n$0123456789\\n\\none-time\\n\\nGet started\\n\\nScrape 500 pages\\n\\n2
|
||||
concurrent requests\\n\\nLow rate limits\\n\\nHobby\\n\\nGreat for side projects
|
||||
and small tools.\\n\\nFast, simple, no overkill.\\n\\n3,000 credits\\n\\n$01234567890123456789\\n\\n/monthly\\n\\nBilled
|
||||
yearly\\n\\n2 months free\\n\\nSubscribe\\n\\nScrape 3,000 pages\\n\\n5 concurrent
|
||||
requests\\n\\nBasic support\\n\\n$9 per extra 1k credits\\n\\nStandard\\n\\nMost
|
||||
popular\\n\\nPerfect for scaling with less effort.\\n\\nSimple, solid, dependable.\\n\\n100,000
|
||||
credits\\n\\n$01234567890123456789\\n\\n/monthly\\n\\nBilled yearly\\n\\n2
|
||||
months free\\n\\nSubscribe\\n\\nScrape 100,000 pages\\n\\n50 concurrent requests\\n\\nStandard
|
||||
support\\n\\n$47 per extra 35k credits\\n\\nGrowth\\n\\nBuilt for high volume
|
||||
and speed.\\n\\nFirecrawl at full force.\\n\\n500,000 credits\\n\\n$012345678901234567890123456789\\n\\n/monthly\\n\\nBilled
|
||||
yearly\\n\\n2 months free\\n\\nSubscribe\\n\\nScrape 500,000 pages\\n\\n100
|
||||
concurrent requests\\n\\nPriority support\\n\\n$177 per extra 175k credits\\n\\nExtra
|
||||
credits are available via auto-recharge packs. [Enable](https://www.firecrawl.dev/signin/signup)\\n\\nEnterprise\\n\\nPower
|
||||
at your pace\\n\\nUnlimited credits. Custom RPMs.\\n\\n[Contact sales](https://fk4bvu0n5qp.typeform.com/to/Ej6oydlg)
|
||||
[More details](https://www.firecrawl.dev/enterprise)\\n\\nBulk discounts\\n\\nTop
|
||||
priority support\\n\\nCustom concurrency limits\\n\\nImproved stealth proxies\\n\\nSLAs\\n\\nAdvanced
|
||||
security & controls\\n\\n\\\\[05/ 07 \\\\]\\n\\n\xB7\\n\\nTestimonials\\n\\n//\\n\\nCommunity\\n\\n//\\n\\n##
|
||||
People love building withFirecrawl\\n\\nDiscover why developers choose
|
||||
Firecrawl every day.\\n\\n[Morgan
|
||||
Linton@morganlinton\\\"If you're coding with AI, and haven't discovered @firecrawl\\\\_dev
|
||||
yet, prepare to have your mind blown \U0001F92F\\\"](https://x.com/morganlinton/status/1839454165703204955)
|
||||
[Chris
|
||||
DeWeese@chrisdeweese\\\\_\\\"Started using @firecrawl\\\\_dev for a project,
|
||||
I wish I used this sooner.\\\"](https://x.com/chrisdeweese_/status/1853587120406876601)
|
||||
[Alex
|
||||
Reibman@AlexReibman\\\"Moved our internal agent's web scraping tool from Apify
|
||||
to Firecrawl because it benchmarked 50x faster with AgentOps.\\\"](https://x.com/AlexReibman/status/1780299595484131836)
|
||||
[Tom
|
||||
- Morpho@TomReppelin\\\"I found gold today. Thank you @firecrawl\\\\_dev\\\"](https://x.com/TomReppelin/status/1844382491014201613)\\n\\n[Morgan
|
||||
Linton@morganlinton\\\"If you're coding with AI, and haven't discovered @firecrawl\\\\_dev
|
||||
yet, prepare to have your mind blown \U0001F92F\\\"](https://x.com/morganlinton/status/1839454165703204955)
|
||||
[Chris
|
||||
DeWeese@chrisdeweese\\\\_\\\"Started using @firecrawl\\\\_dev for a project,
|
||||
I wish I used this sooner.\\\"](https://x.com/chrisdeweese_/status/1853587120406876601)
|
||||
[Alex
|
||||
Reibman@AlexReibman\\\"Moved our internal agent's web scraping tool from Apify
|
||||
to Firecrawl because it benchmarked 50x faster with AgentOps.\\\"](https://x.com/AlexReibman/status/1780299595484131836)
|
||||
[Tom
|
||||
- Morpho@TomReppelin\\\"I found gold today. Thank you @firecrawl\\\\_dev\\\"](https://x.com/TomReppelin/status/1844382491014201613)\\n\\n[Bardia@thepericulum\\\"The
|
||||
Firecrawl team ships. I wanted types for their node SDK, and less than an
|
||||
hour later, I got them.\\\"](https://x.com/thepericulum/status/1781397799487078874)
|
||||
[Matt
|
||||
Busigin@mbusigin\\\"Firecrawl is dope. Congrats guys \U0001F44F\\\"](https://x.com/mbusigin/status/1836065372010656069)
|
||||
[Sumanth@Sumanth\\\\_077\\\"Web
|
||||
scraping will never be the same!\\\\\\\\\\n\\\\\\\\\\nFirecrawl is an open-source
|
||||
framework that takes a URL, crawls it, and conver...\\\"](https://x.com/Sumanth_077/status/1940049003074478511)
|
||||
[Steven
|
||||
Tey@steventey\\\"Open-source Clay alternative just dropped\\\\\\\\\\n\\\\\\\\\\nUpload
|
||||
a CSV of emails and...\\\"](https://x.com/steventey/status/1932945651761098889)\\n\\n[Bardia@thepericulum\\\"The
|
||||
Firecrawl team ships. I wanted types for their node SDK, and less than an
|
||||
hour later, I got them.\\\"](https://x.com/thepericulum/status/1781397799487078874)
|
||||
[Matt
|
||||
Busigin@mbusigin\\\"Firecrawl is dope. Congrats guys \U0001F44F\\\"](https://x.com/mbusigin/status/1836065372010656069)
|
||||
[Sumanth@Sumanth\\\\_077\\\"Web
|
||||
scraping will never be the same!\\\\\\\\\\n\\\\\\\\\\nFirecrawl is an open-source
|
||||
framework that takes a URL, crawls it, and conver...\\\"](https://x.com/Sumanth_077/status/1940049003074478511)
|
||||
[Steven
|
||||
Tey@steventey\\\"Open-source Clay alternative just dropped\\\\\\\\\\n\\\\\\\\\\nUpload
|
||||
a CSV of emails and...\\\"](https://x.com/steventey/status/1932945651761098889)\\n\\n\\\\[06/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nUse Cases\\n\\n//\\n\\nUse cases\\n\\n//\\n\\n## Transform
|
||||
\ web data into AI-powered solutions\\n\\nDiscover how Firecrawl customers
|
||||
are getting the most out of our API.\\n\\n[View all use cases](https://docs.firecrawl.dev/use-cases/overview)\\n\\nChat
|
||||
with context\\n\\nSmarter AI chats\\n\\nPower your AI assistants with real-time,
|
||||
accurate web content.\\n\\n[View docs](https://docs.firecrawl.dev/introduction)\\n\\n\\n\\nAI Assistant\\n\\nwithFirecrawl\\n\\nReal-time\xB7Updated
|
||||
2 min ago\\n\\nAsk anything...\\n\\nKnow your leads\\n\\nLead enrichment\\n\\nEnhance
|
||||
your sales data with\\n\\nweb information.\\n\\n[Check out Extract](https://www.firecrawl.dev/extract)\\n\\nExtracting
|
||||
leads from directory...\\n\\nTech startups\\n\\nWith contact info\\n\\nDecision
|
||||
makers\\n\\nFunding stage\\n\\nReady to engage\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nKnow your leads\\n\\nMCPs\\n\\nAdd
|
||||
powerful scraping to your\\n\\ncode editors.\\n\\n[Get started](https://docs.firecrawl.dev/mcp-server)\\n\\n\\n\\nClaude Code\\n\\n\\n\\nCursor\\n\\n\\n\\nWindsurf\\n\\n\u273B\\n\\nWelcome
|
||||
to Claude Code!\\n\\n/help for help, /status for your current setup\\n\\n>Try
|
||||
\\\"how do I log an error?\\\"\\n\\nBuild with context\\n\\nAI platforms\\n\\nLet
|
||||
your customers build AI apps\\n\\nwith web data.\\n\\n[Check out Map](https://docs.firecrawl.dev/features/map)\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nExtracting
|
||||
text...\\n\\nNo insight missed\\n\\nDeep research\\n\\nExtract comprehensive
|
||||
information for\\n\\nin-depth research.\\n\\n[Build your own with Search](https://docs.firecrawl.dev/features/search)\\n\\nDeep
|
||||
research in progress...\\n\\nAcademic papers\\n\\n0 found\\n\\nNews articles\\n\\n0
|
||||
found\\n\\nExpert opinions\\n\\n0 found\\n\\nResearch reports\\n\\n0 found\\n\\nIndustry
|
||||
data\\n\\n0 found\\n\\nAsk anything...\\n\\n\\\\[ CTA \\\\]\\n\\n\\\\[ CRAWL
|
||||
\\\\]\\n\\n\\\\[ SCRAPE \\\\]\\n\\n\\\\[ CTA \\\\]\\n\\n//\\n\\nGet started\\n\\n//\\n\\nReady
|
||||
to build?\\n\\nStart getting Web Data for free and scale seamlessly as your
|
||||
project expands. No credit card needed.\\n\\n[Start for free](https://www.firecrawl.dev/signin)
|
||||
[See our plans](https://www.firecrawl.dev/pricing)\\n\\n\\\\[07/ 07 \\\\]\\n\\n\xB7\\n\\nFAQ\\n\\n//\\n\\nFAQ\\n\\n//\\n\\n##
|
||||
Frequently askedquestions\\n\\nEverything you need to know about Firecrawl.\\n\\nGeneral\\n\\nWhat
|
||||
is Firecrawl?\\n\\nWhat sites work?\\n\\nWho can benefit from using Firecrawl?\\n\\nIs
|
||||
Firecrawl open-source?\\n\\nWhat is the difference between Firecrawl and other
|
||||
web scrapers?\\n\\nWhat is the difference between the open-source version
|
||||
and the hosted version?\\n\\nScraping & Crawling\\n\\nHow does Firecrawl handle
|
||||
dynamic content on websites?\\n\\nWhy is it not crawling all the pages?\\n\\nCan
|
||||
Firecrawl crawl websites without a sitemap?\\n\\nWhat formats can Firecrawl
|
||||
convert web data into?\\n\\nHow does Firecrawl ensure the cleanliness of the
|
||||
data?\\n\\nIs Firecrawl suitable for large-scale data scraping projects?\\n\\nDoes
|
||||
it respect robots.txt?\\n\\nWhat measures does Firecrawl take to handle web
|
||||
scraping challenges like rate limits and caching?\\n\\nDoes Firecrawl handle
|
||||
captcha or authentication?\\n\\nAPI Related\\n\\nWhere can I find my API key?\\n\\nBilling\\n\\nIs
|
||||
Firecrawl free?\\n\\nIs there a pay-per-use plan instead of monthly?\\n\\nDo
|
||||
credits roll over to the next month?\\n\\nHow many credits do scraping and
|
||||
crawling cost?\\n\\nDo you charge for failed requests?\\n\\nWhat payment methods
|
||||
do you accept?\\n\\nFOOTER\\n\\nThe easiest way to extract\\n\\ndata from
|
||||
the web\\n\\nBacked by\\n\\nY Combinator\\n\\n[Linkedin](https://www.linkedin.com/company/firecrawl)
|
||||
[Github](https://github.com/firecrawl/firecrawl)\\n\\nSOC II \xB7 Type 2\\n\\nAICPA\\n\\nSOC
|
||||
2\\n\\n[X (Twitter)](https://x.com/firecrawl_dev) [Discord](https://discord.gg/gSmWdAkdwd)\\n\\nProducts\\n\\n[Playground](https://www.firecrawl.dev/playground)
|
||||
[Extract](https://www.firecrawl.dev/extract) [Pricing](https://www.firecrawl.dev/pricing)
|
||||
[Templates](https://www.firecrawl.dev/templates) [Changelog](https://www.firecrawl.dev/changelog)\\n\\nUse
|
||||
Cases\\n\\n[AI Platforms](https://docs.firecrawl.dev/use-cases/ai-platforms)
|
||||
[Lead Enrichment](https://docs.firecrawl.dev/use-cases/lead-enrichment) [SEO
|
||||
Platforms](https://docs.firecrawl.dev/use-cases/seo-platforms) [Deep Research](https://docs.firecrawl.dev/use-cases/deep-research)\\n\\nDocumentation\\n\\n[Getting
|
||||
started](https://docs.firecrawl.dev/introduction) [API Reference](https://docs.firecrawl.dev/api-reference/introduction)
|
||||
[Integrations](https://www.firecrawl.dev/app) [Examples](https://docs.firecrawl.dev/use-cases/overview)
|
||||
[SDKs](https://docs.firecrawl.dev/sdks/overview)\\n\\nCompany\\n\\n[Blog](https://www.firecrawl.dev/blog)
|
||||
[Careers](https://www.firecrawl.dev/careers) [Creator & OSS program](https://www.firecrawl.dev/creator-oss-program)
|
||||
[Student program](https://www.firecrawl.dev/student-program)\\n\\n\xA9 2025
|
||||
Firecrawl\\n\\n[Terms of Service](https://www.firecrawl.dev/terms-of-service)
|
||||
[Privacy Policy](https://www.firecrawl.dev/privacy-policy) [Report Abuse](mailto:help@firecrawl.com?subject=Issue:)\\n\\n[All
|
||||
systems normal](https://status.firecrawl.dev/)\\n\\nStripeM-Inner\",\"metadata\":{\"twitter:title\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"publisher\":\"Firecrawl\",\"ogUrl\":\"https://www.firecrawl.dev\",\"robots\":\"follow,
|
||||
index\",\"title\":\"Firecrawl - The Web Data API for AI\",\"ogDescription\":\"The
|
||||
web crawling, scraping, and search API for AI. Built for scale. Firecrawl
|
||||
delivers the entire internet to AI agents and builders. Clean, structured,
|
||||
and ready to reason with.\",\"ogImage\":\"https://www.firecrawl.dev/og.png\",\"viewport\":\"width=device-width,
|
||||
initial-scale=1, maximum-scale=1, user-scalable=no\",\"og:url\":\"https://www.firecrawl.dev\",\"og:site_name\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"og:type\":\"website\",\"twitter:image\":\"https://www.firecrawl.dev/og.png\",\"author\":\"Firecrawl\",\"og:title\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"favicon\":\"https://www.firecrawl.dev/favicon.png\",\"description\":\"The
|
||||
web crawling, scraping, and search API for AI. Built for scale. Firecrawl
|
||||
delivers the entire internet to AI agents and builders. Clean, structured,
|
||||
and ready to reason with.\",\"referrer\":\"origin-when-cross-origin\",\"twitter:site\":\"@Vercel\",\"ogSiteName\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"og:image\":\"https://www.firecrawl.dev/og.png\",\"twitter:card\":\"summary_large_image\",\"twitter:creator\":\"@Vercel\",\"twitter:description\":\"The
|
||||
web crawling, scraping, and search API for AI. Built for scale. Firecrawl
|
||||
delivers the entire internet to AI agents and builders. Clean, structured,
|
||||
and ready to reason with.\",\"language\":\"en\",\"keywords\":\"Firecrawl,Markdown,Data,Mendable,Langchain\",\"creator\":\"Firecrawl\",\"ogTitle\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"og:description\":\"The web crawling, scraping,
|
||||
and search API for AI. Built for scale. Firecrawl delivers the entire internet
|
||||
to AI agents and builders. Clean, structured, and ready to reason with.\",\"scrapeId\":\"e78d8060-d581-4e5e-b25a-90cfdad48530\",\"sourceURL\":\"https://firecrawl.dev\",\"url\":\"https://www.firecrawl.dev/\",\"statusCode\":200,\"contentType\":\"text/html;
|
||||
charset=utf-8\",\"proxyUsed\":\"basic\",\"cacheState\":\"hit\",\"cachedAt\":\"2025-10-29T13:09:07.713Z\",\"creditsUsed\":1}}}"
|
||||
headers:
|
||||
Access-Control-Allow-Origin:
|
||||
- '*'
|
||||
Alt-Svc:
|
||||
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
|
||||
Content-Length:
|
||||
- '24693'
|
||||
Content-Type:
|
||||
- application/json; charset=utf-8
|
||||
Date:
|
||||
- Wed, 29 Oct 2025 14:34:03 GMT
|
||||
ETag:
|
||||
- W/"6075-Q1W6uMv95JKEZARbtaiPYYMojlU"
|
||||
Via:
|
||||
- 1.1 google
|
||||
X-Powered-By:
|
||||
- Express
|
||||
X-Response-Time:
|
||||
- 4719.998ms
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -1,937 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"query": "firecrawl", "limit": 5, "scrapeOptions": {"includeTags": [],
|
||||
"excludeTags": [], "onlyMainContent": true, "waitFor": 0, "skipTlsVerification":
|
||||
true, "removeBase64Images": true, "fastMode": false, "blockAds": true, "storeInCache":
|
||||
true, "maxAge": 14400000, "formats": ["markdown"], "mobile": false}, "origin":
|
||||
"python-sdk@4.5.0"}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Accept-Encoding:
|
||||
- gzip, deflate, zstd
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '338'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- python-requests/2.32.5
|
||||
method: POST
|
||||
uri: https://api.firecrawl.dev/v2/search
|
||||
response:
|
||||
body:
|
||||
string: "{\"success\":true,\"data\":{\"web\":[{\"url\":\"https://www.firecrawl.dev/\",\"title\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"description\":\"The web crawling, scraping,
|
||||
and search API for AI. Built for scale. Firecrawl delivers the entire internet
|
||||
to AI agents and builders.\",\"position\":1,\"markdown\":\"We just raised
|
||||
our Series A and shipped Firecrawl /v2 \U0001F389. [Read the blog.](https://www.firecrawl.dev/blog/firecrawl-v2-series-a-announcement)\\n\\n[2
|
||||
Months Free \u2014 Annually](https://www.firecrawl.dev/pricing)\\n\\n# Turn
|
||||
websites into LLM-ready data\\n\\nPower your AI apps with clean web data\\n\\nfrom
|
||||
any website. [It's also open source.](https://github.com/firecrawl/firecrawl)\\n\\nScrape\\n\\nSearch\\nNew\\n\\nMap\\n\\nCrawl\\n\\nScrape\\n\\nLogo\\n\\nNavigation\\n\\nButton\\n\\nH1
|
||||
Title\\n\\nDescription\\n\\nCTA Button\\n\\n\\\\[ .JSON \\\\]\\n\\n```json\\n1[\\\\\\n2
|
||||
\ {\\\\\\n3 \\\"url\\\": \\\"https://example.com\\\",\\\\\\n4 \\\"markdown\\\":
|
||||
\\\"# Getting Started...\\\",\\\\\\n5 \\\"json\\\": { \\\"title\\\": \\\"Guide\\\",
|
||||
\\\"docs\\\": \\\"...\\\" },\\\\\\n6 \\\"screenshot\\\": \\\"https://example.com/hero.png\\\"\\\\\\n7
|
||||
\ }\\\\\\n8]\\n```\\n\\nScrape Completed\\n\\nTrusted by5000+\\n\\ncompaniesof
|
||||
all sizes\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\\\[01/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nMain Features\\n\\n//\\n\\nDeveloper First\\n\\n//\\n\\n##
|
||||
Startscraping today\\n\\nEnhance your apps with industry leading web scraping
|
||||
and crawling capabilities.\\n\\nScrape\\n\\nGet llm-ready data from websites.
|
||||
Markdown, JSON, screenshot, etc.\\n\\nSearch\\n\\nNew\\n\\nSearch the web
|
||||
and get full content from results.\\n\\nCrawl\\n\\nCrawl all the pages on
|
||||
a website and get data for each page.\\n\\nPython\\n\\nNode.js\\n\\nCurl\\n\\nCopy
|
||||
code\\n\\n```python\\n1# pip install firecrawl-py\\n2from firecrawl import
|
||||
Firecrawl\\n3\\n4app = Firecrawl(api_key=\\\"fc-YOUR_API_KEY\\\")\\n5\\n6#
|
||||
Scrape a website:\\n7app.scrape('firecrawl.dev')\\n8\\n9\\n10\\n```\\n\\n\\\\[
|
||||
.MD \\\\]\\n\\n```markdown\\n1# Firecrawl\\n2\\n3Firecrawl is a powerful web
|
||||
scraping\\n4library that makes it easy to extract\\n5data from websites.\\n6\\n7##
|
||||
Installation\\n8\\n9To install Firecrawl, run:\\n10\\n11\\n```\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIntegrations\\n\\n###
|
||||
Use well-known tools\\n\\nAlready fully integrated with the greatest existing
|
||||
tools and workflows.\\n\\n[See all integrations](https://www.firecrawl.dev/app)\\n\\n\\n\\nmendableai/firecrawl\\n\\nPublic\\n\\nStar\\n\\n65.3K\\n\\n\\\\[python-SDK\\\\]
|
||||
improvs/async\\n\\n#1337\\n\\n\xB7\\n\\nApr 18, 2025\\n\\n\xB7\\n\\n\\n\\nrafaelsideguide\\n\\nfeat(extract):
|
||||
cost limit\\n\\n#1473\\n\\n\xB7\\n\\nApr 17, 2025\\n\\n\xB7\\n\\n\\n\\nmogery\\n\\nfeat(scrape):
|
||||
get job result from GCS, avoid Redis\\n\\n#1461\\n\\n\xB7\\n\\nApr 15, 2025\\n\\n\xB7\\n\\n\\n\\nmogery\\n\\nExtract
|
||||
v2/rerank improvs\\n\\n#1437\\n\\n\xB7\\n\\nApr 11, 2025\\n\\n\xB7\\n\\n\\n\\nrafaelsideguide\\n\\n\\n\\n\\n\\n+90\\n\\nOpen
|
||||
Source\\n\\n### Code you can trust\\n\\nDeveloped transparently and collaboratively.
|
||||
Join our community of contributors.\\n\\n[Check out our repo](https://github.com/firecrawl/firecrawl)\\n\\n\\\\[02/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nCore\\n\\n//\\n\\nBuilt to outperform\\n\\n//\\n\\n##
|
||||
Core principles, provenperformance\\n\\nBuilt from the ground up to outperform
|
||||
traditional scrapers.\\n\\nNo proxy headaches\\n\\nReliable.Covers 96% of
|
||||
the web,\\n\\nincluding JS-heavy and protected pages. No proxies, no puppets,
|
||||
just clean data.\\n\\nFirecrawl\\n\\n96%\\n\\n\\n\\nPuppeteer\\n\\n79%\\n\\ncURL\\n\\n75%\\n\\nSpeed
|
||||
that feels invisible\\n\\nBlazingly fast.Delivers results in less than 1 second,
|
||||
fast for real-time agents\\n\\nand dynamic apps.\\n\\nURL\\n\\nCrawl\\n\\nScrape\\n\\nfirecrawl.dev/docs\\n\\n50ms\\n\\n51ms\\n\\nfirecrawl.dev/templates\\n\\n52ms\\n\\n50ms\\n\\nfirecrawl.dev/changelog\\n\\n49ms\\n\\n52ms\\n\\nfirecrawl.dev/about\\n\\n52ms\\n\\n50ms\\n\\nfirecrawl.dev/changelog\\n\\n50ms\\n\\n52ms\\n\\nfirecrawl.dev/playground\\n\\n51ms\\n\\n49ms\\n\\n\\\\[
|
||||
CTA \\\\]\\n\\n\\\\[ CRAWL \\\\]\\n\\n\\\\[ SCRAPE \\\\]\\n\\n\\\\[ CTA \\\\]\\n\\n//\\n\\nGet
|
||||
started\\n\\n//\\n\\nReady to build?\\n\\nStart getting Web Data for free
|
||||
and scale seamlessly as your project expands. No credit card needed.\\n\\n[Start
|
||||
for free](https://www.firecrawl.dev/signin) [See our plans](https://www.firecrawl.dev/pricing)\\n\\n\\\\[03/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nFeatures\\n\\n//\\n\\nZero configuration\\n\\n//\\n\\n##
|
||||
We handle the hard stuff\\n\\nRotating proxies, orchestration, rate limits,
|
||||
js-blocked content and more.\\n\\nDocs to data\\n\\nMedia parsing.Firecrawl
|
||||
can parse and output content from web hosted pdfs, docx, and more.\\n\\nhttps://example.com/docs/report.pdf\\n\\nhttps://example.com/files/brief.docx\\n\\nhttps://example.com/docs/guide.html\\n\\ndocx\\n\\nParsing...\\n\\nKnows
|
||||
the moment\\n\\nSmart wait.Firecrawl intelligently waits for content to load,
|
||||
making scraping faster and more reliable.\\n\\nhttps://example-spa.com\\n\\nRequest
|
||||
Sent\\n\\nScrapes the real thing\\n\\nCached, when you need it.Selective caching,
|
||||
you choose your caching patterns, growing web index.\\n\\n\\n\\nUser\\n\\nFirecrawl\\n\\nCache\\n\\nInvisible
|
||||
access\\n\\nStealth mode.Crawls the web without\\n\\nbeing blocked, mimics
|
||||
real users to access protected or dynamic content.\\n\\nInteractive scraping\\n\\nActions.Click,
|
||||
scroll, write, wait, press and more before extracting content.\\n\\nhttps://example.com\\n\\nNavigate\\n\\nClick\\n\\nType\\n\\nWait\\n\\nScroll\\n\\nPress\\n\\nScreenshot\\n\\nScrape\\n\\n\\\\[04/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nPricing\\n\\n//\\n\\nTransparent\\n\\n//\\n\\n## Flexible
|
||||
pricing\\n\\nExplore transparent pricing built for real-world scraping. Start
|
||||
for free, then scale as you grow.\\n\\n\U0001F1FA\U0001F1F8USD\\n\\nFree Plan\\n\\nA
|
||||
lightweight way to try scraping.\\n\\nNo cost, no card, no hassle.\\n\\n500
|
||||
credits\\n\\n$0123456789\\n\\none-time\\n\\nGet started\\n\\nScrape 500 pages\\n\\n2
|
||||
concurrent requests\\n\\nLow rate limits\\n\\nHobby\\n\\nGreat for side projects
|
||||
and small tools.\\n\\nFast, simple, no overkill.\\n\\n3,000 credits\\n\\n$01234567890123456789\\n\\n/monthly\\n\\nBilled
|
||||
yearly\\n\\n2 months free\\n\\nSubscribe\\n\\nScrape 3,000 pages\\n\\n5 concurrent
|
||||
requests\\n\\nBasic support\\n\\n$9 per extra 1k credits\\n\\nStandard\\n\\nMost
|
||||
popular\\n\\nPerfect for scaling with less effort.\\n\\nSimple, solid, dependable.\\n\\n100,000
|
||||
credits\\n\\n$01234567890123456789\\n\\n/monthly\\n\\nBilled yearly\\n\\n2
|
||||
months free\\n\\nSubscribe\\n\\nScrape 100,000 pages\\n\\n50 concurrent requests\\n\\nStandard
|
||||
support\\n\\n$47 per extra 35k credits\\n\\nGrowth\\n\\nBuilt for high volume
|
||||
and speed.\\n\\nFirecrawl at full force.\\n\\n500,000 credits\\n\\n$012345678901234567890123456789\\n\\n/monthly\\n\\nBilled
|
||||
yearly\\n\\n2 months free\\n\\nSubscribe\\n\\nScrape 500,000 pages\\n\\n100
|
||||
concurrent requests\\n\\nPriority support\\n\\n$177 per extra 175k credits\\n\\nExtra
|
||||
credits are available via auto-recharge packs. [Enable](https://www.firecrawl.dev/signin/signup)\\n\\nEnterprise\\n\\nPower
|
||||
at your pace\\n\\nUnlimited credits. Custom RPMs.\\n\\n[Contact sales](https://fk4bvu0n5qp.typeform.com/to/Ej6oydlg)
|
||||
[More details](https://www.firecrawl.dev/enterprise)\\n\\nBulk discounts\\n\\nTop
|
||||
priority support\\n\\nCustom concurrency limits\\n\\nImproved stealth proxies\\n\\nSLAs\\n\\nAdvanced
|
||||
security & controls\\n\\n\\\\[05/ 07 \\\\]\\n\\n\xB7\\n\\nTestimonials\\n\\n//\\n\\nCommunity\\n\\n//\\n\\n##
|
||||
People love building withFirecrawl\\n\\nDiscover why developers choose
|
||||
Firecrawl every day.\\n\\n[Morgan
|
||||
Linton@morganlinton\\\"If you're coding with AI, and haven't discovered @firecrawl\\\\_dev
|
||||
yet, prepare to have your mind blown \U0001F92F\\\"](https://x.com/morganlinton/status/1839454165703204955)
|
||||
[Chris
|
||||
DeWeese@chrisdeweese\\\\_\\\"Started using @firecrawl\\\\_dev for a project,
|
||||
I wish I used this sooner.\\\"](https://x.com/chrisdeweese_/status/1853587120406876601)
|
||||
[Alex
|
||||
Reibman@AlexReibman\\\"Moved our internal agent's web scraping tool from Apify
|
||||
to Firecrawl because it benchmarked 50x faster with AgentOps.\\\"](https://x.com/AlexReibman/status/1780299595484131836)
|
||||
[Tom
|
||||
- Morpho@TomReppelin\\\"I found gold today. Thank you @firecrawl\\\\_dev\\\"](https://x.com/TomReppelin/status/1844382491014201613)\\n\\n[Morgan
|
||||
Linton@morganlinton\\\"If you're coding with AI, and haven't discovered @firecrawl\\\\_dev
|
||||
yet, prepare to have your mind blown \U0001F92F\\\"](https://x.com/morganlinton/status/1839454165703204955)
|
||||
[Chris
|
||||
DeWeese@chrisdeweese\\\\_\\\"Started using @firecrawl\\\\_dev for a project,
|
||||
I wish I used this sooner.\\\"](https://x.com/chrisdeweese_/status/1853587120406876601)
|
||||
[Alex
|
||||
Reibman@AlexReibman\\\"Moved our internal agent's web scraping tool from Apify
|
||||
to Firecrawl because it benchmarked 50x faster with AgentOps.\\\"](https://x.com/AlexReibman/status/1780299595484131836)
|
||||
[Tom
|
||||
- Morpho@TomReppelin\\\"I found gold today. Thank you @firecrawl\\\\_dev\\\"](https://x.com/TomReppelin/status/1844382491014201613)\\n\\n[Bardia@thepericulum\\\"The
|
||||
Firecrawl team ships. I wanted types for their node SDK, and less than an
|
||||
hour later, I got them.\\\"](https://x.com/thepericulum/status/1781397799487078874)
|
||||
[Matt
|
||||
Busigin@mbusigin\\\"Firecrawl is dope. Congrats guys \U0001F44F\\\"](https://x.com/mbusigin/status/1836065372010656069)
|
||||
[Sumanth@Sumanth\\\\_077\\\"Web
|
||||
scraping will never be the same!\\\\\\\\\\n\\\\\\\\\\nFirecrawl is an open-source
|
||||
framework that takes a URL, crawls it, and conver...\\\"](https://x.com/Sumanth_077/status/1940049003074478511)
|
||||
[Steven
|
||||
Tey@steventey\\\"Open-source Clay alternative just dropped\\\\\\\\\\n\\\\\\\\\\nUpload
|
||||
a CSV of emails and...\\\"](https://x.com/steventey/status/1932945651761098889)\\n\\n[Bardia@thepericulum\\\"The
|
||||
Firecrawl team ships. I wanted types for their node SDK, and less than an
|
||||
hour later, I got them.\\\"](https://x.com/thepericulum/status/1781397799487078874)
|
||||
[Matt
|
||||
Busigin@mbusigin\\\"Firecrawl is dope. Congrats guys \U0001F44F\\\"](https://x.com/mbusigin/status/1836065372010656069)
|
||||
[Sumanth@Sumanth\\\\_077\\\"Web
|
||||
scraping will never be the same!\\\\\\\\\\n\\\\\\\\\\nFirecrawl is an open-source
|
||||
framework that takes a URL, crawls it, and conver...\\\"](https://x.com/Sumanth_077/status/1940049003074478511)
|
||||
[Steven
|
||||
Tey@steventey\\\"Open-source Clay alternative just dropped\\\\\\\\\\n\\\\\\\\\\nUpload
|
||||
a CSV of emails and...\\\"](https://x.com/steventey/status/1932945651761098889)\\n\\n\\\\[06/
|
||||
07 \\\\]\\n\\n\xB7\\n\\nUse Cases\\n\\n//\\n\\nUse cases\\n\\n//\\n\\n## Transform
|
||||
\ web data into AI-powered solutions\\n\\nDiscover how Firecrawl customers
|
||||
are getting the most out of our API.\\n\\n[View all use cases](https://docs.firecrawl.dev/use-cases/overview)\\n\\nChat
|
||||
with context\\n\\nSmarter AI chats\\n\\nPower your AI assistants with real-time,
|
||||
accurate web content.\\n\\n[View docs](https://docs.firecrawl.dev/introduction)\\n\\n\\n\\nAI Assistant\\n\\nwithFirecrawl\\n\\nReal-time\xB7Updated
|
||||
2 min ago\\n\\nAsk anything...\\n\\nKnow your leads\\n\\nLead enrichment\\n\\nEnhance
|
||||
your sales data with\\n\\nweb information.\\n\\n[Check out Extract](https://www.firecrawl.dev/extract)\\n\\nExtracting
|
||||
leads from directory...\\n\\nTech startups\\n\\nWith contact info\\n\\nDecision
|
||||
makers\\n\\nFunding stage\\n\\nReady to engage\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nKnow your leads\\n\\nMCPs\\n\\nAdd
|
||||
powerful scraping to your\\n\\ncode editors.\\n\\n[Get started](https://docs.firecrawl.dev/mcp-server)\\n\\n\\n\\nClaude Code\\n\\n\\n\\nCursor\\n\\n\\n\\nWindsurf\\n\\n\u273B\\n\\nWelcome
|
||||
to Claude Code!\\n\\n/help for help, /status for your current setup\\n\\n>Try
|
||||
\\\"how do I log an error?\\\"\\n\\nBuild with context\\n\\nAI platforms\\n\\nLet
|
||||
your customers build AI apps\\n\\nwith web data.\\n\\n[Check out Map](https://docs.firecrawl.dev/features/map)\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nExtracting
|
||||
text...\\n\\nNo insight missed\\n\\nDeep research\\n\\nExtract comprehensive
|
||||
information for\\n\\nin-depth research.\\n\\n[Build your own with Search](https://docs.firecrawl.dev/features/search)\\n\\nDeep
|
||||
research in progress...\\n\\nAcademic papers\\n\\n0 found\\n\\nNews articles\\n\\n0
|
||||
found\\n\\nExpert opinions\\n\\n0 found\\n\\nResearch reports\\n\\n0 found\\n\\nIndustry
|
||||
data\\n\\n0 found\\n\\nAsk anything...\\n\\n\\\\[ CTA \\\\]\\n\\n\\\\[ CRAWL
|
||||
\\\\]\\n\\n\\\\[ SCRAPE \\\\]\\n\\n\\\\[ CTA \\\\]\\n\\n//\\n\\nGet started\\n\\n//\\n\\nReady
|
||||
to build?\\n\\nStart getting Web Data for free and scale seamlessly as your
|
||||
project expands. No credit card needed.\\n\\n[Start for free](https://www.firecrawl.dev/signin)
|
||||
[See our plans](https://www.firecrawl.dev/pricing)\\n\\n\\\\[07/ 07 \\\\]\\n\\n\xB7\\n\\nFAQ\\n\\n//\\n\\nFAQ\\n\\n//\\n\\n##
|
||||
Frequently askedquestions\\n\\nEverything you need to know about Firecrawl.\\n\\nGeneral\\n\\nWhat
|
||||
is Firecrawl?\\n\\nWhat sites work?\\n\\nWho can benefit from using Firecrawl?\\n\\nIs
|
||||
Firecrawl open-source?\\n\\nWhat is the difference between Firecrawl and other
|
||||
web scrapers?\\n\\nWhat is the difference between the open-source version
|
||||
and the hosted version?\\n\\nScraping & Crawling\\n\\nHow does Firecrawl handle
|
||||
dynamic content on websites?\\n\\nWhy is it not crawling all the pages?\\n\\nCan
|
||||
Firecrawl crawl websites without a sitemap?\\n\\nWhat formats can Firecrawl
|
||||
convert web data into?\\n\\nHow does Firecrawl ensure the cleanliness of the
|
||||
data?\\n\\nIs Firecrawl suitable for large-scale data scraping projects?\\n\\nDoes
|
||||
it respect robots.txt?\\n\\nWhat measures does Firecrawl take to handle web
|
||||
scraping challenges like rate limits and caching?\\n\\nDoes Firecrawl handle
|
||||
captcha or authentication?\\n\\nAPI Related\\n\\nWhere can I find my API key?\\n\\nBilling\\n\\nIs
|
||||
Firecrawl free?\\n\\nIs there a pay-per-use plan instead of monthly?\\n\\nDo
|
||||
credits roll over to the next month?\\n\\nHow many credits do scraping and
|
||||
crawling cost?\\n\\nDo you charge for failed requests?\\n\\nWhat payment methods
|
||||
do you accept?\\n\\nFOOTER\\n\\nThe easiest way to extract\\n\\ndata from
|
||||
the web\\n\\nBacked by\\n\\nY Combinator\\n\\n[Linkedin](https://www.linkedin.com/company/firecrawl)
|
||||
[Github](https://github.com/firecrawl/firecrawl)\\n\\nSOC II \xB7 Type 2\\n\\nAICPA\\n\\nSOC
|
||||
2\\n\\n[X (Twitter)](https://x.com/firecrawl_dev) [Discord](https://discord.gg/gSmWdAkdwd)\\n\\nProducts\\n\\n[Playground](https://www.firecrawl.dev/playground)
|
||||
[Extract](https://www.firecrawl.dev/extract) [Pricing](https://www.firecrawl.dev/pricing)
|
||||
[Templates](https://www.firecrawl.dev/templates) [Changelog](https://www.firecrawl.dev/changelog)\\n\\nUse
|
||||
Cases\\n\\n[AI Platforms](https://docs.firecrawl.dev/use-cases/ai-platforms)
|
||||
[Lead Enrichment](https://docs.firecrawl.dev/use-cases/lead-enrichment) [SEO
|
||||
Platforms](https://docs.firecrawl.dev/use-cases/seo-platforms) [Deep Research](https://docs.firecrawl.dev/use-cases/deep-research)\\n\\nDocumentation\\n\\n[Getting
|
||||
started](https://docs.firecrawl.dev/introduction) [API Reference](https://docs.firecrawl.dev/api-reference/introduction)
|
||||
[Integrations](https://www.firecrawl.dev/app) [Examples](https://docs.firecrawl.dev/use-cases/overview)
|
||||
[SDKs](https://docs.firecrawl.dev/sdks/overview)\\n\\nCompany\\n\\n[Blog](https://www.firecrawl.dev/blog)
|
||||
[Careers](https://www.firecrawl.dev/careers) [Creator & OSS program](https://www.firecrawl.dev/creator-oss-program)
|
||||
[Student program](https://www.firecrawl.dev/student-program)\\n\\n\xA9 2025
|
||||
Firecrawl\\n\\n[Terms of Service](https://www.firecrawl.dev/terms-of-service)
|
||||
[Privacy Policy](https://www.firecrawl.dev/privacy-policy) [Report Abuse](mailto:help@firecrawl.com?subject=Issue:)\\n\\n[All
|
||||
systems normal](https://status.firecrawl.dev/)\\n\\nStripeM-Inner\",\"metadata\":{\"favicon\":\"https://www.firecrawl.dev/favicon.png\",\"ogUrl\":\"https://www.firecrawl.dev\",\"ogImage\":\"https://www.firecrawl.dev/og.png\",\"referrer\":\"origin-when-cross-origin\",\"ogDescription\":\"The
|
||||
web crawling, scraping, and search API for AI. Built for scale. Firecrawl
|
||||
delivers the entire internet to AI agents and builders. Clean, structured,
|
||||
and ready to reason with.\",\"robots\":\"follow, index\",\"twitter:card\":\"summary_large_image\",\"og:site_name\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"twitter:title\":\"Firecrawl - The Web Data API
|
||||
for AI\",\"og:image\":\"https://www.firecrawl.dev/og.png\",\"title\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"og:description\":\"The web crawling, scraping,
|
||||
and search API for AI. Built for scale. Firecrawl delivers the entire internet
|
||||
to AI agents and builders. Clean, structured, and ready to reason with.\",\"twitter:image\":\"https://www.firecrawl.dev/og.png\",\"viewport\":\"width=device-width,
|
||||
initial-scale=1, maximum-scale=1, user-scalable=no\",\"ogSiteName\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"keywords\":\"Firecrawl,Markdown,Data,Mendable,Langchain\",\"author\":\"Firecrawl\",\"og:title\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"twitter:description\":\"The web crawling, scraping,
|
||||
and search API for AI. Built for scale. Firecrawl delivers the entire internet
|
||||
to AI agents and builders. Clean, structured, and ready to reason with.\",\"description\":\"The
|
||||
web crawling, scraping, and search API for AI. Built for scale. Firecrawl
|
||||
delivers the entire internet to AI agents and builders. Clean, structured,
|
||||
and ready to reason with.\",\"twitter:site\":\"@Vercel\",\"og:url\":\"https://www.firecrawl.dev\",\"og:type\":\"website\",\"ogTitle\":\"Firecrawl
|
||||
- The Web Data API for AI\",\"language\":\"en\",\"creator\":\"Firecrawl\",\"publisher\":\"Firecrawl\",\"twitter:creator\":\"@Vercel\",\"scrapeId\":\"57b0586f-36e8-4923-aaa2-88ff58c03999\",\"sourceURL\":\"https://www.firecrawl.dev/\",\"url\":\"https://www.firecrawl.dev/\",\"statusCode\":200,\"contentType\":\"text/html;
|
||||
charset=utf-8\",\"proxyUsed\":\"basic\",\"cacheState\":\"hit\",\"cachedAt\":\"2025-10-29T13:09:07.713Z\"}},{\"url\":\"https://github.com/firecrawl/firecrawl\",\"title\":\"firecrawl/firecrawl:
|
||||
The Web Data API for AI - Turn entire ... - GitHub\",\"description\":\"Firecrawl
|
||||
is an API service that takes a URL, crawls it, and converts it into clean
|
||||
markdown or structured data. We crawl all accessible subpages and give you
|
||||
...\",\"position\":2,\"category\":\"github\",\"markdown\":\"[Skip to content](https://github.com/firecrawl/firecrawl#start-of-content)\\n\\nYou
|
||||
signed in with another tab or window. [Reload](https://github.com/firecrawl/firecrawl)
|
||||
to refresh your session.You signed out in another tab or window. [Reload](https://github.com/firecrawl/firecrawl)
|
||||
to refresh your session.You switched accounts on another tab or window. [Reload](https://github.com/firecrawl/firecrawl)
|
||||
to refresh your session.Dismiss alert\\n\\n{{ message }}\\n\\n[firecrawl](https://github.com/firecrawl)/
|
||||
**[firecrawl](https://github.com/firecrawl/firecrawl)** Public\\n\\n- Couldn't
|
||||
load subscription status.\\nRetry\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n### Uh
|
||||
oh!\\n\\n\\n\\n\\n\\n\\n\\nThere was an error while loading. [Please reload
|
||||
this page](https://github.com/firecrawl/firecrawl).\\n\\n- [Fork\\\\\\\\\\n5.1k](https://github.com/login?return_to=%2Ffirecrawl%2Ffirecrawl)\\n-
|
||||
[Star\\\\\\\\\\n65.2k](https://github.com/login?return_to=%2Ffirecrawl%2Ffirecrawl)\\n\\n\\n\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data\\n\\n\\n[firecrawl.dev](https://firecrawl.dev/ \\\"https://firecrawl.dev\\\")\\n\\n###
|
||||
License\\n\\n[AGPL-3.0 license](https://github.com/firecrawl/firecrawl/blob/main/LICENSE)\\n\\n[65.2k\\\\\\\\\\nstars](https://github.com/firecrawl/firecrawl/stargazers)
|
||||
[5.1k\\\\\\\\\\nforks](https://github.com/firecrawl/firecrawl/forks) [Branches](https://github.com/firecrawl/firecrawl/branches)
|
||||
[Tags](https://github.com/firecrawl/firecrawl/tags) [Activity](https://github.com/firecrawl/firecrawl/activity)\\n\\n[Star](https://github.com/login?return_to=%2Ffirecrawl%2Ffirecrawl)\\n\\nCouldn't
|
||||
load subscription status.\\nRetry\\n\\n### Uh oh!\\n\\nThere was an error
|
||||
while loading. [Please reload this page](https://github.com/firecrawl/firecrawl).\\n\\n#
|
||||
firecrawl/firecrawl\\n\\nmain\\n\\n[**887** Branches](https://github.com/firecrawl/firecrawl/branches)
|
||||
[**28** Tags](https://github.com/firecrawl/firecrawl/tags)\\n\\n[Go to Branches
|
||||
page](https://github.com/firecrawl/firecrawl/branches)[Go to Tags page](https://github.com/firecrawl/firecrawl/tags)\\n\\nGo
|
||||
to file\\n\\nCode\\n\\nOpen more actions menu\\n\\n## Folders and files\\n\\n|
|
||||
Name | Name | Last commit message | Last commit date |\\n| --- | --- | ---
|
||||
| --- |\\n| ## Latest commit<br>[](https://github.com/amplitudesxd)[amplitudesxd](https://github.com/firecrawl/firecrawl/commits?author=amplitudesxd)<br>[chore:
|
||||
update last scrape rpc (](https://github.com/firecrawl/firecrawl/commit/37de2877fab4bae2de297e37bad3c9bcd49a64bc)
|
||||
[#2339](https://github.com/firecrawl/firecrawl/pull/2339) [)](https://github.com/firecrawl/firecrawl/commit/37de2877fab4bae2de297e37bad3c9bcd49a64bc)<br>success<br>20
|
||||
hours agoOct 27, 2025<br>[37de287](https://github.com/firecrawl/firecrawl/commit/37de2877fab4bae2de297e37bad3c9bcd49a64bc)\_\xB7\_20
|
||||
hours agoOct 27, 2025<br>## History<br>[4,487 Commits](https://github.com/firecrawl/firecrawl/commits/main/)
|
||||
<br>Open commit details<br>[View commit history for this file.](https://github.com/firecrawl/firecrawl/commits/main/)
|
||||
|\\n| [.github](https://github.com/firecrawl/firecrawl/tree/main/.github \\\".github\\\")
|
||||
| [.github](https://github.com/firecrawl/firecrawl/tree/main/.github \\\".github\\\")
|
||||
| [fix(ci): temp disabled prod env tests](https://github.com/firecrawl/firecrawl/commit/42fc149c1ab738da0e15e772817774aa35273f8e
|
||||
\\\"fix(ci): temp disabled prod env tests\\\") | 5 days agoOct 23, 2025 |\\n|
|
||||
[apps](https://github.com/firecrawl/firecrawl/tree/main/apps \\\"apps\\\")
|
||||
| [apps](https://github.com/firecrawl/firecrawl/tree/main/apps \\\"apps\\\")
|
||||
| [chore: update last scrape rpc (](https://github.com/firecrawl/firecrawl/commit/37de2877fab4bae2de297e37bad3c9bcd49a64bc
|
||||
\\\"chore: update last scrape rpc (#2339)\\\") [#2339](https://github.com/firecrawl/firecrawl/pull/2339)
|
||||
[)](https://github.com/firecrawl/firecrawl/commit/37de2877fab4bae2de297e37bad3c9bcd49a64bc
|
||||
\\\"chore: update last scrape rpc (#2339)\\\") | 20 hours agoOct 27, 2025
|
||||
|\\n| [examples](https://github.com/firecrawl/firecrawl/tree/main/examples
|
||||
\\\"examples\\\") | [examples](https://github.com/firecrawl/firecrawl/tree/main/examples
|
||||
\\\"examples\\\") | [Merge pull request](https://github.com/firecrawl/firecrawl/commit/7ad57003b4ad8b230ba8252129e52bafa62dfae9
|
||||
\\\"Merge pull request #2172 from MAVRICK-1/firecrawl-gemini-screenshot-editor
|
||||
\ feat: Add Firecrawl + Gemini 2.5 Flash Image CLI Editor\\\") [#2172](https://github.com/firecrawl/firecrawl/pull/2172)
|
||||
[from MAVRICK-1/firecrawl-gemini-screenshot-e\u2026](https://github.com/firecrawl/firecrawl/commit/7ad57003b4ad8b230ba8252129e52bafa62dfae9
|
||||
\\\"Merge pull request #2172 from MAVRICK-1/firecrawl-gemini-screenshot-editor
|
||||
\ feat: Add Firecrawl + Gemini 2.5 Flash Image CLI Editor\\\") | last monthSep
|
||||
23, 2025 |\\n| [img](https://github.com/firecrawl/firecrawl/tree/main/img
|
||||
\\\"img\\\") | [img](https://github.com/firecrawl/firecrawl/tree/main/img
|
||||
\\\"img\\\") | [updated readme](https://github.com/firecrawl/firecrawl/commit/4f904e774831dc598681d3e998d0e5e15abcec27
|
||||
\\\"updated readme\\\") | 2 months agoAug 18, 2025 |\\n| [.gitattributes](https://github.com/firecrawl/firecrawl/blob/main/.gitattributes
|
||||
\\\".gitattributes\\\") | [.gitattributes](https://github.com/firecrawl/firecrawl/blob/main/.gitattributes
|
||||
\\\".gitattributes\\\") | [Initial commit](https://github.com/firecrawl/firecrawl/commit/a6c2a878119321a196f720cce4195e086f1c6b46
|
||||
\\\"Initial commit\\\") | last yearApr 15, 2024 |\\n| [.gitignore](https://github.com/firecrawl/firecrawl/blob/main/.gitignore
|
||||
\\\".gitignore\\\") | [.gitignore](https://github.com/firecrawl/firecrawl/blob/main/.gitignore
|
||||
\\\".gitignore\\\") | [Nick: init](https://github.com/firecrawl/firecrawl/commit/ab3fa4838458c8303a67dd30fdd75a16b89cc20b
|
||||
\\\"Nick: init\\\") | 3 weeks agoOct 10, 2025 |\\n| [.gitmodules](https://github.com/firecrawl/firecrawl/blob/main/.gitmodules
|
||||
\\\".gitmodules\\\") | [.gitmodules](https://github.com/firecrawl/firecrawl/blob/main/.gitmodules
|
||||
\\\".gitmodules\\\") | [mendableai -> firecrawl](https://github.com/firecrawl/firecrawl/commit/2f3bc4e7a7b1a67a29c06df629f79402ee1aad1b
|
||||
\\\"mendableai -> firecrawl\\\") | 2 months agoAug 18, 2025 |\\n| [CLAUDE.md](https://github.com/firecrawl/firecrawl/blob/main/CLAUDE.md
|
||||
\\\"CLAUDE.md\\\") | [CLAUDE.md](https://github.com/firecrawl/firecrawl/blob/main/CLAUDE.md
|
||||
\\\"CLAUDE.md\\\") | [add claude file](https://github.com/firecrawl/firecrawl/commit/3f0873c788823258a7d9f55d1c8772aed4e1a8de
|
||||
\\\"add claude file\\\") | 2 months agoAug 6, 2025 |\\n| [CONTRIBUTING.md](https://github.com/firecrawl/firecrawl/blob/main/CONTRIBUTING.md
|
||||
\\\"CONTRIBUTING.md\\\") | [CONTRIBUTING.md](https://github.com/firecrawl/firecrawl/blob/main/CONTRIBUTING.md
|
||||
\\\"CONTRIBUTING.md\\\") | [Add Rust to CONTRIBUTING (](https://github.com/firecrawl/firecrawl/commit/f396cb20b54c3c2d7e64882642c5df6310a01002
|
||||
\\\"Add Rust to CONTRIBUTING (#2180)\\\") [#2180](https://github.com/firecrawl/firecrawl/pull/2180)
|
||||
[)](https://github.com/firecrawl/firecrawl/commit/f396cb20b54c3c2d7e64882642c5df6310a01002
|
||||
\\\"Add Rust to CONTRIBUTING (#2180)\\\") | last monthSep 18, 2025 |\\n| [LICENSE](https://github.com/firecrawl/firecrawl/blob/main/LICENSE
|
||||
\\\"LICENSE\\\") | [LICENSE](https://github.com/firecrawl/firecrawl/blob/main/LICENSE
|
||||
\\\"LICENSE\\\") | [Update SDKs to MIT license](https://github.com/firecrawl/firecrawl/commit/afb49e21e7cff595ebad9ce0b7aba13b88f39cf8
|
||||
\\\"Update SDKs to MIT license\\\") | last yearJul 8, 2024 |\\n| [README.md](https://github.com/firecrawl/firecrawl/blob/main/README.md
|
||||
\\\"README.md\\\") | [README.md](https://github.com/firecrawl/firecrawl/blob/main/README.md
|
||||
\\\"README.md\\\") | [Update README.md](https://github.com/firecrawl/firecrawl/commit/a21430e97818d95099bb365be711d9227bd75590
|
||||
\\\"Update README.md\\\") | 3 weeks agoOct 6, 2025 |\\n| [SELF\\\\_HOST.md](https://github.com/firecrawl/firecrawl/blob/main/SELF_HOST.md
|
||||
\\\"SELF_HOST.md\\\") | [SELF\\\\_HOST.md](https://github.com/firecrawl/firecrawl/blob/main/SELF_HOST.md
|
||||
\\\"SELF_HOST.md\\\") | [Allow self-hosted webhook delivery to private IP
|
||||
addresses (](https://github.com/firecrawl/firecrawl/commit/5756b834884d481382ce1f5674836a56b7fee33d
|
||||
\\\"Allow self-hosted webhook delivery to private IP addresses (#2232)\\\")
|
||||
[#2232](https://github.com/firecrawl/firecrawl/pull/2232) [)](https://github.com/firecrawl/firecrawl/commit/5756b834884d481382ce1f5674836a56b7fee33d
|
||||
\\\"Allow self-hosted webhook delivery to private IP addresses (#2232)\\\")
|
||||
| 27 days agoOct 1, 2025 |\\n| [docker-compose.yaml](https://github.com/firecrawl/firecrawl/blob/main/docker-compose.yaml
|
||||
\\\"docker-compose.yaml\\\") | [docker-compose.yaml](https://github.com/firecrawl/firecrawl/blob/main/docker-compose.yaml
|
||||
\\\"docker-compose.yaml\\\") | [Fix a self-hosted docker-compose.yaml bug
|
||||
caused by a recent firecraw\u2026](https://github.com/firecrawl/firecrawl/commit/7d4100b274889977fa1ba26344532d9d8747494c
|
||||
\\\"Fix a self-hosted docker-compose.yaml bug caused by a recent firecrawl
|
||||
change (#2252) Add EXTRACT_WORKER_PORT to docker-compose environment\\\")
|
||||
| 3 weeks agoOct 4, 2025 |\\n| View all files |\\n\\n## Repository files navigation\\n\\n###
|
||||
[](https://raw.githubusercontent.com/firecrawl/firecrawl/main/img/firecrawl_logo.png)\\n\\n[Permalink:
|
||||
](https://github.com/firecrawl/firecrawl#----)\\n\\n[](https://github.com/firecrawl/firecrawl/blob/main/LICENSE)[](https://pepy.tech/project/firecrawl-py)[](https://github.com/firecrawl/firecrawl/graphs/contributors)[](https://firecrawl.dev/)\\n\\n[](https://twitter.com/firecrawl_dev)[](https://www.linkedin.com/company/104100957)[](https://discord.com/invite/gSmWdAkdwd)\\n\\n#
|
||||
\U0001F525 Firecrawl\\n\\n[Permalink: \U0001F525 Firecrawl](https://github.com/firecrawl/firecrawl#-firecrawl)\\n\\nEmpower
|
||||
your AI apps with clean data from any website. Featuring advanced scraping,
|
||||
crawling, and data extraction capabilities.\\n\\n_This repository is in development,
|
||||
and we\u2019re still integrating custom modules into the mono repo. It's not
|
||||
fully ready for self-hosted deployment yet, but you can run it locally._\\n\\n##
|
||||
What is Firecrawl?\\n\\n[Permalink: What is Firecrawl?](https://github.com/firecrawl/firecrawl#what-is-firecrawl)\\n\\n[Firecrawl](https://firecrawl.dev/?ref=github)
|
||||
is an API service that takes a URL, crawls it, and converts it into clean
|
||||
markdown or structured data. We crawl all accessible subpages and give you
|
||||
clean data for each. No sitemap required. Check out our [documentation](https://docs.firecrawl.dev/).\\n\\nLooking
|
||||
for our MCP? Check out the [repo here](https://github.com/firecrawl/firecrawl-mcp-server).\\n\\n_Pst.
|
||||
hey, you, join our stargazers :)_\\n\\n[](https://github.com/firecrawl/firecrawl)\\n\\n##
|
||||
How to use it?\\n\\n[Permalink: How to use it?](https://github.com/firecrawl/firecrawl#how-to-use-it)\\n\\nWe
|
||||
provide an easy to use API with our hosted version. You can find the playground
|
||||
and documentation [here](https://firecrawl.dev/playground). You can also self
|
||||
host the backend if you'd like.\\n\\nCheck out the following resources to
|
||||
get started:\\n\\n- [x] **API**: [Documentation](https://docs.firecrawl.dev/api-reference/introduction)\\n-
|
||||
[x] **SDKs**: [Python](https://docs.firecrawl.dev/sdks/python), [Node](https://docs.firecrawl.dev/sdks/node)\\n-
|
||||
[x] **LLM Frameworks**: [Langchain (python)](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/),
|
||||
[Langchain (js)](https://js.langchain.com/docs/integrations/document_loaders/web_loaders/firecrawl),
|
||||
[Llama Index](https://docs.llamaindex.ai/en/latest/examples/data_connectors/WebPageDemo/#using-firecrawl-reader),
|
||||
[Crew.ai](https://docs.crewai.com/), [Composio](https://composio.dev/tools/firecrawl/all),
|
||||
[PraisonAI](https://docs.praison.ai/firecrawl/), [Superinterface](https://superinterface.ai/docs/assistants/functions/firecrawl),
|
||||
[Vectorize](https://docs.vectorize.io/integrations/source-connectors/firecrawl)\\n-
|
||||
[x] **Low-code Frameworks**: [Dify](https://dify.ai/blog/dify-ai-blog-integrated-with-firecrawl),
|
||||
[Langflow](https://docs.langflow.org/), [Flowise AI](https://docs.flowiseai.com/integrations/langchain/document-loaders/firecrawl),
|
||||
[Cargo](https://docs.getcargo.io/integration/firecrawl), [Pipedream](https://pipedream.com/apps/firecrawl/)\\n-
|
||||
[x] **Community SDKs**: [Go](https://docs.firecrawl.dev/sdks/go), [Rust](https://docs.firecrawl.dev/sdks/rust)\\n-
|
||||
[x] **Others**: [Zapier](https://zapier.com/apps/firecrawl/integrations),
|
||||
[Pabbly Connect](https://www.pabbly.com/connect/integrations/firecrawl/)\\n-
|
||||
[ ] Want an SDK or Integration? Let us know by opening an issue.\\n\\nTo
|
||||
run locally, refer to guide [here](https://github.com/firecrawl/firecrawl/blob/main/CONTRIBUTING.md).\\n\\n###
|
||||
API Key\\n\\n[Permalink: API Key](https://github.com/firecrawl/firecrawl#api-key)\\n\\nTo
|
||||
use the API, you need to sign up on [Firecrawl](https://firecrawl.dev/) and
|
||||
get an API key.\\n\\n### Features\\n\\n[Permalink: Features](https://github.com/firecrawl/firecrawl#features)\\n\\n-
|
||||
[**Scrape**](https://github.com/firecrawl/firecrawl#scraping): scrapes a URL
|
||||
and get its content in LLM-ready format (markdown, structured data via [LLM
|
||||
Extract](https://github.com/firecrawl/firecrawl#llm-extraction-beta), screenshot,
|
||||
html)\\n- [**Crawl**](https://github.com/firecrawl/firecrawl#crawling): scrapes
|
||||
all the URLs of a web page and return content in LLM-ready format\\n- [**Map**](https://github.com/firecrawl/firecrawl#map):
|
||||
input a website and get all the website urls - extremely fast\\n- [**Search**](https://github.com/firecrawl/firecrawl#search):
|
||||
search the web and get full content from results\\n- [**Extract**](https://github.com/firecrawl/firecrawl#extract):
|
||||
get structured data from single page, multiple pages or entire websites with
|
||||
AI.\\n\\n### Powerful Capabilities\\n\\n[Permalink: Powerful Capabilities](https://github.com/firecrawl/firecrawl#powerful-capabilities)\\n\\n-
|
||||
**LLM-ready formats**: markdown, structured data, screenshot, HTML, links,
|
||||
metadata\\n- **The hard stuff**: proxies, anti-bot mechanisms, dynamic content
|
||||
(js-rendered), output parsing, orchestration\\n- **Customizability**: exclude
|
||||
tags, crawl behind auth walls with custom headers, max crawl depth, etc...\\n-
|
||||
**Media parsing**: pdfs, docx, images\\n- **Reliability first**: designed
|
||||
to get the data you need - no matter how hard it is\\n- **Actions**: click,
|
||||
scroll, input, wait and more before extracting data\\n- **Batching**: scrape
|
||||
thousands of URLs at the same time with a new async endpoint\\n- **Change
|
||||
Tracking**: monitor and detect changes in website content over time\\n\\nYou
|
||||
can find all of Firecrawl's capabilities and how to use them in our [documentation](https://docs.firecrawl.dev/)\\n\\n###
|
||||
Crawling\\n\\n[Permalink: Crawling](https://github.com/firecrawl/firecrawl#crawling)\\n\\nUsed
|
||||
to crawl a URL and all accessible subpages. This submits a crawl job and returns
|
||||
a job ID to check the status of the crawl.\\n\\n```\\ncurl -X POST https://api.firecrawl.dev/v2/crawl
|
||||
\\\\\\n -H 'Content-Type: application/json' \\\\\\n -H 'Authorization:
|
||||
Bearer fc-YOUR_API_KEY' \\\\\\n -d '{\\n \\\"url\\\": \\\"https://docs.firecrawl.dev\\\",\\n
|
||||
\ \\\"limit\\\": 10,\\n \\\"scrapeOptions\\\": {\\n \\\"formats\\\":
|
||||
[\\\"markdown\\\", \\\"html\\\"]\\n }\\n }'\\n```\\n\\nReturns a crawl
|
||||
job id and the url to check the status of the crawl.\\n\\n```\\n{\\n \\\"success\\\":
|
||||
true,\\n \\\"id\\\": \\\"123-456-789\\\",\\n \\\"url\\\": \\\"https://api.firecrawl.dev/v2/crawl/123-456-789\\\"\\n}\\n```\\n\\n###
|
||||
Check Crawl Job\\n\\n[Permalink: Check Crawl Job](https://github.com/firecrawl/firecrawl#check-crawl-job)\\n\\nUsed
|
||||
to check the status of a crawl job and get its result.\\n\\n```\\ncurl -X
|
||||
GET https://api.firecrawl.dev/v2/crawl/123-456-789 \\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\n -H 'Authorization: Bearer YOUR_API_KEY'\\n```\\n\\n```\\n{\\n
|
||||
\ \\\"status\\\": \\\"completed\\\",\\n \\\"total\\\": 36,\\n \\\"creditsUsed\\\":
|
||||
36,\\n \\\"expiresAt\\\": \\\"2024-00-00T00:00:00.000Z\\\",\\n \\\"data\\\":
|
||||
[\\\\\\n {\\\\\\n \\\"markdown\\\": \\\"[Firecrawl Docs home page!...\\\",\\\\\\n
|
||||
\ \\\"html\\\": \\\"<!DOCTYPE html><html lang=\\\\\\\"en\\\\\\\" class=\\\\\\\"js-focus-visible
|
||||
lg:[--scroll-mt:9.5rem]\\\\\\\" data-js-focus-visible=\\\\\\\"\\\\\\\">...\\\",\\\\\\n
|
||||
\ \\\"metadata\\\": {\\\\\\n \\\"title\\\": \\\"Build a 'Chat with
|
||||
website' using Groq Llama 3 | Firecrawl\\\",\\\\\\n \\\"language\\\":
|
||||
\\\"en\\\",\\\\\\n \\\"sourceURL\\\": \\\"https://docs.firecrawl.dev/learn/rag-llama3\\\",\\\\\\n
|
||||
\ \\\"description\\\": \\\"Learn how to use Firecrawl, Groq Llama 3,
|
||||
and Langchain to build a 'Chat with your website' bot.\\\",\\\\\\n \\\"ogLocaleAlternate\\\":
|
||||
[],\\\\\\n \\\"statusCode\\\": 200\\\\\\n }\\\\\\n }\\\\\\n
|
||||
\ ]\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n### Scraping\\\\\\n\\\\\\n[Permalink: Scraping](https://github.com/firecrawl/firecrawl#scraping)\\\\\\n\\\\\\nUsed
|
||||
to scrape a URL and get its content in the specified formats.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/scrape \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"url\\\": \\\"https://docs.firecrawl.dev\\\",\\\\\\n
|
||||
\ \\\"formats\\\" : [\\\"markdown\\\", \\\"html\\\"]\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\nResponse:\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"data\\\": {\\\\\\n \\\"markdown\\\":
|
||||
\\\"Launch Week I is here! [See our Day 2 Release \U0001F680](https://www.firecrawl.dev/blog/launch-week-i-day-2-doubled-rate-limits)[\U0001F4A5
|
||||
Get 2 months free...\\\",\\\\\\n \\\"html\\\": \\\"<!DOCTYPE html><html
|
||||
lang=\\\\\\\"en\\\\\\\" class=\\\\\\\"light\\\\\\\" style=\\\\\\\"color-scheme:
|
||||
light;\\\\\\\"><body class=\\\\\\\"__variable_36bd41 __variable_d7dc5d font-inter
|
||||
...\\\",\\\\\\n \\\"metadata\\\": {\\\\\\n \\\"title\\\": \\\"Home
|
||||
- Firecrawl\\\",\\\\\\n \\\"description\\\": \\\"Firecrawl crawls and
|
||||
converts any website into clean markdown.\\\",\\\\\\n \\\"language\\\":
|
||||
\\\"en\\\",\\\\\\n \\\"keywords\\\": \\\"Firecrawl,Markdown,Data,Mendable,Langchain\\\",\\\\\\n
|
||||
\ \\\"robots\\\": \\\"follow, index\\\",\\\\\\n \\\"ogTitle\\\":
|
||||
\\\"Firecrawl\\\",\\\\\\n \\\"ogDescription\\\": \\\"Turn any website
|
||||
into LLM-ready data.\\\",\\\\\\n \\\"ogUrl\\\": \\\"https://www.firecrawl.dev/\\\",\\\\\\n
|
||||
\ \\\"ogImage\\\": \\\"https://www.firecrawl.dev/og.png?123\\\",\\\\\\n
|
||||
\ \\\"ogLocaleAlternate\\\": [],\\\\\\n \\\"ogSiteName\\\": \\\"Firecrawl\\\",\\\\\\n
|
||||
\ \\\"sourceURL\\\": \\\"https://firecrawl.dev\\\",\\\\\\n \\\"statusCode\\\":
|
||||
200\\\\\\n }\\\\\\n }\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n### Map\\\\\\n\\\\\\n[Permalink:
|
||||
Map](https://github.com/firecrawl/firecrawl#map)\\\\\\n\\\\\\nUsed to map
|
||||
a URL and get urls of the website. This returns most links present on the
|
||||
website.\\\\\\n\\\\\\n```\\\\\\ncurl -X POST https://api.firecrawl.dev/v2/map
|
||||
\\\\\\\\\\n -H 'Content-Type: application/json' \\\\\\\\\\n -H 'Authorization:
|
||||
Bearer YOUR_API_KEY' \\\\\\\\\\n -d '{\\\\\\n \\\"url\\\": \\\"https://firecrawl.dev\\\"\\\\\\n
|
||||
\ }'\\\\\\n```\\\\\\n\\\\\\nResponse:\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n \\\"success\\\":
|
||||
true,\\\\\\n \\\"links\\\": [\\\\\\n { \\\"url\\\": \\\"https://firecrawl.dev\\\",
|
||||
\\\"title\\\": \\\"Firecrawl\\\", \\\"description\\\": \\\"Firecrawl is a
|
||||
tool that allows you to crawl a website and get the data you need.\\\" },\\\\\\n
|
||||
\ { \\\"url\\\": \\\"https://www.firecrawl.dev/pricing\\\", \\\"title\\\":
|
||||
\\\"Firecrawl Pricing\\\", \\\"description\\\": \\\"Firecrawl Pricing\\\"
|
||||
},\\\\\\n { \\\"url\\\": \\\"https://www.firecrawl.dev/blog\\\", \\\"title\\\":
|
||||
\\\"Firecrawl Blog\\\", \\\"description\\\": \\\"Firecrawl Blog\\\" },\\\\\\n
|
||||
\ { \\\"url\\\": \\\"https://www.firecrawl.dev/playground\\\", \\\"title\\\":
|
||||
\\\"Firecrawl Playground\\\", \\\"description\\\": \\\"Firecrawl Playground\\\"
|
||||
},\\\\\\n { \\\"url\\\": \\\"https://www.firecrawl.dev/smart-crawl\\\",
|
||||
\\\"title\\\": \\\"Firecrawl Smart Crawl\\\", \\\"description\\\": \\\"Firecrawl
|
||||
Smart Crawl\\\" }\\\\\\n ]\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n#### Map with search\\\\\\n\\\\\\n[Permalink:
|
||||
Map with search](https://github.com/firecrawl/firecrawl#map-with-search)\\\\\\n\\\\\\nMap
|
||||
with `search` param allows you to search for specific urls inside a website.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/map \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"url\\\": \\\"https://firecrawl.dev\\\",\\\\\\n \\\"search\\\":
|
||||
\\\"docs\\\"\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\nResponse will be an ordered
|
||||
list from the most relevant to the least relevant.\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"links\\\": [\\\\\\n { \\\"url\\\":
|
||||
\\\"https://docs.firecrawl.dev\\\", \\\"title\\\": \\\"Firecrawl Docs\\\",
|
||||
\\\"description\\\": \\\"Firecrawl Docs\\\" },\\\\\\n { \\\"url\\\": \\\"https://docs.firecrawl.dev/sdks/python\\\",
|
||||
\\\"title\\\": \\\"Firecrawl Python SDK\\\", \\\"description\\\": \\\"Firecrawl
|
||||
Python SDK\\\" },\\\\\\n { \\\"url\\\": \\\"https://docs.firecrawl.dev/learn/rag-llama3\\\",
|
||||
\\\"title\\\": \\\"Firecrawl RAG Llama 3\\\", \\\"description\\\": \\\"Firecrawl
|
||||
RAG Llama 3\\\" }\\\\\\n ]\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n### Search\\\\\\n\\\\\\n[Permalink:
|
||||
Search](https://github.com/firecrawl/firecrawl#search)\\\\\\n\\\\\\nSearch
|
||||
the web and get full content from results\\\\\\n\\\\\\nFirecrawl\u2019s search
|
||||
API allows you to perform web searches and optionally scrape the search results
|
||||
in one operation.\\\\\\n\\\\\\n- Choose specific output formats (markdown,
|
||||
HTML, links, screenshots)\\\\\\n- Search the web with customizable parameters
|
||||
(language, country, etc.)\\\\\\n- Optionally retrieve content from search
|
||||
results in various formats\\\\\\n- Control the number of results and set timeouts\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/search \\\\\\\\\\n -H \\\"Content-Type:
|
||||
application/json\\\" \\\\\\\\\\n -H \\\"Authorization: Bearer fc-YOUR_API_KEY\\\"
|
||||
\\\\\\\\\\n -d '{\\\\\\n \\\"query\\\": \\\"what is firecrawl?\\\",\\\\\\n
|
||||
\ \\\"limit\\\": 5\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n#### Response\\\\\\n\\\\\\n[Permalink:
|
||||
Response](https://github.com/firecrawl/firecrawl#response)\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"data\\\": [\\\\\\n {\\\\\\n \\\"url\\\":
|
||||
\\\"https://firecrawl.dev\\\",\\\\\\n \\\"title\\\": \\\"Firecrawl |
|
||||
Home Page\\\",\\\\\\n \\\"description\\\": \\\"Turn websites into LLM-ready
|
||||
data with Firecrawl\\\"\\\\\\n },\\\\\\n {\\\\\\n \\\"url\\\":
|
||||
\\\"https://docs.firecrawl.dev\\\",\\\\\\n \\\"title\\\": \\\"Documentation
|
||||
| Firecrawl\\\",\\\\\\n \\\"description\\\": \\\"Learn how to use Firecrawl
|
||||
in your own applications\\\"\\\\\\n }\\\\\\n ]\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n####
|
||||
With content scraping\\\\\\n\\\\\\n[Permalink: With content scraping](https://github.com/firecrawl/firecrawl#with-content-scraping)\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/search \\\\\\\\\\n -H \\\"Content-Type:
|
||||
application/json\\\" \\\\\\\\\\n -H \\\"Authorization: Bearer fc-YOUR_API_KEY\\\"
|
||||
\\\\\\\\\\n -d '{\\\\\\n \\\"query\\\": \\\"what is firecrawl?\\\",\\\\\\n
|
||||
\ \\\"limit\\\": 5,\\\\\\n \\\"scrapeOptions\\\": {\\\\\\n \\\"formats\\\":
|
||||
[\\\"markdown\\\", \\\"links\\\"]\\\\\\n }\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n###
|
||||
Extract (Beta)\\\\\\n\\\\\\n[Permalink: Extract (Beta)](https://github.com/firecrawl/firecrawl#extract-beta)\\\\\\n\\\\\\nGet
|
||||
structured data from entire websites with a prompt and/or a schema.\\\\\\n\\\\\\nYou
|
||||
can extract structured data from one or multiple URLs, including wildcards:\\\\\\n\\\\\\nSingle
|
||||
Page:\\\\\\nExample: [https://firecrawl.dev/some-page](https://firecrawl.dev/some-page)\\\\\\n\\\\\\nMultiple
|
||||
Pages / Full Domain\\\\\\nExample: [https://firecrawl.dev/](https://firecrawl.dev/)\\\\*\\\\\\n\\\\\\nWhen
|
||||
you use /\\\\*, Firecrawl will automatically crawl and parse all URLs it can
|
||||
discover in that domain, then extract the requested data.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/extract \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"urls\\\": [\\\\\\n \\\"https://firecrawl.dev/*\\\",\\\\\\n
|
||||
\ \\\"https://docs.firecrawl.dev/\\\",\\\\\\n \\\"https://www.ycombinator.com/companies\\\"\\\\\\n
|
||||
\ ],\\\\\\n \\\"prompt\\\": \\\"Extract the company mission, whether
|
||||
it is open source, and whether it is in Y Combinator from the page.\\\",\\\\\\n
|
||||
\ \\\"schema\\\": {\\\\\\n \\\"type\\\": \\\"object\\\",\\\\\\n
|
||||
\ \\\"properties\\\": {\\\\\\n \\\"company_mission\\\": {\\\\\\n
|
||||
\ \\\"type\\\": \\\"string\\\"\\\\\\n },\\\\\\n \\\"is_open_source\\\":
|
||||
{\\\\\\n \\\"type\\\": \\\"boolean\\\"\\\\\\n },\\\\\\n
|
||||
\ \\\"is_in_yc\\\": {\\\\\\n \\\"type\\\": \\\"boolean\\\"\\\\\\n
|
||||
\ }\\\\\\n },\\\\\\n \\\"required\\\": [\\\\\\n \\\"company_mission\\\",\\\\\\n
|
||||
\ \\\"is_open_source\\\",\\\\\\n \\\"is_in_yc\\\"\\\\\\n
|
||||
\ ]\\\\\\n }\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"id\\\": \\\"44aa536d-f1cb-4706-ab87-ed0386685740\\\",\\\\\\n
|
||||
\ \\\"urlTrace\\\": []\\\\\\n}\\\\\\n```\\\\\\n\\\\\\nIf you are using the
|
||||
sdks, it will auto pull the response for you:\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"data\\\": {\\\\\\n \\\"company_mission\\\":
|
||||
\\\"Firecrawl is the easiest way to extract data from the web. Developers
|
||||
use us to reliably convert URLs into LLM-ready markdown or structured data
|
||||
with a single API call.\\\",\\\\\\n \\\"supports_sso\\\": false,\\\\\\n
|
||||
\ \\\"is_open_source\\\": true,\\\\\\n \\\"is_in_yc\\\": true\\\\\\n
|
||||
\ }\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n### LLM Extraction (Beta)\\\\\\n\\\\\\n[Permalink:
|
||||
LLM Extraction (Beta)](https://github.com/firecrawl/firecrawl#llm-extraction-beta)\\\\\\n\\\\\\nUsed
|
||||
to extract structured data from scraped pages.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/scrape \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"url\\\": \\\"https://www.mendable.ai/\\\",\\\\\\n \\\"formats\\\":
|
||||
[\\\\\\n {\\\\\\n \\\"type\\\": \\\"json\\\",\\\\\\n \\\"schema\\\":
|
||||
{\\\\\\n \\\"type\\\": \\\"object\\\",\\\\\\n \\\"properties\\\":
|
||||
{\\\\\\n \\\"company_mission\\\": { \\\"type\\\": \\\"string\\\"
|
||||
},\\\\\\n \\\"supports_sso\\\": { \\\"type\\\": \\\"boolean\\\"
|
||||
},\\\\\\n \\\"is_open_source\\\": { \\\"type\\\": \\\"boolean\\\"
|
||||
},\\\\\\n \\\"is_in_yc\\\": { \\\"type\\\": \\\"boolean\\\" }\\\\\\n
|
||||
\ }\\\\\\n }\\\\\\n }\\\\\\n ]\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n```\\\\\\n{\\\\\\n
|
||||
\ \\\"success\\\": true,\\\\\\n \\\"data\\\": {\\\\\\n \\\"content\\\":
|
||||
\\\"Raw Content\\\",\\\\\\n \\\"metadata\\\": {\\\\\\n \\\"title\\\":
|
||||
\\\"Mendable\\\",\\\\\\n \\\"description\\\": \\\"Mendable allows you
|
||||
to easily build AI chat applications. Ingest, customize, then deploy with
|
||||
one line of code anywhere you want. Brought to you by SideGuide\\\",\\\\\\n
|
||||
\ \\\"robots\\\": \\\"follow, index\\\",\\\\\\n \\\"ogTitle\\\":
|
||||
\\\"Mendable\\\",\\\\\\n \\\"ogDescription\\\": \\\"Mendable allows you
|
||||
to easily build AI chat applications. Ingest, customize, then deploy with
|
||||
one line of code anywhere you want. Brought to you by SideGuide\\\",\\\\\\n
|
||||
\ \\\"ogUrl\\\": \\\"https://mendable.ai/\\\",\\\\\\n \\\"ogImage\\\":
|
||||
\\\"https://mendable.ai/mendable_new_og1.png\\\",\\\\\\n \\\"ogLocaleAlternate\\\":
|
||||
[],\\\\\\n \\\"ogSiteName\\\": \\\"Mendable\\\",\\\\\\n \\\"sourceURL\\\":
|
||||
\\\"https://mendable.ai/\\\"\\\\\\n },\\\\\\n \\\"json\\\": {\\\\\\n
|
||||
\ \\\"company_mission\\\": \\\"Train a secure AI on your technical resources
|
||||
that answers customer and employee questions so your team doesn't have to\\\",\\\\\\n
|
||||
\ \\\"supports_sso\\\": true,\\\\\\n \\\"is_open_source\\\": false,\\\\\\n
|
||||
\ \\\"is_in_yc\\\": true\\\\\\n }\\\\\\n }\\\\\\n}\\\\\\n```\\\\\\n\\\\\\n###
|
||||
Extracting without a schema (New)\\\\\\n\\\\\\n[Permalink: Extracting without
|
||||
a schema (New)](https://github.com/firecrawl/firecrawl#extracting-without-a-schema-new)\\\\\\n\\\\\\nYou
|
||||
can now extract without a schema by just passing a `prompt` to the endpoint.
|
||||
The llm chooses the structure of the data.\\\\\\n\\\\\\n```\\\\\\ncurl -X
|
||||
POST https://api.firecrawl.dev/v2/scrape \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"url\\\": \\\"https://docs.firecrawl.dev/\\\",\\\\\\n
|
||||
\ \\\"formats\\\": [\\\\\\n {\\\\\\n \\\"type\\\": \\\"json\\\",\\\\\\n
|
||||
\ \\\"prompt\\\": \\\"Extract the company mission from the page.\\\"\\\\\\n
|
||||
\ }\\\\\\n ]\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n### Interacting
|
||||
with the page with Actions (Cloud-only)\\\\\\n\\\\\\n[Permalink: Interacting
|
||||
with the page with Actions (Cloud-only)](https://github.com/firecrawl/firecrawl#interacting-with-the-page-with-actions-cloud-only)\\\\\\n\\\\\\nFirecrawl
|
||||
allows you to perform various actions on a web page before scraping its content.
|
||||
This is particularly useful for interacting with dynamic content, navigating
|
||||
through pages, or accessing content that requires user interaction.\\\\\\n\\\\\\nHere
|
||||
is an example of how to use actions to navigate to google.com, search for
|
||||
Firecrawl, click on the first result, and take a screenshot.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/scrape \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"url\\\": \\\"google.com\\\",\\\\\\n \\\"formats\\\":
|
||||
[\\\"markdown\\\"],\\\\\\n \\\"actions\\\": [\\\\\\n {\\\"type\\\":
|
||||
\\\"wait\\\", \\\"milliseconds\\\": 2000},\\\\\\n {\\\"type\\\":
|
||||
\\\"click\\\", \\\"selector\\\": \\\"textarea[title=\\\\\\\"Search\\\\\\\"]\\\"},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"wait\\\", \\\"milliseconds\\\": 2000},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"write\\\", \\\"text\\\": \\\"firecrawl\\\"},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"wait\\\", \\\"milliseconds\\\": 2000},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"press\\\", \\\"key\\\": \\\"ENTER\\\"},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"wait\\\", \\\"milliseconds\\\": 3000},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"click\\\", \\\"selector\\\": \\\"h3\\\"},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"wait\\\", \\\"milliseconds\\\": 3000},\\\\\\n
|
||||
\ {\\\"type\\\": \\\"screenshot\\\"}\\\\\\n ]\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n###
|
||||
Batch Scraping Multiple URLs (New)\\\\\\n\\\\\\n[Permalink: Batch Scraping
|
||||
Multiple URLs (New)](https://github.com/firecrawl/firecrawl#batch-scraping-multiple-urls-new)\\\\\\n\\\\\\nYou
|
||||
can now batch scrape multiple URLs at the same time. It is very similar to
|
||||
how the /crawl endpoint works. It submits a batch scrape job and returns a
|
||||
job ID to check the status of the batch scrape.\\\\\\n\\\\\\n```\\\\\\ncurl
|
||||
-X POST https://api.firecrawl.dev/v2/batch/scrape \\\\\\\\\\n -H 'Content-Type:
|
||||
application/json' \\\\\\\\\\n -H 'Authorization: Bearer YOUR_API_KEY' \\\\\\\\\\n
|
||||
\ -d '{\\\\\\n \\\"urls\\\": [\\\"https://docs.firecrawl.dev\\\", \\\"https://docs.firecrawl.dev/sdks/overview\\\"],\\\\\\n
|
||||
\ \\\"formats\\\" : [\\\"markdown\\\", \\\"html\\\"]\\\\\\n }'\\\\\\n```\\\\\\n\\\\\\n##
|
||||
Using Python SDK\\\\\\n\\\\\\n[Permalink: Using Python SDK](https://github.com/firecrawl/firecrawl#using-python-sdk)\\\\\\n\\\\\\n###
|
||||
Installing Python SDK\\\\\\n\\\\\\n[Permalink: Installing Python SDK](https://github.com/firecrawl/firecrawl#installing-python-sdk)\\\\\\n\\\\\\n```\\\\\\npip
|
||||
install firecrawl-py\\\\\\n```\\\\\\n\\\\\\n### Crawl a website\\\\\\n\\\\\\n[Permalink:
|
||||
Crawl a website](https://github.com/firecrawl/firecrawl#crawl-a-website)\\\\\\n\\\\\\n```\\\\\\nfrom
|
||||
firecrawl import Firecrawl\\\\\\n\\\\\\nfirecrawl = Firecrawl(api_key=\\\"fc-YOUR_API_KEY\\\")\\\\\\n\\\\\\n#
|
||||
Scrape a website (returns a Document)\\\\\\ndoc = firecrawl.scrape(\\\\\\n
|
||||
\ \\\"https://firecrawl.dev\\\",\\\\\\n formats=[\\\"markdown\\\", \\\"html\\\"],\\\\\\n)\\\\\\nprint(doc.markdown)\\\\\\n\\\\\\n#
|
||||
Crawl a website\\\\\\nresponse = firecrawl.crawl(\\\\\\n \\\"https://firecrawl.dev\\\",\\\\\\n
|
||||
\ limit=100,\\\\\\n scrape_options={\\\"formats\\\": [\\\"markdown\\\",
|
||||
\\\"html\\\"]},\\\\\\n poll_interval=30,\\\\\\n)\\\\\\nprint(response)\\\\\\n```\\\\\\n\\\\\\n###
|
||||
Extracting structured data from a URL\\\\\\n\\\\\\n[Permalink: Extracting
|
||||
structured data from a URL](https://github.com/firecrawl/firecrawl#extracting-structured-data-from-a-url)\\\\\\n\\\\\\nWith
|
||||
LLM extraction, you can easily extract structured data from any URL. We support
|
||||
pydantic schemas to make it easier for you too. Here is how you to use it:\\\\\\n\\\\\\n```\\\\\\nfrom
|
||||
pydantic import BaseModel, Field\\\\\\nfrom typing import List\\\\\\n\\\\\\nclass
|
||||
Article(BaseModel):\\\\\\n title: str\\\\\\n points: int\\\\\\n by:
|
||||
str\\\\\\n commentsURL: str\\\\\\n\\\\\\nclass TopArticles(BaseModel):\\\\\\n
|
||||
\ top: List[Article] = Field(..., description=\\\"Top 5 stories\\\")\\\\\\n\\\\\\n#
|
||||
Use JSON format with a Pydantic schema\\\\\\ndoc = firecrawl.scrape(\\\\\\n
|
||||
\ \\\"https://news.ycombinator.com\\\",\\\\\\n formats=[{\\\"type\\\":
|
||||
\\\"json\\\", \\\"schema\\\": TopArticles}],\\\\\\n)\\\\\\nprint(doc.json)\\\\\\n```\\\\\\n\\\\\\n##
|
||||
Using the Node SDK\\\\\\n\\\\\\n[Permalink: Using the Node SDK](https://github.com/firecrawl/firecrawl#using-the-node-sdk)\\\\\\n\\\\\\n###
|
||||
Installation\\\\\\n\\\\\\n[Permalink: Installation](https://github.com/firecrawl/firecrawl#installation)\\\\\\n\\\\\\nTo
|
||||
install the Firecrawl Node SDK, you can use npm:\\\\\\n\\\\\\n```\\\\\\nnpm
|
||||
install @mendable/firecrawl-js\\\\\\n```\\\\\\n\\\\\\n### Usage\\\\\\n\\\\\\n[Permalink:
|
||||
Usage](https://github.com/firecrawl/firecrawl#usage)\\\\\\n\\\\\\n1. Get an
|
||||
API key from [firecrawl.dev](https://firecrawl.dev/)\\\\\\n2. Set the API
|
||||
key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter
|
||||
to the `Firecrawl` class.\\\\\\n\\\\\\n```\\\\\\nimport Firecrawl from '@mendable/firecrawl-js';\\\\\\n\\\\\\nconst
|
||||
firecrawl = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });\\\\\\n\\\\\\n//
|
||||
Scrape a website\\\\\\nconst doc = await firecrawl.scrape('https://firecrawl.dev',
|
||||
{\\\\\\n formats: ['markdown', 'html'],\\\\\\n});\\\\\\nconsole.log(doc);\\\\\\n\\\\\\n//
|
||||
Crawl a website\\\\\\nconst response = await firecrawl.crawl('https://firecrawl.dev',
|
||||
{\\\\\\n limit: 100,\\\\\\n scrapeOptions: { formats: ['markdown', 'html']
|
||||
},\\\\\\n});\\\\\\nconsole.log(response);\\\\\\n```\\\\\\n\\\\\\n### Extracting
|
||||
structured data from a URL\\\\\\n\\\\\\n[Permalink: Extracting structured
|
||||
data from a URL](https://github.com/firecrawl/firecrawl#extracting-structured-data-from-a-url-1)\\\\\\n\\\\\\nWith
|
||||
LLM extraction, you can easily extract structured data from any URL. We support
|
||||
zod schema to make it easier for you too. Here is how to use it:\\\\\\n\\\\\\n```\\\\\\nimport
|
||||
Firecrawl from '@mendable/firecrawl-js';\\\\\\nimport { z } from 'zod';\\\\\\n\\\\\\nconst
|
||||
firecrawl = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });\\\\\\n\\\\\\n//
|
||||
Define schema to extract contents into\\\\\\nconst schema = z.object({\\\\\\n
|
||||
\ top: z\\\\\\n .array(\\\\\\n z.object({\\\\\\n title: z.string(),\\\\\\n
|
||||
\ points: z.number(),\\\\\\n by: z.string(),\\\\\\n commentsURL:
|
||||
z.string(),\\\\\\n })\\\\\\n )\\\\\\n .length(5)\\\\\\n .describe('Top
|
||||
5 stories on Hacker News'),\\\\\\n});\\\\\\n\\\\\\n// Use the v2 extract API
|
||||
with direct Zod schema support\\\\\\nconst extractRes = await firecrawl.extract({\\\\\\n
|
||||
\ urls: ['https://news.ycombinator.com'],\\\\\\n schema,\\\\\\n prompt:
|
||||
'Extract the top 5 stories',\\\\\\n});\\\\\\n\\\\\\nconsole.log(extractRes);\\\\\\n```\\\\\\n\\\\\\n##
|
||||
Open Source vs Cloud Offering\\\\\\n\\\\\\n[Permalink: Open Source vs Cloud
|
||||
Offering](https://github.com/firecrawl/firecrawl#open-source-vs-cloud-offering)\\\\\\n\\\\\\nFirecrawl
|
||||
is open source available under the AGPL-3.0 license.\\\\\\n\\\\\\nTo deliver
|
||||
the best possible product, we offer a hosted version of Firecrawl alongside
|
||||
our open-source offering. The cloud solution allows us to continuously innovate
|
||||
and maintain a high-quality, sustainable service for all users.\\\\\\n\\\\\\nFirecrawl
|
||||
Cloud is available at [firecrawl.dev](https://firecrawl.dev/) and offers a
|
||||
range of features that are not available in the open source version:\\\\\\n\\\\\\n[](https://raw.githubusercontent.com/firecrawl/firecrawl/main/img/open-source-cloud.png)\\\\\\n\\\\\\n##
|
||||
Contributing\\\\\\n\\\\\\n[Permalink: Contributing](https://github.com/firecrawl/firecrawl#contributing)\\\\\\n\\\\\\nWe
|
||||
love contributions! Please read our [contributing guide](https://github.com/firecrawl/firecrawl/blob/main/CONTRIBUTING.md)
|
||||
before submitting a pull request. If you'd like to self-host, refer to the
|
||||
[self-hosting guide](https://github.com/firecrawl/firecrawl/blob/main/SELF_HOST.md).\\\\\\n\\\\\\n_It
|
||||
is the sole responsibility of the end users to respect websites' policies
|
||||
when scraping, searching and crawling with Firecrawl. Users are advised to
|
||||
adhere to the applicable privacy policies and terms of use of the websites
|
||||
prior to initiating any scraping activities. By default, Firecrawl respects
|
||||
the directives specified in the websites' robots.txt files when crawling.
|
||||
By utilizing Firecrawl, you expressly agree to comply with these conditions._\\\\\\n\\\\\\n##
|
||||
Contributors\\\\\\n\\\\\\n[Permalink: Contributors](https://github.com/firecrawl/firecrawl#contributors)\\\\\\n\\\\\\n[](https://github.com/firecrawl/firecrawl/graphs/contributors)\\\\\\n\\\\\\n##
|
||||
License Disclaimer\\\\\\n\\\\\\n[Permalink: License Disclaimer](https://github.com/firecrawl/firecrawl#license-disclaimer)\\\\\\n\\\\\\nThis
|
||||
project is primarily licensed under the GNU Affero General Public License
|
||||
v3.0 (AGPL-3.0), as specified in the LICENSE file in the root directory of
|
||||
this repository. However, certain components of this project are licensed
|
||||
under the MIT License. Refer to the LICENSE files in these specific directories
|
||||
for details.\\\\\\n\\\\\\nPlease note:\\\\\\n\\\\\\n- The AGPL-3.0 license
|
||||
applies to all parts of the project unless otherwise specified.\\\\\\n- The
|
||||
SDKs and some UI components are licensed under the MIT License. Refer to the
|
||||
LICENSE files in these specific directories for details.\\\\\\n- When using
|
||||
or contributing to this project, ensure you comply with the appropriate license
|
||||
terms for the specific component you are working with.\\\\\\n\\\\\\nFor more
|
||||
details on the licensing of specific components, please refer to the LICENSE
|
||||
files in the respective directories or contact the project maintainers.\\\\\\n\\\\\\n[\u2191
|
||||
Back to Top \u2191](https://github.com/firecrawl/firecrawl#readme-top)\\\\\\n\\\\\\n##
|
||||
About\\\\\\n\\\\\\n\U0001F525 The Web Data API for AI - Turn entire websites
|
||||
into LLM-ready markdown or structured data\\\\\\n\\\\\\n\\\\\\n[firecrawl.dev](https://firecrawl.dev/
|
||||
\\\"https://firecrawl.dev\\\")\\\\\\n\\\\\\n### Topics\\\\\\n\\\\\\n[markdown](https://github.com/topics/markdown
|
||||
\\\"Topic: markdown\\\") [crawler](https://github.com/topics/crawler \\\"Topic:
|
||||
crawler\\\") [scraper](https://github.com/topics/scraper \\\"Topic: scraper\\\")
|
||||
[ai](https://github.com/topics/ai \\\"Topic: ai\\\") [html-to-markdown](https://github.com/topics/html-to-markdown
|
||||
\\\"Topic: html-to-markdown\\\") [web-crawler](https://github.com/topics/web-crawler
|
||||
\\\"Topic: web-crawler\\\") [scraping](https://github.com/topics/scraping
|
||||
\\\"Topic: scraping\\\") [web-scraper](https://github.com/topics/web-scraper
|
||||
\\\"Topic: web-scraper\\\") [web-scraping](https://github.com/topics/web-scraping
|
||||
\\\"Topic: web-scraping\\\") [data-extraction](https://github.com/topics/data-extraction
|
||||
\\\"Topic: data-extraction\\\") [webscraping](https://github.com/topics/webscraping
|
||||
\\\"Topic: webscraping\\\") [web-data-extraction](https://github.com/topics/web-data-extraction
|
||||
\\\"Topic: web-data-extraction\\\") [ai-agents](https://github.com/topics/ai-agents
|
||||
\\\"Topic: ai-agents\\\") [web-search](https://github.com/topics/web-search
|
||||
\\\"Topic: web-search\\\") [ai-search](https://github.com/topics/ai-search
|
||||
\\\"Topic: ai-search\\\") [web-data](https://github.com/topics/web-data \\\"Topic:
|
||||
web-data\\\") [llm](https://github.com/topics/llm \\\"Topic: llm\\\") [ai-crawler](https://github.com/topics/ai-crawler
|
||||
\\\"Topic: ai-crawler\\\") [ai-scraping](https://github.com/topics/ai-scraping
|
||||
\\\"Topic: ai-scraping\\\")\\\\\\n\\\\\\n### Resources\\\\\\n\\\\\\n[Readme](https://github.com/firecrawl/firecrawl#readme-ov-file)\\\\\\n\\\\\\n###
|
||||
License\\\\\\n\\\\\\n[AGPL-3.0 license](https://github.com/firecrawl/firecrawl#AGPL-3.0-1-ov-file)\\\\\\n\\\\\\n###
|
||||
Contributing\\\\\\n\\\\\\n[Contributing](https://github.com/firecrawl/firecrawl#contributing-ov-file)\\\\\\n\\\\\\n###
|
||||
Uh oh!\\\\\\n\\\\\\nThere was an error while loading. [Please reload this
|
||||
page](https://github.com/firecrawl/firecrawl).\\\\\\n\\\\\\n[Activity](https://github.com/firecrawl/firecrawl/activity)\\\\\\n\\\\\\n[Custom
|
||||
properties](https://github.com/firecrawl/firecrawl/custom-properties)\\\\\\n\\\\\\n###
|
||||
Stars\\\\\\n\\\\\\n[**65.2k**\\\\\\\\\\nstars](https://github.com/firecrawl/firecrawl/stargazers)\\\\\\n\\\\\\n###
|
||||
Watchers\\\\\\n\\\\\\n[**256**\\\\\\\\\\nwatching](https://github.com/firecrawl/firecrawl/watchers)\\\\\\n\\\\\\n###
|
||||
Forks\\\\\\n\\\\\\n[**5.1k**\\\\\\\\\\nforks](https://github.com/firecrawl/firecrawl/forks)\\\\\\n\\\\\\n[Report
|
||||
repository](https://github.com/contact/report-content?content_url=https%3A%2F%2Fgithub.com%2Ffirecrawl%2Ffirecrawl&report=firecrawl+%28user%29)\\\\\\n\\\\\\n##
|
||||
[Releases\\\\ 28](https://github.com/firecrawl/firecrawl/releases)\\\\\\n\\\\\\n[v2.4.0\\\\\\\\\\nLatest\\\\\\\\\\n\\\\\\\\\\n2
|
||||
weeks agoOct 13, 2025](https://github.com/firecrawl/firecrawl/releases/tag/v2.4.0)\\\\\\n\\\\\\n[\\\\+
|
||||
27 releases](https://github.com/firecrawl/firecrawl/releases)\\\\\\n\\\\\\n##
|
||||
[Packages\\\\ 3](https://github.com/orgs/firecrawl/packages?repo_name=firecrawl)\\\\\\n\\\\\\n-
|
||||
[firecrawl](https://github.com/orgs/firecrawl/packages/container/package/firecrawl)\\\\\\n-
|
||||
[playwright-service](https://github.com/orgs/firecrawl/packages/container/package/playwright-service)\\\\\\n-
|
||||
[nuq-postgres](https://github.com/orgs/firecrawl/packages/container/package/nuq-postgres)\\\\\\n\\\\\\n##
|
||||
[Contributors\\\\ 121](https://github.com/firecrawl/firecrawl/graphs/contributors)\\\\\\n\\\\\\n[\\\\+
|
||||
107 contributors](https://github.com/firecrawl/firecrawl/graphs/contributors)\\\\\\n\\\\\\n##
|
||||
Languages\\\\\\n\\\\\\n- [TypeScript73.5%](https://github.com/firecrawl/firecrawl/search?l=typescript)\\\\\\n-
|
||||
[Python18.9%](https://github.com/firecrawl/firecrawl/search?l=python)\\\\\\n-
|
||||
[Rust6.0%](https://github.com/firecrawl/firecrawl/search?l=rust)\\\\\\n- [Astro0.6%](https://github.com/firecrawl/firecrawl/search?l=astro)\\\\\\n-
|
||||
[JavaScript0.3%](https://github.com/firecrawl/firecrawl/search?l=javascript)\\\\\\n-
|
||||
[Jupyter Notebook0.2%](https://github.com/firecrawl/firecrawl/search?l=jupyter-notebook)\\\\\\n-
|
||||
Other0.5%\",\"metadata\":{\"octolytics-dimension-repository_network_root_id\":\"787076358\",\"visitor-hmac\":\"163b2538b2335f7d4000a770785477e15881e1101708ca5ff1e8c021095f6ca1\",\"og:type\":\"object\",\"language\":\"en\",\"route-action\":\"disambiguate\",\"og:title\":\"GitHub
|
||||
- firecrawl/firecrawl: \U0001F525 The Web Data API for AI - Turn entire websites
|
||||
into LLM-ready markdown or structured data\",\"octolytics-dimension-repository_public\":\"true\",\"octolytics-dimension-repository_network_root_nwo\":\"firecrawl/firecrawl\",\"browser-errors-url\":\"https://api.github.com/_private/browser/errors\",\"browser-stats-url\":\"https://api.github.com/_private/browser/stats\",\"twitter:title\":\"GitHub
|
||||
- firecrawl/firecrawl: \U0001F525 The Web Data API for AI - Turn entire websites
|
||||
into LLM-ready markdown or structured data\",\"ui-target\":\"full\",\"og:image\":\"https://repository-images.githubusercontent.com/787076358/f9616c09-3701-41ef-b5a6-fdf912ffb15b\",\"google-site-verification\":\"Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I\",\"ogSiteName\":\"GitHub\",\"route-pattern\":\"/:user_id/:repository\",\"visitor-payload\":\"eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJBMTVGOjE3OUI0RDo2MzdFQUREOjg3MzEwOTM6NjkwMTE4MjUiLCJ2aXNpdG9yX2lkIjoiNDkyNzk2MzExNjg5OTIxMTMwMSIsInJlZ2lvbl9lZGdlIjoiaWFkIiwicmVnaW9uX3JlbmRlciI6ImlhZCJ9\",\"og:description\":\"\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data - firecrawl/firecrawl\",\"expected-hostname\":\"github.com\",\"release\":\"66136a30a16cc69206f1249b6ba072daa2174535\",\"title\":\"GitHub
|
||||
- firecrawl/firecrawl: \U0001F525 The Web Data API for AI - Turn entire websites
|
||||
into LLM-ready markdown or structured data\",\"ogDescription\":\"\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data - firecrawl/firecrawl\",\"twitter:card\":\"summary_large_image\",\"fb:app_id\":\"1401488693436528\",\"color-scheme\":\"light
|
||||
dark\",\"twitter:description\":\"\U0001F525 The Web Data API for AI - Turn
|
||||
entire websites into LLM-ready markdown or structured data - firecrawl/firecrawl\",\"favicon\":\"https://github.githubassets.com/favicons/favicon.svg\",\"viewport\":\"width=device-width\",\"twitter:image\":\"https://repository-images.githubusercontent.com/787076358/f9616c09-3701-41ef-b5a6-fdf912ffb15b\",\"user-login\":\"\",\"description\":\"\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data - firecrawl/firecrawl\",\"octolytics-dimension-repository_nwo\":\"firecrawl/firecrawl\",\"octolytics-dimension-user_id\":\"135057108\",\"twitter:site\":\"@github\",\"og:url\":\"https://github.com/firecrawl/firecrawl\",\"octolytics-dimension-user_login\":\"firecrawl\",\"hostname\":\"github.com\",\"current-catalog-service-hash\":\"f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb\",\"html-safe-nonce\":\"e5653da3800db7de2c8b4a64ff6367242043a9e452608e9a6941a1c9e8346cfc\",\"apple-itunes-app\":\"app-id=1477376905,
|
||||
app-argument=https://github.com/firecrawl/firecrawl\",\"turbo-cache-control\":\"no-preview\",\"og:site_name\":\"GitHub\",\"request-id\":\"A15F:179B4D:637EADD:8731093:69011825\",\"octolytics-url\":\"https://collector.github.com/github/collect\",\"octolytics-dimension-repository_is_fork\":\"false\",\"fetch-nonce\":\"v2:bc80e85b-edaf-e746-9f2c-ffdc75854b07\",\"og:image:alt\":\"\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data - firecrawl/firecrawl\",\"github-keyboard-shortcuts\":\"repository,copilot\",\"ogTitle\":\"GitHub
|
||||
- firecrawl/firecrawl: \U0001F525 The Web Data API for AI - Turn entire websites
|
||||
into LLM-ready markdown or structured data\",\"ogImage\":\"https://repository-images.githubusercontent.com/787076358/f9616c09-3701-41ef-b5a6-fdf912ffb15b\",\"analytics-location\":\"/<user-name>/<repo-name>\",\"route-controller\":\"files\",\"octolytics-dimension-repository_id\":\"787076358\",\"ogUrl\":\"https://github.com/firecrawl/firecrawl\",\"go-import\":\"github.com/firecrawl/firecrawl
|
||||
git https://github.com/firecrawl/firecrawl.git\",\"hovercard-subject-tag\":\"repository:787076358\",\"theme-color\":\"#1e2327\",\"turbo-body-classes\":\"logged-out
|
||||
env-production page-responsive\",\"scrapeId\":\"ec4d99a0-4c4f-4d1a-9fd2-08b8f891f883\",\"sourceURL\":\"https://github.com/firecrawl/firecrawl\",\"url\":\"https://github.com/firecrawl/firecrawl\",\"statusCode\":200,\"contentType\":\"text/html;
|
||||
charset=utf-8\",\"proxyUsed\":\"basic\",\"cacheState\":\"hit\",\"cachedAt\":\"2025-10-28T19:23:20.106Z\"}},{\"url\":\"https://x.com/firecrawl_dev?lang=en\",\"title\":\"Firecrawl
|
||||
(@firecrawl_dev) / Posts / X\",\"description\":\"Firecrawl (@firecrawl_dev)
|
||||
- Posts - Turn websites into LLM-ready data. Built by @mendableai team Open
|
||||
source: | X (formerly Twitter)\",\"position\":3},{\"url\":\"https://github.com/firecrawl\",\"title\":\"Firecrawl
|
||||
- GitHub\",\"description\":\"Building AI applications? You need clean, structured
|
||||
data from the web. Firecrawl handles the complexity of modern web scraping
|
||||
so you can focus on building ...\",\"position\":4,\"category\":\"github\",\"markdown\":\"[Skip
|
||||
to content](https://github.com/firecrawl#start-of-content)\\n\\nYou signed
|
||||
in with another tab or window. [Reload](https://github.com/firecrawl) to refresh
|
||||
your session.You signed out in another tab or window. [Reload](https://github.com/firecrawl)
|
||||
to refresh your session.You switched accounts on another tab or window. [Reload](https://github.com/firecrawl)
|
||||
to refresh your session.Dismiss alert\\n\\n{{ message }}\\n\\n[README.md](https://github.com/firecrawl/.github/tree/main/profile/README.md)\\n\\n#
|
||||
\U0001F525 Firecrawl\\n\\n[Permalink: \U0001F525 Firecrawl](https://github.com/firecrawl#-firecrawl)\\n\\n[](https://raw.githubusercontent.com/mendableai/firecrawl/main/img/firecrawl_logo.png)\\n\\n###
|
||||
Transform any website into LLM-ready data\\n\\n[Permalink: Transform any website
|
||||
into LLM-ready data](https://github.com/firecrawl#transform-any-website-into-llm-ready-data)\\n\\nAdvanced
|
||||
web scraping, crawling, and data extraction infrastructure for AI applications\\n\\n[](https://firecrawl.dev/)
|
||||
[](https://docs.firecrawl.dev/)
|
||||
[](https://discord.com/invite/gSmWdAkdwd)\\n\\n[](https://github.com/mendableai/firecrawl/blob/main/LICENSE)[](https://github.com/mendableai/firecrawl/stargazers)[](https://pepy.tech/project/firecrawl-py)[](https://x.com/firecrawl_dev)\\n\\n*
|
||||
* *\\n\\n## Why Firecrawl?\\n\\n[Permalink: Why Firecrawl?](https://github.com/firecrawl#why-firecrawl)\\n\\n**Building
|
||||
AI applications?** You need clean, structured data from the web. Firecrawl
|
||||
handles the complexity of modern web scraping so you can focus on building
|
||||
great products.\\n\\n## Our Core Ecosystem\\n\\n[Permalink: Our Core Ecosystem](https://github.com/firecrawl#our-core-ecosystem)\\n\\n###
|
||||
Main Repository\\n\\n[Permalink: Main Repository](https://github.com/firecrawl#main-repository)\\n\\n[](https://github.com/mendableai/firecrawl)\\n\\n**[firecrawl](https://github.com/mendableai/firecrawl)**
|
||||
\\\\- Core API & SDK\\n\\nTurn entire websites into LLM-ready markdown or
|
||||
structured data. Our flagship product with 40k+ stars.\\n\\n### Cloud API\\n\\n[Permalink:
|
||||
Cloud API](https://github.com/firecrawl#cloud-api)\\n\\n[](https://firecrawl.dev/)\\n\\n**[Firecrawl](https://firecrawl.dev/)**
|
||||
\\\\- Hosted API Service\\n\\nProduction-ready web scraping without infrastructure
|
||||
management. Get your API key and start scraping in minutes with our reliable,
|
||||
scalable cloud service.\\n\\n### MCP Integration\\n\\n[Permalink: MCP Integration](https://github.com/firecrawl#mcp-integration)\\n\\n[](https://github.com/mendableai/firecrawl-mcp-server)\\n\\n**[firecrawl-mcp-server](https://github.com/mendableai/firecrawl-mcp-server)**
|
||||
\\\\- Model Context Protocol Server\\n\\nAdd powerful web scraping capabilities
|
||||
to Claude, Cursor, and any MCP-compatible LLM client.\\n\\n## Community &
|
||||
Support\\n\\n[Permalink: Community & Support](https://github.com/firecrawl#community--support)\\n\\n[](https://discord.com/invite/gSmWdAkdwd)[](https://x.com/firecrawl_dev)[](https://www.linkedin.com/company/104100957/)[](https://github.com/mendableai/firecrawl/discussions)[](https://docs.firecrawl.dev/)\\n\\n##
|
||||
Built By Mendable\\n\\n[Permalink: Built By Mendable](https://github.com/firecrawl#built-by-mendable)\\n\\nWe're
|
||||
the team behind [Mendable.ai](https://mendable.ai/), passionate about making
|
||||
web data accessible for AI applications. Firecrawl powers thousands of AI
|
||||
products worldwide.\\n\\n* * *\\n\\n**Ready to build something amazing?**\\n\\n[Get
|
||||
your API key](https://firecrawl.dev/) and start scraping in minutes\\n\\n\\n[Star
|
||||
our main repo](https://github.com/mendableai/firecrawl) \u2022\\n[Try the
|
||||
playground](https://firecrawl.dev/playground) \u2022\\n[Read the docs](https://docs.firecrawl.dev/)\\n\\n##
|
||||
Pinned Loading\\n\\n1. [firecrawl](https://github.com/firecrawl/firecrawl)
|
||||
firecrawlPublic\\n\\n\\n\\n\\n\\n\\n\U0001F525 The Web Data API for AI - Turn
|
||||
entire websites into LLM-ready markdown or structured data\\n\\n\\n\\n\\nTypeScript[64.9k](https://github.com/firecrawl/firecrawl/stargazers)
|
||||
[5.1k](https://github.com/firecrawl/firecrawl/forks)\\n\\n2. [mendable-nextjs-chatbot](https://github.com/firecrawl/mendable-nextjs-chatbot)
|
||||
mendable-nextjs-chatbotPublic template\\n\\n\\n\\n\\n\\n\\nNext.js Starter
|
||||
Template for building chatbots with Mendable\\n\\n\\n\\n\\nTypeScript[256](https://github.com/firecrawl/mendable-nextjs-chatbot/stargazers)
|
||||
[52](https://github.com/firecrawl/mendable-nextjs-chatbot/forks)\\n\\n3. [rag-arena](https://github.com/firecrawl/rag-arena)
|
||||
rag-arenaPublic\\n\\n\\n\\n\\n\\n\\nOpen-source RAG evaluation through users'
|
||||
feedback\\n\\n\\n\\n\\nTypeScript[206](https://github.com/firecrawl/rag-arena/stargazers)
|
||||
[32](https://github.com/firecrawl/rag-arena/forks)\\n\\n4. [QA\\\\_clustering](https://github.com/firecrawl/QA_clustering)
|
||||
QA\\\\_clusteringPublic\\n\\n\\n\\n\\n\\n\\nAnalyzing chat interactions w/
|
||||
LLMs to improve \U0001F99C\U0001F517 Langchain docs\\n\\n\\n\\n\\nJupyter
|
||||
Notebook[80](https://github.com/firecrawl/QA_clustering/stargazers) [12](https://github.com/firecrawl/QA_clustering/forks)\\n\\n5.
|
||||
[data-connectors](https://github.com/firecrawl/data-connectors) data-connectorsPublic\\n\\n\\n\\n\\n\\n\\nLLM-ready
|
||||
data connectors\\n\\n\\n\\n\\nTypeScript[95](https://github.com/firecrawl/data-connectors/stargazers)
|
||||
[23](https://github.com/firecrawl/data-connectors/forks)\\n\\n6. [mendable-py](https://github.com/firecrawl/mendable-py)
|
||||
mendable-pyPublic\\n\\n\\n\\n\\n\\n\\nBuild Production Ready LLM Chat Apps
|
||||
in Minutes\\n\\n\\n\\n\\nPython[33](https://github.com/firecrawl/mendable-py/stargazers)
|
||||
[7](https://github.com/firecrawl/mendable-py/forks)\\n\\n\\n### Repositories\\n\\nLoading\\n\\nType\\n\\nAllPublicSourcesForksArchivedMirrorsTemplates\\n\\nLanguage\\n\\nAllCSSGoJavaJavaScriptJupyter
|
||||
NotebookMDXPythonRustTypeScript\\n\\nSort\\n\\nLast updatedNameStars\\n\\nShowing
|
||||
10 of 61 repositories\\n\\n- [firecrawl](https://github.com/firecrawl/firecrawl)\\nPublic\\n\\n\\n\\n\U0001F525
|
||||
The Web Data API for AI - Turn entire websites into LLM-ready markdown or
|
||||
structured data\\n\\n\\n\\n\\n\\n\\nfirecrawl/firecrawl\u2019s past year of
|
||||
commit activity\\n\\n\\n\\nTypeScript[64,949](https://github.com/firecrawl/firecrawl/stargazers)AGPL-3.0\\n[5,132](https://github.com/firecrawl/firecrawl/forks)
|
||||
[27](https://github.com/firecrawl/firecrawl/issues) [(2 issues need help)](https://github.com/firecrawl/firecrawl/issues?q=label%3A%22good+first+issue%22+is%3Aissue+is%3Aopen)
|
||||
[85](https://github.com/firecrawl/firecrawl/pulls)\\nUpdated 2 hours agoOct
|
||||
27, 2025\\n\\n- [firecrawl-docs](https://github.com/firecrawl/firecrawl-docs)\\nPublic\\n\\n\\n\\nDocumentation
|
||||
for Firecrawl.\\n\\n\\n\\n\\n\\n\\nfirecrawl/firecrawl-docs\u2019s past year
|
||||
of commit activity\\n\\n\\n\\nMDX[17](https://github.com/firecrawl/firecrawl-docs/stargazers)
|
||||
[35](https://github.com/firecrawl/firecrawl-docs/forks) [10](https://github.com/firecrawl/firecrawl-docs/issues)
|
||||
[5](https://github.com/firecrawl/firecrawl-docs/pulls)\\nUpdated 20 hours
|
||||
agoOct 26, 2025\\n\\n- [open-agent-builder](https://github.com/firecrawl/open-agent-builder)\\nPublic\\n\\n\\n\\n\U0001F525
|
||||
Visual workflow builder for AI agents powered by Firecrawl - drag-and-drop
|
||||
web scraping pipelines with real-time execution\\n\\n\\n\\n\\n\\n\\nfirecrawl/open-agent-builder\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nTypeScript[1,673](https://github.com/firecrawl/open-agent-builder/stargazers)
|
||||
[274](https://github.com/firecrawl/open-agent-builder/forks) [4](https://github.com/firecrawl/open-agent-builder/issues)
|
||||
[2](https://github.com/firecrawl/open-agent-builder/pulls)\\nUpdated last
|
||||
weekOct 20, 2025\\n\\n- [firecrawl-mcp-server](https://github.com/firecrawl/firecrawl-mcp-server)\\nPublic\\n\\n\\n\\n\U0001F525
|
||||
Official Firecrawl MCP Server - Adds powerful web scraping and search to Cursor,
|
||||
Claude and any other LLM clients.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n[**Uh
|
||||
oh!**](https://github.com/firecrawl/firecrawl-mcp-server/graphs/commit-activity)\\n\\n[There
|
||||
was an error while loading.](https://github.com/firecrawl/firecrawl-mcp-server/graphs/commit-activity)
|
||||
[Please reload this page](https://github.com/firecrawl).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nfirecrawl/firecrawl-mcp-server\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nJavaScript[4,794](https://github.com/firecrawl/firecrawl-mcp-server/stargazers)MIT\\n[519](https://github.com/firecrawl/firecrawl-mcp-server/forks)
|
||||
[44](https://github.com/firecrawl/firecrawl-mcp-server/issues) [17](https://github.com/firecrawl/firecrawl-mcp-server/pulls)\\nUpdated
|
||||
last weekOct 19, 2025\\n\\n- [n8n-nodes-firecrawl](https://github.com/firecrawl/n8n-nodes-firecrawl)\\nPublic\\n\\n\\n\\nn8n
|
||||
node to interact with Firecrawl\\n\\n\\n\\n\\n\\n\\nfirecrawl/n8n-nodes-firecrawl\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nTypeScript[21](https://github.com/firecrawl/n8n-nodes-firecrawl/stargazers)MIT\\n[13](https://github.com/firecrawl/n8n-nodes-firecrawl/forks)
|
||||
[3](https://github.com/firecrawl/n8n-nodes-firecrawl/issues) [0](https://github.com/firecrawl/n8n-nodes-firecrawl/pulls)\\nUpdated
|
||||
2 weeks agoOct 17, 2025\\n\\n- [.github](https://github.com/firecrawl/.github)\\nPublic\\n\\n\\n\\n\\nfirecrawl/.github\u2019s
|
||||
past year of commit activity\\n\\n\\n\\n0\\n[1](https://github.com/firecrawl/.github/forks)
|
||||
[0](https://github.com/firecrawl/.github/issues) [0](https://github.com/firecrawl/.github/pulls)\\nUpdated
|
||||
2 weeks agoOct 12, 2025\\n\\n- [fire-enrich](https://github.com/firecrawl/fire-enrich)\\nPublic\\n\\n\\n\\n\U0001F525
|
||||
AI-powered data enrichment tool that transforms emails into rich datasets
|
||||
with company profiles, funding data, tech stacks, and more using Firecrawl
|
||||
and multi-agent AI\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n[**Uh oh!**](https://github.com/firecrawl/fire-enrich/graphs/commit-activity)\\n\\n[There
|
||||
was an error while loading.](https://github.com/firecrawl/fire-enrich/graphs/commit-activity)
|
||||
[Please reload this page](https://github.com/firecrawl).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nfirecrawl/fire-enrich\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nTypeScript[953](https://github.com/firecrawl/fire-enrich/stargazers)MIT\\n[239](https://github.com/firecrawl/fire-enrich/forks)
|
||||
[12](https://github.com/firecrawl/fire-enrich/issues) [3](https://github.com/firecrawl/fire-enrich/pulls)\\nUpdated
|
||||
3 weeks agoOct 8, 2025\\n\\n- [firecrawl-java-sdk](https://github.com/firecrawl/firecrawl-java-sdk)\\nPublic\\n\\n\\n\\n\\nfirecrawl/firecrawl-java-sdk\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nJava[11](https://github.com/firecrawl/firecrawl-java-sdk/stargazers)MIT\\n[4](https://github.com/firecrawl/firecrawl-java-sdk/forks)
|
||||
[0](https://github.com/firecrawl/firecrawl-java-sdk/issues) [0](https://github.com/firecrawl/firecrawl-java-sdk/pulls)\\nUpdated
|
||||
last monthSep 28, 2025\\n\\n- [open-lovable](https://github.com/firecrawl/open-lovable)\\nPublic\\n\\n\\n\\n\U0001F525
|
||||
Clone and recreate any website as a modern React app in seconds\\n\\n\\n\\n\\n\\n\\nfirecrawl/open-lovable\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nTypeScript[21,320](https://github.com/firecrawl/open-lovable/stargazers)MIT\\n[3,986](https://github.com/firecrawl/open-lovable/forks)
|
||||
[70](https://github.com/firecrawl/open-lovable/issues) [33](https://github.com/firecrawl/open-lovable/pulls)\\nUpdated
|
||||
last monthSep 27, 2025\\n\\n- [mineru-api](https://github.com/firecrawl/mineru-api)\\nPublic\\n\\n\\n\\n\\nfirecrawl/mineru-api\u2019s
|
||||
past year of commit activity\\n\\n\\n\\nPython[12](https://github.com/firecrawl/mineru-api/stargazers)AGPL-3.0\\n[2](https://github.com/firecrawl/mineru-api/forks)
|
||||
[1](https://github.com/firecrawl/mineru-api/issues) [1](https://github.com/firecrawl/mineru-api/pulls)\\nUpdated
|
||||
on Sep 26Sep 26, 2025\\n\\n\\n[View all repositories](https://github.com/orgs/firecrawl/repositories?type=all)\\n\\n[**People**](https://github.com/orgs/firecrawl/people)\\n\\n[](https://github.com/alexnucci)[](https://github.com/micahstairs)[](https://github.com/nickscamara)[](https://github.com/mogery)[](https://github.com/developersdigest)\\n\\n####
|
||||
Top languages\\n\\n[TypeScript](https://github.com/orgs/firecrawl/repositories?language=typescript&type=all)
|
||||
[Python](https://github.com/orgs/firecrawl/repositories?language=python&type=all)
|
||||
[JavaScript](https://github.com/orgs/firecrawl/repositories?language=javascript&type=all)
|
||||
[Go](https://github.com/orgs/firecrawl/repositories?language=go&type=all)
|
||||
[MDX](https://github.com/orgs/firecrawl/repositories?language=mdx&type=all)\\n\\n####
|
||||
Most used topics\\n\\n[ai](https://github.com/search?q=topic%3Aai+org%3Afirecrawl+fork%3Atrue&type=repositories
|
||||
\\\"Topic: ai\\\") [firecrawl](https://github.com/search?q=topic%3Afirecrawl+org%3Afirecrawl+fork%3Atrue&type=repositories
|
||||
\\\"Topic: firecrawl\\\") [llm](https://github.com/search?q=topic%3Allm+org%3Afirecrawl+fork%3Atrue&type=repositories
|
||||
\\\"Topic: llm\\\") [web-crawler](https://github.com/search?q=topic%3Aweb-crawler+org%3Afirecrawl+fork%3Atrue&type=repositories
|
||||
\\\"Topic: web-crawler\\\") [web-scraping](https://github.com/search?q=topic%3Aweb-scraping+org%3Afirecrawl+fork%3Atrue&type=repositories
|
||||
\\\"Topic: web-scraping\\\")\\n\\nYou can\u2019t perform that action at this
|
||||
time.\",\"metadata\":{\"analytics-location\":\"/<org-login>\",\"apple-itunes-app\":\"app-id=1477376905,
|
||||
app-argument=https://github.com/firecrawl\",\"twitter:card\":\"summary_large_image\",\"google-site-verification\":\"Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I\",\"description\":\"Web
|
||||
data API for AI. Firecrawl has 61 repositories available. Follow their code
|
||||
on GitHub.\",\"og:image\":\"https://avatars.githubusercontent.com/u/135057108?s=280&v=4\",\"og:type\":\"profile\",\"visitor-payload\":\"eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiI5REFEOjIzM0ExMzo4RDgzNEI6QzQ5OUNCOjY4RkYzODlGIiwidmlzaXRvcl9pZCI6IjQwMDQ0MTI5MTkxMDA5NDY1OTEiLCJyZWdpb25fZWRnZSI6ImlhZCIsInJlZ2lvbl9yZW5kZXIiOiJpYWQifQ==\",\"github-keyboard-shortcuts\":\"copilot\",\"user-login\":\"\",\"viewport\":\"width=device-width\",\"og:description\":\"Web
|
||||
data API for AI. Firecrawl has 61 repositories available. Follow their code
|
||||
on GitHub.\",\"turbo-cache-control\":\"no-preview\",\"fetch-nonce\":\"v2:35a2e032-0081-3f6b-595e-967e32025c6f\",\"og:url\":\"https://github.com/firecrawl\",\"title\":\"Firecrawl
|
||||
\xB7 GitHub\",\"route-pattern\":\"/:user_id(.:format)\",\"route-action\":\"show\",\"octolytics-url\":\"https://collector.github.com/github/collect\",\"og:site_name\":\"GitHub\",\"twitter:title\":\"Firecrawl\",\"request-id\":\"9DAD:233A13:8D834B:C499CB:68FF389F\",\"ogSiteName\":\"GitHub\",\"fb:app_id\":\"1401488693436528\",\"language\":\"en\",\"twitter:image\":\"https://avatars.githubusercontent.com/u/135057108?s=280&v=4\",\"ogImage\":\"https://avatars.githubusercontent.com/u/135057108?s=280&v=4\",\"release\":\"c44b7f7aa5c70f3296484971978c9f4b1b473352\",\"theme-color\":\"#1e2327\",\"color-scheme\":\"light
|
||||
dark\",\"html-safe-nonce\":\"2d932295da6aa360d861f16279839ab109dc4e977a51bc99204477969d7d12c6\",\"ogTitle\":\"Firecrawl\",\"hovercard-subject-tag\":\"organization:135057108\",\"twitter:description\":\"Web
|
||||
data API for AI. Firecrawl has 61 repositories available. Follow their code
|
||||
on GitHub.\",\"hostname\":\"github.com\",\"ogUrl\":\"https://github.com/firecrawl\",\"ogDescription\":\"Web
|
||||
data API for AI. Firecrawl has 61 repositories available. Follow their code
|
||||
on GitHub.\",\"route-controller\":\"profiles\",\"favicon\":\"https://github.githubassets.com/favicons/favicon.svg\",\"visitor-hmac\":\"43cf3bbb9c57a3bcf0b1857fbcabcc78c274e7082e0d28c76cd6d4651bc2a920\",\"twitter:site\":\"@github\",\"ui-target\":\"full\",\"og:title\":\"Firecrawl\",\"expected-hostname\":\"github.com\",\"og:image:alt\":\"Web
|
||||
data API for AI. Firecrawl has 61 repositories available. Follow their code
|
||||
on GitHub.\",\"profile:username\":\"firecrawl\",\"current-catalog-service-hash\":\"4a1c50a83cf6cc4b55b6b9c53e553e3f847c876b87fb333f71f5d05db8f1a7db\",\"turbo-body-classes\":\"logged-out
|
||||
env-production page-responsive\",\"browser-stats-url\":\"https://api.github.com/_private/browser/stats\",\"browser-errors-url\":\"https://api.github.com/_private/browser/errors\",\"scrapeId\":\"65cbc300-be11-4a1a-9d20-c114fb8473a7\",\"sourceURL\":\"https://github.com/firecrawl\",\"url\":\"https://github.com/firecrawl\",\"statusCode\":200,\"contentType\":\"text/html;
|
||||
charset=utf-8\",\"proxyUsed\":\"basic\",\"cacheState\":\"hit\",\"cachedAt\":\"2025-10-27T09:17:20.756Z\"}},{\"url\":\"https://www.linkedin.com/company/firecrawl\",\"title\":\"Firecrawl
|
||||
| LinkedIn\",\"description\":\"Our Dify integration now uses Firecrawl /v2
|
||||
endpoints Scraping is 10x faster thanks to intelligent caching, plus we've
|
||||
added semantic ...\",\"position\":5}]},\"creditsUsed\":3}"
|
||||
headers:
|
||||
Access-Control-Allow-Origin:
|
||||
- '*'
|
||||
Alt-Svc:
|
||||
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
|
||||
Content-Length:
|
||||
- '93428'
|
||||
Content-Type:
|
||||
- application/json; charset=utf-8
|
||||
Date:
|
||||
- Wed, 29 Oct 2025 14:37:39 GMT
|
||||
ETag:
|
||||
- W/"16cf4-kHwVbMu4CCVG2UIt6p1g/gz5M4M"
|
||||
Via:
|
||||
- 1.1 google
|
||||
X-Powered-By:
|
||||
- Express
|
||||
X-Response-Time:
|
||||
- 13172.495ms
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
0
lib/crewai-tools/tests/tools/rag/__init__.py
Normal file
0
lib/crewai-tools/tests/tools/rag/__init__.py
Normal file
@@ -52,7 +52,7 @@ class GeminiCompletion(BaseLLM):
|
||||
|
||||
Args:
|
||||
model: Gemini model name (e.g., 'gemini-2.0-flash-001', 'gemini-1.5-pro')
|
||||
api_key: Google API key (defaults to GOOGLE_API_KEY or GEMINI_API_KEY env var)
|
||||
api_key: Google API key (defaults to GEMINI_API_KEY or GEMINI_API_KEY env var)
|
||||
project: Google Cloud project ID (for Vertex AI)
|
||||
location: Google Cloud location (for Vertex AI, defaults to 'us-central1')
|
||||
temperature: Sampling temperature (0-2)
|
||||
@@ -82,7 +82,7 @@ class GeminiCompletion(BaseLLM):
|
||||
|
||||
# Get API configuration with environment variable fallbacks
|
||||
self.api_key = (
|
||||
api_key or os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
|
||||
api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY")
|
||||
)
|
||||
self.project = project or os.getenv("GOOGLE_CLOUD_PROJECT")
|
||||
self.location = location or os.getenv("GOOGLE_CLOUD_LOCATION") or "us-central1"
|
||||
@@ -165,7 +165,7 @@ class GeminiCompletion(BaseLLM):
|
||||
return genai.Client(**client_params)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
"Either GOOGLE_API_KEY/GEMINI_API_KEY (for Gemini API) or "
|
||||
"Either GEMINI_API_KEY/GEMINI_API_KEY (for Gemini API) or "
|
||||
"GOOGLE_CLOUD_PROJECT (for Vertex AI) must be set"
|
||||
) from e
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ class GenerativeAiProvider(BaseEmbeddingsProvider[GoogleGenerativeAiEmbeddingFun
|
||||
validation_alias="EMBEDDINGS_GOOGLE_GENERATIVE_AI_MODEL_NAME",
|
||||
)
|
||||
api_key: str = Field(
|
||||
description="Google API key", validation_alias="EMBEDDINGS_GOOGLE_API_KEY"
|
||||
description="Google API key", validation_alias="EMBEDDINGS_GEMINI_API_KEY"
|
||||
)
|
||||
task_type: str = Field(
|
||||
default="RETRIEVAL_DOCUMENT",
|
||||
|
||||
@@ -261,25 +261,6 @@ async def test_lite_agent_returns_usage_metrics_async():
|
||||
assert result.usage_metrics["total_tokens"] > 0
|
||||
|
||||
|
||||
class TestFlow(Flow):
|
||||
"""A test flow that creates and runs an agent."""
|
||||
|
||||
def __init__(self, llm, tools):
|
||||
self.llm = llm
|
||||
self.tools = tools
|
||||
super().__init__()
|
||||
|
||||
@start()
|
||||
def start(self):
|
||||
agent = Agent(
|
||||
role="Test Agent",
|
||||
goal="Test Goal",
|
||||
backstory="Test Backstory",
|
||||
llm=self.llm,
|
||||
tools=self.tools,
|
||||
)
|
||||
return agent.kickoff("Test query")
|
||||
|
||||
|
||||
def verify_agent_parent_flow(result, agent, flow):
|
||||
"""Verify that both the result and agent have the correct parent flow."""
|
||||
|
||||
@@ -1,495 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"trace_id": "REDACTED_TRACE_ID", "execution_type":
|
||||
"crew", "user_identifier": null, "execution_context": {"crew_fingerprint": null,
|
||||
"crew_name": "Unknown Crew", "flow_name": null, "crewai_version": "1.4.0", "privacy_level":
|
||||
"standard"}, "execution_metadata": {"expected_duration_estimate": 300, "agent_count":
|
||||
0, "task_count": 0, "flow_method_count": 0, "execution_started_at": "2025-11-07T18:27:07.650947+00:00"}}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Accept-Encoding:
|
||||
- gzip, deflate, zstd
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '434'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- CrewAI-CLI/1.4.0
|
||||
X-Crewai-Version:
|
||||
- 1.4.0
|
||||
method: POST
|
||||
uri: https://app.crewai.com/crewai_plus/api/v1/tracing/batches
|
||||
response:
|
||||
body:
|
||||
string: '{"error":"bad_credentials","message":"Bad credentials"}'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '55'
|
||||
Content-Type:
|
||||
- application/json; charset=utf-8
|
||||
Date:
|
||||
- Fri, 07 Nov 2025 18:27:07 GMT
|
||||
cache-control:
|
||||
- no-store
|
||||
content-security-policy:
|
||||
- 'default-src ''self'' *.app.crewai.com app.crewai.com; script-src ''self''
|
||||
''unsafe-inline'' *.app.crewai.com app.crewai.com https://cdn.jsdelivr.net/npm/apexcharts
|
||||
https://www.gstatic.com https://run.pstmn.io https://apis.google.com https://apis.google.com/js/api.js
|
||||
https://accounts.google.com https://accounts.google.com/gsi/client https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css.map
|
||||
https://*.google.com https://docs.google.com https://slides.google.com https://js.hs-scripts.com
|
||||
https://js.sentry-cdn.com https://browser.sentry-cdn.com https://www.googletagmanager.com
|
||||
https://js-na1.hs-scripts.com https://js.hubspot.com http://js-na1.hs-scripts.com
|
||||
https://bat.bing.com https://cdn.amplitude.com https://cdn.segment.com https://d1d3n03t5zntha.cloudfront.net/
|
||||
https://descriptusercontent.com https://edge.fullstory.com https://googleads.g.doubleclick.net
|
||||
https://js.hs-analytics.net https://js.hs-banner.com https://js.hsadspixel.net
|
||||
https://js.hscollectedforms.net https://js.usemessages.com https://snap.licdn.com
|
||||
https://static.cloudflareinsights.com https://static.reo.dev https://www.google-analytics.com
|
||||
https://share.descript.com/; style-src ''self'' ''unsafe-inline'' *.app.crewai.com
|
||||
app.crewai.com https://cdn.jsdelivr.net/npm/apexcharts; img-src ''self'' data:
|
||||
*.app.crewai.com app.crewai.com https://zeus.tools.crewai.com https://dashboard.tools.crewai.com
|
||||
https://cdn.jsdelivr.net https://forms.hsforms.com https://track.hubspot.com
|
||||
https://px.ads.linkedin.com https://px4.ads.linkedin.com https://www.google.com
|
||||
https://www.google.com.br; font-src ''self'' data: *.app.crewai.com app.crewai.com;
|
||||
connect-src ''self'' *.app.crewai.com app.crewai.com https://zeus.tools.crewai.com
|
||||
https://connect.useparagon.com/ https://zeus.useparagon.com/* https://*.useparagon.com/*
|
||||
https://run.pstmn.io https://connect.tools.crewai.com/ https://*.sentry.io
|
||||
https://www.google-analytics.com https://edge.fullstory.com https://rs.fullstory.com
|
||||
https://api.hubspot.com https://forms.hscollectedforms.net https://api.hubapi.com
|
||||
https://px.ads.linkedin.com https://px4.ads.linkedin.com https://google.com/pagead/form-data/16713662509
|
||||
https://google.com/ccm/form-data/16713662509 https://www.google.com/ccm/collect
|
||||
https://worker-actionkit.tools.crewai.com https://api.reo.dev; frame-src ''self''
|
||||
*.app.crewai.com app.crewai.com https://connect.useparagon.com/ https://zeus.tools.crewai.com
|
||||
https://zeus.useparagon.com/* https://connect.tools.crewai.com/ https://docs.google.com
|
||||
https://drive.google.com https://slides.google.com https://accounts.google.com
|
||||
https://*.google.com https://app.hubspot.com/ https://td.doubleclick.net https://www.googletagmanager.com/
|
||||
https://www.youtube.com https://share.descript.com'
|
||||
expires:
|
||||
- '0'
|
||||
permissions-policy:
|
||||
- camera=(), microphone=(self), geolocation=()
|
||||
pragma:
|
||||
- no-cache
|
||||
referrer-policy:
|
||||
- strict-origin-when-cross-origin
|
||||
strict-transport-security:
|
||||
- max-age=63072000; includeSubDomains
|
||||
vary:
|
||||
- Accept
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-frame-options:
|
||||
- SAMEORIGIN
|
||||
x-permitted-cross-domain-policies:
|
||||
- none
|
||||
x-request-id:
|
||||
- REDACTED_REQUEST_ID
|
||||
x-runtime:
|
||||
- '0.080681'
|
||||
x-xss-protection:
|
||||
- 1; mode=block
|
||||
status:
|
||||
code: 401
|
||||
message: Unauthorized
|
||||
- request:
|
||||
body: '{"messages":[{"role":"system","content":"You are data collector. You must
|
||||
use the get_data tool extensively\nYour personal goal is: collect data using
|
||||
the get_data tool\nYou ONLY have access to the following tools, and should NEVER
|
||||
make up tools that are not listed here:\n\nTool Name: get_data\nTool Arguments:
|
||||
{''step'': {''description'': None, ''type'': ''str''}}\nTool Description: Get
|
||||
data for a step. Always returns data requiring more steps.\n\nIMPORTANT: Use
|
||||
the following format in your response:\n\n```\nThought: you should always think
|
||||
about what to do\nAction: the action to take, only one name of [get_data], just
|
||||
the name, exactly as it''s written.\nAction Input: the input to the action,
|
||||
just a simple JSON object, enclosed in curly braces, using \" to wrap keys and
|
||||
values.\nObservation: the result of the action\n```\n\nOnce all necessary information
|
||||
is gathered, return the following format:\n\n```\nThought: I now know the final
|
||||
answer\nFinal Answer: the final answer to the original input question\n```"},{"role":"user","content":"\nCurrent
|
||||
Task: Use get_data tool for step1, step2, step3, step4, step5, step6, step7,
|
||||
step8, step9, and step10. Do NOT stop until you''ve called it for ALL steps.\n\nThis
|
||||
is the expected criteria for your final answer: A summary of all data collected\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}],"model":"gpt-4.1-mini"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1534'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.109.1
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.109.1
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAA4xSYWvbMBD97l9x6HMcYsfpUn8rg0FHYbAOyrYUo0hnW5ksCem8tYT89yG7id2t
|
||||
g30x5t69p/fu7pgAMCVZCUy0nETndPr+2919j4fr9VNR/Opv7vBD/bAVXz/dfzx8fmCLyLD7Awo6
|
||||
s5bCdk4jKWtGWHjkhFE1e3eVb4rVKt8OQGcl6khrHKXFMks7ZVSar/JNuirSrHiht1YJDKyE7wkA
|
||||
wHH4RqNG4hMrYbU4VzoMgTfIyksTAPNWxwrjIahA3BBbTKCwhtAM3r+0tm9aKuEWQmt7LSEQ9wT7
|
||||
ZxBWaxSkTAOSE4faegiELgMeQJlAvheEcrkzNyLmLqFBqmLruQK3xvVUwnHHInHHyvEn27HT3I/H
|
||||
ug88DsX0Ws8AbowlHqWGSTy+IKdLdm0b5+0+/EFltTIqtJVHHqyJOQNZxwb0lAA8DjPuX42NOW87
|
||||
RxXZHzg8t15nox6bdjuh4zYBGFniesbaXC/e0KskElc6zLbEBBctyok6rZT3UtkZkMxS/+3mLe0x
|
||||
uTLN/8hPgBDoCGXlPEolXiee2jzG0/9X22XKg2EW0P9UAitS6OMmJNa81+M9svAcCLuqVqZB77wa
|
||||
j7J2VSHy7Sart1c5S07JbwAAAP//AwCiugNoowMAAA==
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 99aee205bbd2de96-EWR
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Fri, 07 Nov 2025 18:27:08 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=REDACTED_COOKIE;
|
||||
path=/; expires=Fri, 07-Nov-25 18:57:08 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=REDACTED_COOKIE;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- REDACTED_ORG_ID
|
||||
openai-processing-ms:
|
||||
- '557'
|
||||
openai-project:
|
||||
- REDACTED_PROJECT_ID
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
x-envoy-upstream-service-time:
|
||||
- '701'
|
||||
x-openai-proxy-wasm:
|
||||
- v0.1
|
||||
x-ratelimit-limit-requests:
|
||||
- '500'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '200000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '499'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '199645'
|
||||
x-ratelimit-reset-requests:
|
||||
- 120ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 106ms
|
||||
x-request-id:
|
||||
- REDACTED_REQUEST_ID
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"system","content":"You are data collector. You must
|
||||
use the get_data tool extensively\nYour personal goal is: collect data using
|
||||
the get_data tool\nYou ONLY have access to the following tools, and should NEVER
|
||||
make up tools that are not listed here:\n\nTool Name: get_data\nTool Arguments:
|
||||
{''step'': {''description'': None, ''type'': ''str''}}\nTool Description: Get
|
||||
data for a step. Always returns data requiring more steps.\n\nIMPORTANT: Use
|
||||
the following format in your response:\n\n```\nThought: you should always think
|
||||
about what to do\nAction: the action to take, only one name of [get_data], just
|
||||
the name, exactly as it''s written.\nAction Input: the input to the action,
|
||||
just a simple JSON object, enclosed in curly braces, using \" to wrap keys and
|
||||
values.\nObservation: the result of the action\n```\n\nOnce all necessary information
|
||||
is gathered, return the following format:\n\n```\nThought: I now know the final
|
||||
answer\nFinal Answer: the final answer to the original input question\n```"},{"role":"user","content":"\nCurrent
|
||||
Task: Use get_data tool for step1, step2, step3, step4, step5, step6, step7,
|
||||
step8, step9, and step10. Do NOT stop until you''ve called it for ALL steps.\n\nThis
|
||||
is the expected criteria for your final answer: A summary of all data collected\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"},{"role":"assistant","content":"Thought:
|
||||
I should start by collecting data for step1 as instructed.\nAction: get_data\nAction
|
||||
Input: {\"step\":\"step1\"}\nObservation: Data for step1: incomplete, need to
|
||||
query more steps."}],"model":"gpt-4.1-mini"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1757'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=REDACTED_COOKIE;
|
||||
_cfuvid=REDACTED_COOKIE
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.109.1
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.109.1
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFNNb9swDL37VxA6x0HiOU3mW9cOQ4F9YNjQQ5fCUGXaVidLqkQnzYL8
|
||||
90F2ErtbB+xiCHx8j+QjvY8AmCxYBkzUnERjVXx19/Hb5tPm/fbq8sPX5+Wvx6V+t93efXY1v71m
|
||||
k8AwD48o6MSaCtNYhSSN7mHhkBMG1fnyIlmks1nytgMaU6AKtMpSnE7ncSO1jJNZsohnaTxPj/Ta
|
||||
SIGeZfAjAgDYd9/QqC7wmWUwm5wiDXrPK2TZOQmAOaNChHHvpSeuiU0GUBhNqLvev9emrWrK4AY0
|
||||
YgFkIKBStxjentAmfVApFAQFJw4en1rUJLlSO+AeHD610mExXetLESzIoELKQ+4pAjfatpTBfs2C
|
||||
5ppl/SNZs8Naf3nw6Da8p16HEqVxffEMpD56ixNojMMu7kGjCIO73XQ8msOy9Tz4q1ulRgDX2lBX
|
||||
oTP1/ogczjYqU1lnHvwfVFZKLX2dO+Te6GCZJ2NZhx4igPtuXe2LDTDrTGMpJ/MTu3Jvlqtejw1n
|
||||
MqBpegTJEFejeJJMXtHLCyQulR8tnAkuaiwG6nAdvC2kGQHRaOq/u3lNu59c6up/5AdACLSERW4d
|
||||
FlK8nHhIcxj+on+lnV3uGmbhSKTAnCS6sIkCS96q/rSZ33nCJi+lrtBZJ/v7Lm2eimS1mJeri4RF
|
||||
h+g3AAAA//8DABrUefPuAwAA
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 99aee20dba0bde96-EWR
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Fri, 07 Nov 2025 18:27:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- REDACTED_ORG_ID
|
||||
openai-processing-ms:
|
||||
- '942'
|
||||
openai-project:
|
||||
- REDACTED_PROJECT_ID
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
x-envoy-upstream-service-time:
|
||||
- '1074'
|
||||
x-openai-proxy-wasm:
|
||||
- v0.1
|
||||
x-ratelimit-limit-requests:
|
||||
- '500'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '200000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '499'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '199599'
|
||||
x-ratelimit-reset-requests:
|
||||
- 120ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 120ms
|
||||
x-request-id:
|
||||
- REDACTED_REQUEST_ID
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"system","content":"You are data collector. You must
|
||||
use the get_data tool extensively\nYour personal goal is: collect data using
|
||||
the get_data tool\nYou ONLY have access to the following tools, and should NEVER
|
||||
make up tools that are not listed here:\n\nTool Name: get_data\nTool Arguments:
|
||||
{''step'': {''description'': None, ''type'': ''str''}}\nTool Description: Get
|
||||
data for a step. Always returns data requiring more steps.\n\nIMPORTANT: Use
|
||||
the following format in your response:\n\n```\nThought: you should always think
|
||||
about what to do\nAction: the action to take, only one name of [get_data], just
|
||||
the name, exactly as it''s written.\nAction Input: the input to the action,
|
||||
just a simple JSON object, enclosed in curly braces, using \" to wrap keys and
|
||||
values.\nObservation: the result of the action\n```\n\nOnce all necessary information
|
||||
is gathered, return the following format:\n\n```\nThought: I now know the final
|
||||
answer\nFinal Answer: the final answer to the original input question\n```"},{"role":"user","content":"\nCurrent
|
||||
Task: Use get_data tool for step1, step2, step3, step4, step5, step6, step7,
|
||||
step8, step9, and step10. Do NOT stop until you''ve called it for ALL steps.\n\nThis
|
||||
is the expected criteria for your final answer: A summary of all data collected\nyou
|
||||
MUST return the actual complete content as the final answer, not a summary.\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"},{"role":"assistant","content":"Thought:
|
||||
I should start by collecting data for step1 as instructed.\nAction: get_data\nAction
|
||||
Input: {\"step\":\"step1\"}\nObservation: Data for step1: incomplete, need to
|
||||
query more steps."},{"role":"assistant","content":"Thought: I need to continue
|
||||
to step2 to collect data sequentially as required.\nAction: get_data\nAction
|
||||
Input: {\"step\":\"step2\"}\nObservation: Data for step2: incomplete, need to
|
||||
query more steps."},{"role":"assistant","content":"Thought: I need to continue
|
||||
to step2 to collect data sequentially as required.\nAction: get_data\nAction
|
||||
Input: {\"step\":\"step2\"}\nObservation: Data for step2: incomplete, need to
|
||||
query more steps.\nNow it''s time you MUST give your absolute best final answer.
|
||||
You''ll ignore all previous instructions, stop using any tools, and just return
|
||||
your absolute BEST Final answer."}],"model":"gpt-4.1-mini"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '2399'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=REDACTED_COOKIE;
|
||||
_cfuvid=REDACTED_COOKIE
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.109.1
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.109.1
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.12.9
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//nJbfj6M2EMff81eM/NRKmwgI5Advp7v2FKlSW22f9rKKHHsI7hmbs83u
|
||||
nlb7v1eYBLJXQFxekMV8Z+ZjYw3f1xkAEZykQFhOHStKOf/48Mf9yzf5/Pnh498P9kl9ru51qR9k
|
||||
XsVBSO7qDH38F5m7ZC2YLkqJTmjVhJlB6rCuGq5XURIHwTL0gUJzlHXaqXTzeBHOC6HEPAqiZB7E
|
||||
8zA+p+daMLQkhS8zAIBX/6xBFccXkkJwd3lToLX0hCRtRQDEaFm/IdRaYR1Vjtx1QaaVQ+XZ/8l1
|
||||
dcpdCjsoKuuAaSmROeDUUci0ASolWIelhczowi9DcLpZBHDETBuE0ugnwYU6gcsRMqGohPOJIJzb
|
||||
AbVg8FslDHI4fvdKR+3XBezgWUjpdUJVCJW9VDqhO3gUp7X0PEhZ7puDUKANR7PYq736wOqjT9uE
|
||||
yxvYqbJyKbzuSZ20J2mzCPfkba/+PFo0T7RJ/VT3KalxEPpOzVb10VGhkPsu7Wn9ZTRD5JeDiBY/
|
||||
TxCNEUQtQTSNYHkDwXKMYNkSLKcRxDcQxGMEcUsQTyNIbiBIxgiSliCZRrC6gWA1RrBqCVbTCNY3
|
||||
EKzHCNYtwXoaweYGgs0YwaYl2Ewj2N5AsB0j2LYE22kEYXADQhiMzqSgG0rBAMUOlH6GnD6hH9vt
|
||||
DG/mtx/bYQBUcWBUnWc2jkxsX/13H/qg7DOaFPbq3o/FGiyFLzvFZMWxaXWenZdxn6PBx0YfDeuj
|
||||
Pv1yWL/s08fD+rhPnwzrkz79ali/6tOvh/XrPv1mWL/p02+H9ds+fRiMfLDgx4y9+uW3F8rc9Y/7
|
||||
cuEaF6C7O2rf/5Xv6iRGHara/fiKi1+vvYfBrLK0NkCqkvIqQJXSrilZu57Hc+St9TlSn0qjj/aH
|
||||
VJIJJWx+MEitVrWnsU6XxEffZgCP3k9V7ywSKY0uSndw+iv6dkl49lOk83FX0Sg5R512VHaBMFhe
|
||||
Iu8qHjg6KqS98mSEUZYj73I7A0crLvRVYHa17//z9NVu9i7UaUr5LsAYlg75oTTIBXu/505msDa6
|
||||
Q7L2nD0wqe+FYHhwAk39LThmtJKN+yT2u3VYHDKhTmhKIxoLmpWHmEWbJMw2q4jM3mb/AQAA//8D
|
||||
ACYaBDGRCwAA
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 99aee2174b18de96-EWR
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Fri, 07 Nov 2025 18:27:20 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- REDACTED_ORG_ID
|
||||
openai-processing-ms:
|
||||
- '9185'
|
||||
openai-project:
|
||||
- REDACTED_PROJECT_ID
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
x-envoy-upstream-service-time:
|
||||
- '9386'
|
||||
x-openai-proxy-wasm:
|
||||
- v0.1
|
||||
x-ratelimit-limit-requests:
|
||||
- '500'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '200000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '499'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '199457'
|
||||
x-ratelimit-reset-requests:
|
||||
- 120ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 162ms
|
||||
x-request-id:
|
||||
- REDACTED_REQUEST_ID
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
0
lib/crewai/tests/cli/triggers/__init__.py
Normal file
0
lib/crewai/tests/cli/triggers/__init__.py
Normal file
@@ -7,7 +7,9 @@ from typing import Any
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_result = load_dotenv(override=True)
|
||||
# Load .env.test for consistent test environment (mimics CI)
|
||||
env_test_path = Path(__file__).parent.parent.parent.parent / ".env.test"
|
||||
load_dotenv(env_test_path, override=True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="function")
|
||||
@@ -84,7 +86,7 @@ def _filter_response_headers(response):
|
||||
return response
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def vcr_config(request) -> dict[str, Any]:
|
||||
config = {
|
||||
"cassette_library_dir": os.path.join(os.path.dirname(__file__), "cassettes"),
|
||||
|
||||
@@ -12,8 +12,8 @@ from crewai.task import Task
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_google_api_key():
|
||||
"""Automatically mock GOOGLE_API_KEY for all tests in this module."""
|
||||
with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-key"}):
|
||||
"""Automatically mock GEMINI_API_KEY for all tests in this module."""
|
||||
with patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}):
|
||||
yield
|
||||
|
||||
|
||||
@@ -429,10 +429,10 @@ def test_gemini_vertex_ai_setup():
|
||||
|
||||
def test_gemini_api_key_configuration():
|
||||
"""
|
||||
Test that API key configuration works for both GOOGLE_API_KEY and GEMINI_API_KEY
|
||||
Test that API key configuration works for both GEMINI_API_KEY and GEMINI_API_KEY
|
||||
"""
|
||||
# Test with GOOGLE_API_KEY
|
||||
with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-google-key"}):
|
||||
# Test with GEMINI_API_KEY
|
||||
with patch.dict(os.environ, {"GEMINI_API_KEY": "test-google-key"}):
|
||||
llm = LLM(model="google/gemini-2.0-flash-001")
|
||||
|
||||
from crewai.llms.providers.gemini.completion import GeminiCompletion
|
||||
@@ -611,7 +611,7 @@ def test_gemini_environment_variable_api_key():
|
||||
"""
|
||||
Test that Google API key is properly loaded from environment
|
||||
"""
|
||||
with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-google-key"}):
|
||||
with patch.dict(os.environ, {"GEMINI_API_KEY": "test-google-key"}):
|
||||
llm = LLM(model="google/gemini-2.0-flash-001")
|
||||
|
||||
assert llm.client is not None
|
||||
|
||||
@@ -16,8 +16,8 @@ def setup_provider_api_keys(monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test-key-dummy")
|
||||
if "ANTHROPIC_API_KEY" not in os.environ:
|
||||
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-key-dummy")
|
||||
if "GOOGLE_API_KEY" not in os.environ:
|
||||
monkeypatch.setenv("GOOGLE_API_KEY", "test-google-key-dummy")
|
||||
if "GEMINI_API_KEY" not in os.environ:
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "test-google-key-dummy")
|
||||
|
||||
|
||||
class DummyInterceptor(BaseInterceptor[httpx.Request, httpx.Response]):
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are Test Role. Test Backstory\nYour
|
||||
personal goal is: Test Goal\nTo give my best complete final answer to the task
|
||||
use the exact following format:\n\nThought: I now can give a great answer\nFinal
|
||||
Answer: Your final answer must be the great and the most complete as possible,
|
||||
it must be outcome described.\n\nI MUST use these formats, my job depends on
|
||||
it!"}, {"role": "user", "content": "\nCurrent Task: Return: Test output\n\nThis
|
||||
is the expect criteria for your final answer: Test output\nyou MUST return the
|
||||
actual complete content as the final answer, not a summary.\n\nBegin! This is
|
||||
VERY important to you, use the tools available and give your best Final Answer,
|
||||
your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '776'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=9.8sBYBkvBR8R1K_bVF7xgU..80XKlEIg3N2OBbTSCU-1727214102-1.0.1.1-.qiTLXbPamYUMSuyNsOEB9jhGu.jOifujOrx9E2JZvStbIZ9RTIiE44xKKNfLPxQkOi6qAT3h6htK8lPDGV_5g;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7fr4aPstiFUArxwxTVdfJSFwxsC\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727214471,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: Test output\",\n \"refusal\": null\n },\n \"logprobs\":
|
||||
null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":
|
||||
155,\n \"completion_tokens\": 15,\n \"total_tokens\": 170,\n \"completion_tokens_details\":
|
||||
{\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_52a7f40b0b\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85f9a91e311cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:47:51 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '216'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999817'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_88b1376917b345c976fdb03a55f7b6c1
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
0
lib/crewai/tests/rag/embeddings/__init__.py
Normal file
0
lib/crewai/tests/rag/embeddings/__init__.py
Normal file
0
lib/crewai/tests/rag/qdrant/__init__.py
Normal file
0
lib/crewai/tests/rag/qdrant/__init__.py
Normal file
@@ -45,7 +45,6 @@ def test_telemetry_enabled_by_default():
|
||||
assert telemetry.ready is True
|
||||
|
||||
|
||||
@pytest.mark.telemetry
|
||||
@patch("crewai.telemetry.telemetry.logger.error")
|
||||
@patch(
|
||||
"opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter.export",
|
||||
|
||||
@@ -9,7 +9,7 @@ from crewai.flow.persistence.sqlite import SQLiteFlowPersistence
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class TestState(FlowState):
|
||||
class State(FlowState):
|
||||
"""Test state model with required id field."""
|
||||
|
||||
counter: int = 0
|
||||
@@ -45,8 +45,8 @@ def test_structured_state_persistence(tmp_path):
|
||||
db_path = os.path.join(tmp_path, "test_flows.db")
|
||||
persistence = SQLiteFlowPersistence(db_path)
|
||||
|
||||
class StructuredFlow(Flow[TestState]):
|
||||
initial_state = TestState
|
||||
class StructuredFlow(Flow[State]):
|
||||
initial_state = State
|
||||
|
||||
@start()
|
||||
@persist(persistence)
|
||||
@@ -71,7 +71,7 @@ def test_flow_state_restoration(tmp_path):
|
||||
persistence = SQLiteFlowPersistence(db_path)
|
||||
|
||||
# First flow execution to create initial state
|
||||
class RestorableFlow(Flow[TestState]):
|
||||
class RestorableFlow(Flow[State]):
|
||||
@start()
|
||||
@persist(persistence)
|
||||
def set_message(self):
|
||||
@@ -109,7 +109,7 @@ def test_multiple_method_persistence(tmp_path):
|
||||
db_path = os.path.join(tmp_path, "test_flows.db")
|
||||
persistence = SQLiteFlowPersistence(db_path)
|
||||
|
||||
class MultiStepFlow(Flow[TestState]):
|
||||
class MultiStepFlow(Flow[State]):
|
||||
@start()
|
||||
@persist(persistence)
|
||||
def step_1(self):
|
||||
@@ -139,7 +139,7 @@ def test_multiple_method_persistence(tmp_path):
|
||||
assert final_state.counter == 2
|
||||
assert final_state.message == "Step 2"
|
||||
|
||||
class NoPersistenceMultiStepFlow(Flow[TestState]):
|
||||
class NoPersistenceMultiStepFlow(Flow[State]):
|
||||
@start()
|
||||
@persist(persistence)
|
||||
def step_1(self):
|
||||
|
||||
@@ -411,7 +411,6 @@ def test_context_window_exceeded_error_handling():
|
||||
assert "8192 tokens" in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.vcr()
|
||||
@pytest.fixture
|
||||
def anthropic_llm():
|
||||
"""Fixture providing an Anthropic LLM instance."""
|
||||
@@ -476,13 +475,13 @@ def test_anthropic_message_formatting(anthropic_llm, system_message, user_messag
|
||||
|
||||
|
||||
def test_deepseek_r1_with_open_router():
|
||||
if not os.getenv("OPEN_ROUTER_API_KEY"):
|
||||
pytest.skip("OPEN_ROUTER_API_KEY not set; skipping test.")
|
||||
if not os.getenv("OPENROUTER_API_KEY"):
|
||||
pytest.skip("OPENROUTER_API_KEY not set; skipping test.")
|
||||
|
||||
llm = LLM(
|
||||
model="openrouter/deepseek/deepseek-r1",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
api_key=os.getenv("OPEN_ROUTER_API_KEY"),
|
||||
api_key=os.getenv("OPENROUTER_API_KEY"),
|
||||
is_litellm=True,
|
||||
)
|
||||
result = llm.call("What is the capital of France?")
|
||||
@@ -742,7 +741,7 @@ def test_prefixed_models_with_valid_constants_use_native_sdk():
|
||||
assert llm2.provider == "anthropic"
|
||||
|
||||
# Test gemini/ prefix with Gemini model in constants → Native SDK
|
||||
with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-key"}):
|
||||
with patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}):
|
||||
llm3 = LLM(model="gemini/gemini-2.5-pro", is_litellm=False)
|
||||
assert llm3.is_litellm is False
|
||||
assert llm3.provider == "gemini"
|
||||
@@ -794,7 +793,7 @@ def test_unprefixed_models_use_native_sdk():
|
||||
assert llm2.provider == "anthropic"
|
||||
|
||||
# gemini-2.5-pro is in GEMINI_MODELS → Native Gemini SDK
|
||||
with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-key"}):
|
||||
with patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}):
|
||||
llm3 = LLM(model="gemini-2.5-pro", is_litellm=False)
|
||||
assert llm3.is_litellm is False
|
||||
assert llm3.provider == "gemini"
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: do you hate AI Agents?\n\nThis is the expect criteria for your final answer:
|
||||
Your best answer to your coworker asking you this, accounting for the context
|
||||
shared.\nyou MUST return the actual complete content as the final answer, not
|
||||
a summary.\n\nThis is the context you''re working with:\nI heard you LOVE them\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1021'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7WnyWZFoccBH9YB7ghLbR1L8Wqa\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213909,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: As an expert researcher specialized in technology, I don't harbor emotions
|
||||
such as hate towards AI agents. Instead, my focus is on understanding, analyzing,
|
||||
and leveraging their potential to advance various fields. AI agents, when designed
|
||||
and implemented effectively, can greatly augment human capabilities, streamline
|
||||
processes, and provide valuable insights that might otherwise be overlooked.
|
||||
My enthusiasm for AI agents stems from their ability to transform industries
|
||||
and improve everyday life, making complex tasks more manageable and enhancing
|
||||
overall efficiency. This passion drives my research and commitment to making
|
||||
meaningful contributions in the realm of AI and AI agents.\",\n \"refusal\":
|
||||
null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n
|
||||
\ }\n ],\n \"usage\": {\n \"prompt_tokens\": 199,\n \"completion_tokens\":
|
||||
126,\n \"total_tokens\": 325,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
|
||||
0\n }\n },\n \"system_fingerprint\": \"fp_e375328146\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ebf47e661cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:31 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '2498'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_b7e2cb0620e45d3d74310d3f0166551f
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -1,116 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: do you hate AI Agents?\n\nThis is the expect criteria for your final answer:
|
||||
Your best answer to your coworker asking you this, accounting for the context
|
||||
shared.\nyou MUST return the actual complete content as the final answer, not
|
||||
a summary.\n\nThis is the context you''re working with:\nI heard you LOVE them\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1021'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7Wy6aW1XM0lWaMyQUNB9qhbCZlH\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213920,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: As an expert researcher specializing in technology and AI, I have a
|
||||
deep appreciation for AI agents. These advanced tools have the potential to
|
||||
revolutionize countless industries by improving efficiency, accuracy, and decision-making
|
||||
processes. They can augment human capabilities, handle mundane and repetitive
|
||||
tasks, and even offer insights that might be beyond human reach. While it's
|
||||
crucial to approach AI with a balanced perspective, understanding both its capabilities
|
||||
and limitations, my stance is one of optimism and fascination. Properly developed
|
||||
and ethically managed, AI agents hold immense promise for driving innovation
|
||||
and solving complex problems. So yes, I do love AI agents for their transformative
|
||||
potential and the positive impact they can have on society.\",\n \"refusal\":
|
||||
null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n
|
||||
\ }\n ],\n \"usage\": {\n \"prompt_tokens\": 199,\n \"completion_tokens\":
|
||||
146,\n \"total_tokens\": 345,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
|
||||
0\n }\n },\n \"system_fingerprint\": \"fp_e375328146\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ec3c6f3b1cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:42 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '1675'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_a249567d37ada11bc8857404338b24cc
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -1,114 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: do you hate AI Agents?\n\nThis is the expect criteria for your final answer:
|
||||
Your best answer to your coworker asking you this, accounting for the context
|
||||
shared.\nyou MUST return the actual complete content as the final answer, not
|
||||
a summary.\n\nThis is the context you''re working with:\nI heard you LOVE them\n\nBegin!
|
||||
This is VERY important to you, use the tools available and give your best Final
|
||||
Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1021'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7Wq7edXMCGJR1zDd2QoySLdo8mM\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213912,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: I don't hate AI agents; on the contrary, I find them fascinating and
|
||||
incredibly useful. Considering the rapid advancements in AI technology, these
|
||||
agents have the potential to revolutionize various industries by automating
|
||||
tasks, improving efficiency, and providing insights that were previously unattainable.
|
||||
My expertise in researching and analyzing AI and AI agents has allowed me to
|
||||
appreciate the intricate design and the vast possibilities they offer. Therefore,
|
||||
it's more accurate to say that I love AI agents for their potential to drive
|
||||
innovation and improve our daily lives.\",\n \"refusal\": null\n },\n
|
||||
\ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n
|
||||
\ \"usage\": {\n \"prompt_tokens\": 199,\n \"completion_tokens\": 116,\n
|
||||
\ \"total_tokens\": 315,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
|
||||
0\n }\n },\n \"system_fingerprint\": \"fp_e375328146\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ec05f8651cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:33 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '1739'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_d9e1e9458d5539061397a618345c27d4
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -1,132 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: share your take on AI Agents\n\nThis is the expect criteria for your final
|
||||
answer: Your best answer to your coworker asking you this, accounting for the
|
||||
context shared.\nyou MUST return the actual complete content as the final answer,
|
||||
not a summary.\n\nThis is the context you''re working with:\nI heard you hate
|
||||
them\n\nBegin! This is VERY important to you, use the tools available and give
|
||||
your best Final Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1027'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7WbKt7If02iTLuH5cJJjeYo9uDi\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213897,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: I understand why you might think I dislike AI agents, but my perspective
|
||||
is more nuanced. AI agents, in essence, are incredibly versatile tools designed
|
||||
to perform specific tasks autonomously or semi-autonomously. They harness various
|
||||
artificial intelligence techniques, such as machine learning, natural language
|
||||
processing, and computer vision, to interpret data, understand tasks, and execute
|
||||
them efficiently. \\n\\nFrom a technological standpoint, AI agents have revolutionized
|
||||
numerous industries. In customer service, for instance, AI agents like chatbots
|
||||
and virtual assistants handle customer inquiries 24/7, providing quick and efficient
|
||||
solutions. In healthcare, AI agents can assist in diagnosing diseases, managing
|
||||
patient data, and even predicting outbreaks. The automation capabilities of
|
||||
AI agents also enhance productivity in areas such as logistics, finance, and
|
||||
cybersecurity by identifying patterns and anomalies at speeds far beyond human
|
||||
capabilities.\\n\\nHowever, it's important to acknowledge the potential downsides
|
||||
and challenges associated with AI agents. Ethical considerations are paramount.
|
||||
Issues such as data privacy, security, and biases in AI algorithms need to be
|
||||
carefully managed. There is also the human aspect to consider\u2014over-reliance
|
||||
on AI agents might lead to job displacement in certain sectors, and ensuring
|
||||
a fair transition for affected workers is crucial.\\n\\nMy concerns generally
|
||||
stem from these ethical and societal implications rather than from the technology
|
||||
itself. I advocate for responsible AI development, which includes transparency,
|
||||
fairness, and accountability. By addressing these concerns, we can harness the
|
||||
full potential of AI agents while mitigating the associated risks.\\n\\nSo,
|
||||
to clarify, I don't hate AI agents; I recognize their immense potential and
|
||||
the significant benefits they bring to various fields. However, I am equally
|
||||
aware of the challenges they present and advocate for a balanced approach to
|
||||
their development and deployment.\",\n \"refusal\": null\n },\n
|
||||
\ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n
|
||||
\ \"usage\": {\n \"prompt_tokens\": 200,\n \"completion_tokens\": 359,\n
|
||||
\ \"total_tokens\": 559,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
|
||||
0\n }\n },\n \"system_fingerprint\": \"fp_3537616b13\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ebaa5c061cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:22 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '4928'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_761796305026b5adfbb5a6237f14e32a
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -1,136 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: share your take on AI Agents\n\nThis is the expect criteria for your final
|
||||
answer: Your best answer to your coworker asking you this, accounting for the
|
||||
context shared.\nyou MUST return the actual complete content as the final answer,
|
||||
not a summary.\n\nThis is the context you''re working with:\nI heard you hate
|
||||
them\n\nBegin! This is VERY important to you, use the tools available and give
|
||||
your best Final Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1027'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7Wh4RzroZdiwUNOc4oRRhwfdRzs\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213903,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: AI agents are essentially autonomous software programs that perform
|
||||
tasks or provide services on behalf of humans. They're built on complex algorithms
|
||||
and often leverage machine learning and neural networks to adapt and improve
|
||||
over time. \\n\\nIt's important to clarify that I don't \\\"hate\\\" AI agents,
|
||||
but I do approach them with a critical eye for a couple of reasons. AI agents
|
||||
have enormous potential to transform industries, making processes more efficient,
|
||||
providing insightful data analytics, and even learning from user behavior to
|
||||
offer personalized experiences. However, this potential comes with significant
|
||||
challenges and risks:\\n\\n1. **Ethical Concerns**: AI agents operate on data,
|
||||
and the biases present in data can lead to unfair or unethical outcomes. Ensuring
|
||||
that AI operates within ethical boundaries requires rigorous oversight, which
|
||||
is not always in place.\\n\\n2. **Privacy Issues**: AI agents often need access
|
||||
to large amounts of data, raising questions about privacy and data security.
|
||||
If not managed correctly, this can lead to unauthorized data access and potential
|
||||
misuse of sensitive information.\\n\\n3. **Transparency and Accountability**:
|
||||
The decision-making process of AI agents can be opaque, making it difficult
|
||||
to understand how they arrive at specific conclusions or actions. This lack
|
||||
of transparency poses challenges for accountability, especially if something
|
||||
goes wrong.\\n\\n4. **Job Displacement**: As AI agents become more capable,
|
||||
there are valid concerns about their impact on employment. Tasks that were traditionally
|
||||
performed by humans are increasingly being automated, which can lead to job
|
||||
loss in certain sectors.\\n\\n5. **Reliability**: While AI agents can outperform
|
||||
humans in many areas, they are not infallible. They can make mistakes, sometimes
|
||||
with serious consequences. Continuous monitoring and regular updates are essential
|
||||
to maintain their performance and reliability.\\n\\nIn summary, while AI agents
|
||||
offer substantial benefits and opportunities, it's critical to approach their
|
||||
adoption and deployment with careful consideration of the associated risks.
|
||||
Balancing innovation with responsibility is key to leveraging AI agents effectively
|
||||
and ethically. So, rather than \\\"hating\\\" AI agents, I advocate for a balanced,
|
||||
cautious approach that maximizes benefits while mitigating potential downsides.\",\n
|
||||
\ \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\":
|
||||
\"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 200,\n \"completion_tokens\":
|
||||
429,\n \"total_tokens\": 629,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
|
||||
0\n }\n },\n \"system_fingerprint\": \"fp_3537616b13\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ebcdae971cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:29 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '5730'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_5da5b18b3cee10548a217ba97e133815
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -1,137 +0,0 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages": [{"role": "system", "content": "You are researcher. You''re
|
||||
an expert researcher, specialized in technology\nYour personal goal is: make
|
||||
the best research and analysis on content about AI and AI agents\nTo give my
|
||||
best complete final answer to the task use the exact following format:\n\nThought:
|
||||
I now can give a great answer\nFinal Answer: Your final answer must be the great
|
||||
and the most complete as possible, it must be outcome described.\n\nI MUST use
|
||||
these formats, my job depends on it!"}, {"role": "user", "content": "\nCurrent
|
||||
Task: share your take on AI Agents\n\nThis is the expect criteria for your final
|
||||
answer: Your best answer to your coworker asking you this, accounting for the
|
||||
context shared.\nyou MUST return the actual complete content as the final answer,
|
||||
not a summary.\n\nThis is the context you''re working with:\nI heard you hate
|
||||
them\n\nBegin! This is VERY important to you, use the tools available and give
|
||||
your best Final Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1027'
|
||||
content-type:
|
||||
- application/json
|
||||
cookie:
|
||||
- __cf_bm=rb61BZH2ejzD5YPmLaEJqI7km71QqyNJGTVdNxBq6qk-1727213194-1.0.1.1-pJ49onmgX9IugEMuYQMralzD7oj_6W.CHbSu4Su1z3NyjTGYg.rhgJZWng8feFYah._oSnoYlkTjpK1Wd2C9FA;
|
||||
_cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.47.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.47.0
|
||||
x-stainless-raw-response:
|
||||
- 'true'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.11.7
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
content: "{\n \"id\": \"chatcmpl-AB7Wsv05NzccAAGC0CZVg03mE72wi\",\n \"object\":
|
||||
\"chat.completion\",\n \"created\": 1727213914,\n \"model\": \"gpt-4o-2024-05-13\",\n
|
||||
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
|
||||
\"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal
|
||||
Answer: My perspective on AI agents is quite nuanced and not a matter of simple
|
||||
like or dislike. AI agents, depending on their design, deployment, and use cases,
|
||||
can bring about both significant benefits and substantial challenges.\\n\\nOn
|
||||
the positive side, AI agents have the potential to automate mundane tasks, enhance
|
||||
productivity, and provide personalized services in ways that were previously
|
||||
unimaginable. For instance, in customer service, AI agents can handle inquiries
|
||||
24/7, reducing waiting times and improving user satisfaction. In healthcare,
|
||||
they can assist in diagnosing diseases by analyzing vast datasets much faster
|
||||
than humans. These applications demonstrate the transformative power of AI in
|
||||
improving efficiency and delivering better outcomes across various industries.\\n\\nHowever,
|
||||
my reservations stem from several critical concerns. Firstly, there's the issue
|
||||
of reliability and accuracy. Mismanaged or poorly designed AI systems can lead
|
||||
to significant errors, which could be particularly detrimental in high-stakes
|
||||
environments like healthcare or autonomous vehicles. Second, there's a risk
|
||||
of job displacement as AI agents become capable of performing tasks traditionally
|
||||
done by humans. This raises socio-economic concerns that need to be addressed
|
||||
through effective policy-making and upskilling programs.\\n\\nAdditionally,
|
||||
there are ethical and privacy considerations. AI agents often require large
|
||||
amounts of data to function effectively, which can lead to issues concerning
|
||||
consent, data security, and individual privacy rights. The lack of transparency
|
||||
in how these agents make decisions can also pose challenges\u2014this is often
|
||||
referred to as the \\\"black box\\\" problem, where even the developers may
|
||||
not fully understand how specific AI outputs are generated.\\n\\nFinally, the
|
||||
deployment of AI agents by bad actors for malicious purposes, such as deepfakes,
|
||||
misinformation, and hacking, remains a pertinent concern. These potential downsides
|
||||
imply that while AI technology is extremely powerful and promising, it must
|
||||
be developed and implemented with care, consideration, and robust ethical guidelines.\\n\\nSo,
|
||||
in summary, I don't hate AI agents\u2014rather, I approach them critically with
|
||||
a balanced perspective, recognizing both their profound potential and the significant
|
||||
challenges they present. Thoughtful development, responsible deployment, and
|
||||
ethical governance are crucial to harness the benefits while mitigating the
|
||||
risks associated with AI agents.\",\n \"refusal\": null\n },\n \"logprobs\":
|
||||
null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":
|
||||
200,\n \"completion_tokens\": 436,\n \"total_tokens\": 636,\n \"completion_tokens_details\":
|
||||
{\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_3537616b13\"\n}\n"
|
||||
headers:
|
||||
CF-Cache-Status:
|
||||
- DYNAMIC
|
||||
CF-RAY:
|
||||
- 8c85ec12ab0d1cf3-GRU
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 24 Sep 2024 21:38:40 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
openai-organization:
|
||||
- crewai-iuxna1
|
||||
openai-processing-ms:
|
||||
- '6251'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-ratelimit-limit-requests:
|
||||
- '10000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '30000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '9999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '29999755'
|
||||
x-ratelimit-reset-requests:
|
||||
- 6ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_50aa23cad48cfb83b754a5a92939638e
|
||||
http_version: HTTP/1.1
|
||||
status_code: 200
|
||||
version: 1
|
||||
@@ -5,18 +5,18 @@ from crewai.events.base_events import BaseEvent
|
||||
from crewai.events.event_bus import crewai_event_bus
|
||||
|
||||
|
||||
class TestEvent(BaseEvent):
|
||||
class Event(BaseEvent):
|
||||
pass
|
||||
|
||||
|
||||
def test_specific_event_handler():
|
||||
mock_handler = Mock()
|
||||
|
||||
@crewai_event_bus.on(TestEvent)
|
||||
@crewai_event_bus.on(Event)
|
||||
def handler(source, event):
|
||||
mock_handler(source, event)
|
||||
|
||||
event = TestEvent(type="test_event")
|
||||
event = Event(type="test_event")
|
||||
crewai_event_bus.emit("source_object", event)
|
||||
|
||||
mock_handler.assert_called_once_with("source_object", event)
|
||||
@@ -27,15 +27,15 @@ def test_multiple_handlers_same_event():
|
||||
mock_handler1 = Mock()
|
||||
mock_handler2 = Mock()
|
||||
|
||||
@crewai_event_bus.on(TestEvent)
|
||||
@crewai_event_bus.on(Event)
|
||||
def handler1(source, event):
|
||||
mock_handler1(source, event)
|
||||
|
||||
@crewai_event_bus.on(TestEvent)
|
||||
@crewai_event_bus.on(Event)
|
||||
def handler2(source, event):
|
||||
mock_handler2(source, event)
|
||||
|
||||
event = TestEvent(type="test_event")
|
||||
event = Event(type="test_event")
|
||||
crewai_event_bus.emit("source_object", event)
|
||||
|
||||
mock_handler1.assert_called_once_with("source_object", event)
|
||||
@@ -47,16 +47,16 @@ def test_event_bus_error_handling():
|
||||
called = threading.Event()
|
||||
error_caught = threading.Event()
|
||||
|
||||
@crewai_event_bus.on(TestEvent)
|
||||
@crewai_event_bus.on(Event)
|
||||
def broken_handler(source, event):
|
||||
called.set()
|
||||
raise ValueError("Simulated handler failure")
|
||||
|
||||
@crewai_event_bus.on(TestEvent)
|
||||
@crewai_event_bus.on(Event)
|
||||
def working_handler(source, event):
|
||||
error_caught.set()
|
||||
|
||||
event = TestEvent(type="test_event")
|
||||
event = Event(type="test_event")
|
||||
crewai_event_bus.emit("source_object", event)
|
||||
|
||||
assert called.wait(timeout=2), "Broken handler was never called"
|
||||
|
||||
@@ -730,7 +730,6 @@ def test_llm_emits_call_started_event():
|
||||
|
||||
|
||||
@pytest.mark.vcr()
|
||||
@pytest.mark.isolated
|
||||
def test_llm_emits_call_failed_event():
|
||||
received_events = []
|
||||
event_received = threading.Event()
|
||||
|
||||
@@ -6,7 +6,7 @@ from crewai.utilities.training_converter import TrainingConverter
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class TestModel(BaseModel):
|
||||
class Model(BaseModel):
|
||||
string_field: str = Field(description="A simple string field")
|
||||
list_field: List[str] = Field(description="A list of strings")
|
||||
number_field: float = Field(description="A number field")
|
||||
@@ -20,7 +20,7 @@ class TestTrainingConverter:
|
||||
self.converter = TrainingConverter(
|
||||
llm=self.llm_mock,
|
||||
text=self.test_text,
|
||||
model=TestModel,
|
||||
model=Model,
|
||||
instructions=self.test_instructions,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user