diff --git a/AGENTS.md b/AGENTS.md index b68e48dbb..d2f796c82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,3 +24,5 @@ Follow these guidelines when contributing: may reference them. 4. If you want to preview your changes locally, use `cd docs && mintlify dev`. To check for broken links, run `cd docs && mintlify broken-links`. +5. After editing English docs, sync translations to `ar`, `ko`, and `pt-BR` + before finishing the task. Follow [DOCS_TRANSLATIONS.md](DOCS_TRANSLATIONS.md). diff --git a/DOCS_TRANSLATIONS.md b/DOCS_TRANSLATIONS.md new file mode 100644 index 000000000..58a04b54f --- /dev/null +++ b/DOCS_TRANSLATIONS.md @@ -0,0 +1,81 @@ +# Sync Docs Translations + +After English documentation changes, sync the same updates to Arabic (`ar`), +Korean (`ko`), and Brazilian Portuguese (`pt-BR`). + +Supported locales: `ar`, `ko`, `pt-BR`. + +## Step 1 — Find changed English files with git + +From the repo root: + +```bash +# Uncommitted changes (staged or unstaged) +git diff --name-only HEAD -- docs/edge/en/ + +# All changes on this branch vs main +git diff --name-only main...HEAD -- docs/edge/en/ + +# Newly added files +git status --porcelain docs/edge/en/ +``` + +Only process `*.mdx` under `docs/edge/en/`. Do not edit `docs/v*/` snapshots. + +## Step 2 — Map each file to locale targets + +For `docs/edge/en/.mdx`, update or create: + +- `docs/edge/ar/.mdx` +- `docs/edge/ko/.mdx` +- `docs/edge/pt-BR/.mdx` + +If English is a **new page**, also add matching entries in `docs/docs.json` +navigation for each locale. + +## Step 3 — Translate + +Use the updated English file as source of truth. When locale files already +exist, apply the same semantic change — do not rewrite unrelated sections. + +Rules: + +- Translate prose and frontmatter values (`title`, `description`, `sidebarTitle`) +- Keep MDX/JSX tags, code blocks, URLs, and identifiers unchanged +- Keep terms like Agent, Crew, Task, Flow, LLM, API, CLI, MCP in English where + appropriate +- Rewrite internal links: `/en/` → `/{lang}/` (`/ar/`, `/ko/`, `/pt-BR/`) +- Do not add translator notes + +## Step 4 — Verify (optional) + +```bash +cd docs && mintlify broken-links +``` + +Commit English and locale files together. + +## Checklist + +```markdown +- [ ] Git: listed changed docs/edge/en/*.mdx files +- [ ] ar: updated/created matching files +- [ ] ko: updated/created matching files +- [ ] pt-BR: updated/created matching files +- [ ] Links use /{lang}/ prefix; code blocks unchanged +- [ ] docs/docs.json updated if new English page added +``` + +## Example + +`git diff --name-only HEAD -- docs/edge/en/` returns: + +```text +docs/edge/en/concepts/llms.mdx +``` + +Update: + +- `docs/edge/ar/concepts/llms.mdx` +- `docs/edge/ko/concepts/llms.mdx` +- `docs/edge/pt-BR/concepts/llms.mdx` diff --git a/README.md b/README.md index f0e4b08d9..94977f4a3 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ standard for production-ready agentic automation. # CrewAI AMP Suite -For organizations that need a commercial control plane around CrewAI, [CrewAI AMP Suite](https://www.crewai.com/enterprise) adds managed deployment, observability, governance, security, and enterprise support. +For organizations that need a commercial control plane around CrewAI, [CrewAI AMP Suite](https://crewai.com/amp) adds managed deployment, observability, governance, security, and enterprise support. You can try one part of the suite, the [Crew Control Plane, for free](https://app.crewai.com). @@ -88,8 +88,12 @@ intelligent automations. - [Build with AI](#build-with-ai) - [Why CrewAI?](#why-crewai) - [Getting Started](#getting-started) + - [Learning Resources](#learning-resources) + - [Understanding Flows and Crews](#understanding-flows-and-crews) + - [Installation](#1-installation) + - [Setting Up Your Crew](#2-setting-up-your-crew-with-the-yaml-configuration) + - [Running Your Crew](#3-running-your-crew) - [Key Features](#key-features) -- [Understanding Flows and Crews](#understanding-flows-and-crews) - [Examples](#examples) - [Quick Tutorial](#quick-tutorial) - [Write Job Descriptions](#write-job-descriptions) @@ -151,9 +155,7 @@ Setup and run your first CrewAI agents by following this tutorial. [![CrewAI Getting Started Tutorial](https://img.youtube.com/vi/-kSOTtYzgEw/hqdefault.jpg)](https://www.youtube.com/watch?v=-kSOTtYzgEw "CrewAI Getting Started Tutorial") -### - -Learning Resources +### Learning Resources Learn CrewAI through our comprehensive courses: @@ -307,7 +309,7 @@ research_task: description: > Conduct a thorough research about {topic} Make sure you find any interesting and relevant information given - the current year is 2025. + the current year is 2026. expected_output: > A list with 10 bullet points of the most relevant information about {topic} agent: researcher @@ -317,7 +319,7 @@ reporting_task: Review the context you got and expand each topic into a full section for a report. Make sure the report is detailed and contains any and all relevant information. expected_output: > - A fully fledge reports with the mains topics, each with a full section of information. + A fully fledged report with the main topics, each with a full section of information. Formatted as markdown without '```' agent: reporting_analyst output_file: report.md @@ -430,7 +432,7 @@ crewai update You should see the output in the console and the `report.md` file should be created in the root of your project with the full final report. -In addition to the sequential process, you can use the hierarchical process, which automatically assigns a manager to the defined crew to properly coordinate the planning and execution of tasks through delegation and validation of results. [See more about the processes here](https://docs.crewai.com/core-concepts/Processes/). +In addition to the sequential process, you can use the hierarchical process, which automatically assigns a manager to the defined crew to properly coordinate the planning and execution of tasks through delegation and validation of results. [See more about the processes here](https://docs.crewai.com/en/concepts/processes). ## Key Features @@ -451,7 +453,7 @@ Choose CrewAI to build powerful, adaptable, and production-ready AI automations. You can test different real life examples of AI crews in the [CrewAI-examples repo](https://github.com/crewAIInc/crewAI-examples?tab=readme-ov-file): - [Landing Page Generator](https://github.com/crewAIInc/crewAI-examples/tree/main/crews/landing_page_generator) -- [Having Human input on the execution](https://docs.crewai.com/how-to/Human-Input-on-Execution) +- [Having Human input on the execution](https://docs.crewai.com/en/learn/human-input-on-execution) - [Trip Planner](https://github.com/crewAIInc/crewAI-examples/tree/main/crews/trip_planner) - [Stock Analysis](https://github.com/crewAIInc/crewAI-examples/tree/main/crews/stock_analysis) @@ -483,7 +485,7 @@ CrewAI's power truly shines when combining Crews with Flows to create sophistica CrewAI flows support logical operators like `or_` and `and_` to combine multiple conditions. This can be used with `@start`, `@listen`, or `@router` decorators to create complex triggering conditions. - `or_`: Triggers when any of the specified conditions are met. -- `and_`Triggers when all of the specified conditions are met. +- `and_`: Triggers when all of the specified conditions are met. Here's how you can orchestrate multiple Crews within a Flow: @@ -580,7 +582,7 @@ This example demonstrates how to: CrewAI supports using various LLMs through a variety of connection options. By default your agents will use the OpenAI API when querying the model. However, there are several other ways to allow your agents to connect to models. For example, you can configure your agents to use a local model via the Ollama tool. -Please refer to the [Connect CrewAI to LLMs](https://docs.crewai.com/how-to/LLM-Connections/) page for details on configuring your agents' connections to models. +Please refer to the [Connect CrewAI to LLMs](https://docs.crewai.com/en/learn/llm-connections) page for details on configuring your agents' connections to models. ## When to Use CrewAI @@ -596,13 +598,26 @@ CrewAI is especially useful when you want to: ## Contribution -CrewAI is open-source and we welcome contributions. If you're looking to contribute, please: +CrewAI is open-source and we welcome contributions. See +[`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md) for the full setup guide, +branching conventions, and PR checklist. -- Fork the repository. -- Create a new branch for your feature. -- Add your feature or improvement. -- Send a pull request. -- We appreciate your input! +Quick start: + +```bash +git clone https://github.com/crewAIInc/crewAI.git +cd crewAI +uv sync --all-groups --all-extras +uv run pre-commit install +``` + +```bash +# Tests +uv run pytest lib/crewai/tests/ -x -q + +# Type checks +uv run mypy lib/ +``` ### Contributing to the docs @@ -614,51 +629,8 @@ immediately and are frozen into a new versioned snapshot under `docs/v/` at the next release cut. Frozen snapshots are immutable — CI rejects PRs that modify them without a `[docs-freeze]` title prefix. The release CLI (`devtools release`) handles the freeze automatically; see -[`AGENTS.md`](AGENTS.md) for the full contributor guide and -[`RELEASING.md`](RELEASING.md) for the release-cut runbook. - -### Installing Dependencies - -```bash -uv lock -uv sync -``` - -### Virtual Env - -```bash -uv venv -``` - -### Pre-commit hooks - -```bash -pre-commit install -``` - -### Running Tests - -```bash -uv run pytest . -``` - -### Running static type checks - -```bash -uvx mypy src -``` - -### Packaging - -```bash -uv build -``` - -### Installing Locally - -```bash -uv pip install dist/*.tar.gz -``` +[`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md) for contributor guidance and +[`lib/devtools/README.md`](lib/devtools/README.md) for release tooling. ## Telemetry @@ -729,7 +701,7 @@ A: CrewAI is a lean, fast Python framework built specifically for orchestrating ### Q: How do I install CrewAI? -A: Install CrewAI using pip: +A: Install CrewAI with [UV](https://docs.astral.sh/uv/): ```shell uv pip install crewai @@ -751,7 +723,7 @@ A: Yes. CrewAI excels at both simple and highly complex real-world scenarios, of ### Q: Can I use CrewAI with local AI models? -A: Absolutely! CrewAI supports various language models, including local ones. Tools like Ollama and LM Studio allow seamless integration. Check the [LLM Connections documentation](https://docs.crewai.com/how-to/LLM-Connections/) for more details. +A: Absolutely! CrewAI supports various language models, including local ones. Tools like Ollama and LM Studio allow seamless integration. Check the [LLM Connections documentation](https://docs.crewai.com/en/learn/llm-connections) for more details. ### Q: What makes Crews different from Flows? @@ -771,7 +743,7 @@ A: Check out practical examples in the [CrewAI-examples repository](https://gith ### Q: How can I contribute to CrewAI? -A: Contributions are warmly welcomed! Fork the repository, create your branch, implement your changes, and submit a pull request. See the Contribution section of the README for detailed guidelines. +A: Contributions are warmly welcomed! Fork the repository, create your branch, implement your changes, and submit a pull request. See [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md) for detailed guidelines. ### Q: What additional features does CrewAI AMP offer? diff --git a/docs/edge/ar/concepts/crews.mdx b/docs/edge/ar/concepts/crews.mdx index 05f643c90..178129690 100644 --- a/docs/edge/ar/concepts/crews.mdx +++ b/docs/edge/ar/concepts/crews.mdx @@ -172,6 +172,8 @@ class YourCrewName: بعد تنفيذ الطاقم، يمكنك الوصول إلى خاصية `usage_metrics` لعرض مقاييس استخدام نموذج اللغة (LLM) لجميع المهام المنفذة. +`total_tokens` هو الإجمالي المفوتر (`prompt_tokens + completion_tokens`). حقول التفصيل مثل `cached_prompt_tokens` و`cache_creation_tokens` تصف أجزاءً مُدرجة بالفعل ضمن تلك الإجماليات ولا تُضاف مرة أخرى إلى `total_tokens`. راجع قسم **UsageMetrics field semantics** في توثيق مفهوم Flows للحصول على العقد الكامل. + ```python Code crew = Crew(agents=[agent1, agent2], tasks=[task1, task2]) crew.kickoff() diff --git a/docs/edge/ar/concepts/flows.mdx b/docs/edge/ar/concepts/flows.mdx index 62d34b335..45d54114d 100644 --- a/docs/edge/ar/concepts/flows.mdx +++ b/docs/edge/ar/concepts/flows.mdx @@ -266,6 +266,24 @@ print(flow.usage_metrics) كلما احتجت إلى الإجمالي **الكامل** للتوكنات لتنفيذ التدفق. +### دلالات حقول UsageMetrics + +يستخدم كائن [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) المُعاد عقدًا محايدًا للمزود: + +| الحقل | المعنى | +| --- | --- | +| `total_tokens` | الإجمالي المفوتر: `prompt_tokens + completion_tokens` | +| `prompt_tokens` | إجمالي رموز الإدخال/المطالبة المفوترة للطلب | +| `completion_tokens` | رموز الإخراج/الإكمال المفوترة للطلب | +| `cached_prompt_tokens` | جزء قراءة الذاكرة المؤقتة من رموز المطالبة (تفصيل فقط) | +| `cache_creation_tokens` | جزء كتابة الذاكرة المؤقتة من رموز المطالبة (تفصيل فقط، Anthropic) | +| `reasoning_tokens` | جزء التفكير/الاستدلال حيث يبلّغ المزود عنه بشكل منفصل (تفصيل فقط) | +| `successful_requests` | عدد استدعاءات LLM المُجمّعة | + +حقول التفصيل مثل `cached_prompt_tokens` و`cache_creation_tokens` و`reasoning_tokens` **لا تُضاف** فوق `total_tokens` — بل تصف أجزاءً مُدرجة بالفعل ضمن `prompt_tokens` أو `completion_tokens`. + +بالنسبة إلى Anthropic، تُدمج عدادات قراءة وكتابة الذاكرة المؤقتة ضمن `prompt_tokens`، لذا تنعكس أعباء العمل المخزنة مؤقتًا بالكامل في `total_tokens`. يُدرج مزودو OpenAI الرموز المخزنة مؤقتًا بالفعل داخل `prompt_tokens`؛ يعرض CrewAI الجزء المخزن مؤقتًا بشكل منفصل للوضوح. + كل حقل في [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) المُعاد هو مجموع جميع استدعاءات نموذج اللغة التي حدثت خلال استدعاء واحد لـ `flow.kickoff()`. تتم إعادة تعيين العدادات عند الاستدعاء التالي لـ `kickoff()` (وفي كل تكرار من `kickoff_for_each`)، لذلك لن تتكرر العدّات عبر التشغيلات المتتالية. يمكن قراءة هذه الخاصية بأمان في أي وقت بعد اكتمال `kickoff()`؛ قراءتها أثناء التنفيذ تُرجع المجموع الجزئي المتراكم حتى تلك اللحظة. ## إدارة حالة التدفق diff --git a/docs/edge/ar/concepts/llms.mdx b/docs/edge/ar/concepts/llms.mdx index a0ec6a26d..a2b3d9653 100644 --- a/docs/edge/ar/concepts/llms.mdx +++ b/docs/edge/ar/concepts/llms.mdx @@ -392,6 +392,22 @@ mode: "wide" - تتبع استخدام الرموز - محادثات استخدام أدوات متعددة الأدوار + **استخدام الرموز والتخزين المؤقت للمطالبة:** + + يُبلّغ Anthropic عن الإدخال المفوتر في عدادات منفصلة — `input_tokens` (غير المخزن مؤقتًا)، و`cache_read_input_tokens`، و`cache_creation_input_tokens`. يدمج CrewAI الثلاثة ضمن `prompt_tokens` (و`input_tokens` الأصلي في استجابات المزود) بحيث يعكس `total_tokens` الاستخدام المفوتر الكامل على أعباء العمل المخزنة مؤقتًا. + + يسجّل `cached_prompt_tokens` جزء قراءة الذاكرة المؤقتة كتفصيل فقط؛ وهو مُدرج بالفعل ضمن `prompt_tokens` ولا يجب إضافته مرة أخرى إلى `total_tokens`. يسجّل `cache_creation_tokens` عمليات الكتابة في الذاكرة المؤقتة بنفس الطريقة. + + ```python Code + usage = llm.get_token_usage_summary() + # total_tokens == prompt_tokens + completion_tokens + # prompt_tokens includes cache read + cache write for Anthropic + ``` + + راجع قسم **UsageMetrics field semantics** في توثيق مفهوم Flows + للحصول على العقد المحايد للمزود المستخدم في `crew.usage_metrics` + و`flow.usage_metrics`. + **ملاحظات مهمة:** - `max_tokens` معامل **مطلوب** لجميع نماذج Anthropic - يستخدم Claude `stop_sequences` بدلاً من `stop` diff --git a/docs/edge/en/concepts/crews.mdx b/docs/edge/en/concepts/crews.mdx index 2c43b63c7..2e4f21032 100644 --- a/docs/edge/en/concepts/crews.mdx +++ b/docs/edge/en/concepts/crews.mdx @@ -322,6 +322,8 @@ Caches can be employed to store the results of tools' execution, making the proc After the crew execution, you can access the `usage_metrics` attribute to view the language model (LLM) usage metrics for all tasks executed by the crew. This provides insights into operational efficiency and areas for improvement. +`total_tokens` is the billed total (`prompt_tokens + completion_tokens`). Breakdown fields such as `cached_prompt_tokens` and `cache_creation_tokens` describe subsets already included in those totals and are not added on top of `total_tokens`. See the **UsageMetrics field semantics** section in the Flows concept documentation for the full contract. + ```python Code # Access the crew's usage metrics crew = Crew(agents=[agent1, agent2], tasks=[task1, task2]) diff --git a/docs/edge/en/concepts/flows.mdx b/docs/edge/en/concepts/flows.mdx index 647512545..f4f0a95ff 100644 --- a/docs/edge/en/concepts/flows.mdx +++ b/docs/edge/en/concepts/flows.mdx @@ -267,7 +267,27 @@ print(flow.usage_metrics) execution. -Each entry in the returned [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) is the sum across all LLM calls made within a single `flow.kickoff()` invocation. Counters reset on the next `kickoff()` call (or on each iteration of `kickoff_for_each`), so successive runs don't double-count. The property is safe to read at any point after `kickoff()` completes; reading it during execution returns the partial total accumulated so far. +### UsageMetrics field semantics + +The returned [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) object uses a provider-neutral contract: + +| Field | Meaning | +| --- | --- | +| `total_tokens` | Billed total: `prompt_tokens + completion_tokens` | +| `prompt_tokens` | Full input/prompt tokens billed for the request | +| `completion_tokens` | Output/completion tokens billed for the request | +| `cached_prompt_tokens` | Cache-read subset of prompt tokens (breakdown only) | +| `cache_creation_tokens` | Cache-write subset of prompt tokens (breakdown only, Anthropic) | +| `reasoning_tokens` | Reasoning/thinking subset where the provider reports it separately (breakdown only) | +| `successful_requests` | Number of LLM calls aggregated | + +Breakdown fields such as `cached_prompt_tokens`, `cache_creation_tokens`, and +`reasoning_tokens` are **not** added on top of `total_tokens` — they describe +portions already included in `prompt_tokens` or `completion_tokens`. + +For Anthropic, cache read and cache write counters are folded into `prompt_tokens`, so cached workloads are fully reflected in `total_tokens`. OpenAI-style providers already include cached input inside `prompt_tokens`; CrewAI surfaces the cached portion separately for visibility. + +Each entry in the returned `UsageMetrics` is the sum across all LLM calls made within a single `flow.kickoff()` invocation. Counters reset on the next `kickoff()` call (or on each iteration of `kickoff_for_each`), so successive runs don't double-count. The property is safe to read at any point after `kickoff()` completes; reading it during execution returns the partial total accumulated so far. ## Flow State Management diff --git a/docs/edge/en/concepts/llms.mdx b/docs/edge/en/concepts/llms.mdx index 3c4470644..02fb97314 100644 --- a/docs/edge/en/concepts/llms.mdx +++ b/docs/edge/en/concepts/llms.mdx @@ -418,6 +418,27 @@ In this section, you'll find detailed examples that help you select, configure, - Token usage tracking - Multi-turn tool use conversations + **Token usage and prompt caching:** + + Anthropic reports billed input in separate counters — `input_tokens` (uncached), + `cache_read_input_tokens`, and `cache_creation_input_tokens`. CrewAI folds all + three into `prompt_tokens` (and native `input_tokens` in provider responses) so + `total_tokens` reflects full billed usage on cached workloads. + + `cached_prompt_tokens` records the cache-read portion as a breakdown only; it is + already included in `prompt_tokens` and must not be added again to + `total_tokens`. `cache_creation_tokens` records cache writes the same way. + + ```python Code + usage = llm.get_token_usage_summary() + # total_tokens == prompt_tokens + completion_tokens + # prompt_tokens includes cache read + cache write for Anthropic + ``` + + See the **UsageMetrics field semantics** section in the Flows concept + documentation for the provider-neutral contract used by `crew.usage_metrics` + and `flow.usage_metrics`. + **Important Notes:** - `max_tokens` is a **required** parameter for all Anthropic models - Claude uses `stop_sequences` instead of `stop` diff --git a/docs/edge/ko/concepts/crews.mdx b/docs/edge/ko/concepts/crews.mdx index e4cfadc1c..d62f76243 100644 --- a/docs/edge/ko/concepts/crews.mdx +++ b/docs/edge/ko/concepts/crews.mdx @@ -324,6 +324,8 @@ crew는 메모리(단기, 장기 및 엔티티 메모리)를 활용하여 시간 crew 실행 후, `usage_metrics` 속성에 접근하여 crew가 실행한 모든 작업에 대한 언어 모델(LLM) 사용 메트릭을 확인할 수 있습니다. 이를 통해 운영 효율성과 개선이 필요한 영역에 대한 인사이트를 얻을 수 있습니다. +`total_tokens`는 청구된 총합(`prompt_tokens + completion_tokens`)입니다. `cached_prompt_tokens` 및 `cache_creation_tokens`와 같은 breakdown 필드는 이미 해당 총합에 포함된 부분 집합을 설명하며 `total_tokens` 위에 다시 더하지 않습니다. 전체 계약은 Flows 개념 문서의 **UsageMetrics field semantics** 섹션을 참조하세요. + ```python Code # Access the crew's usage metrics crew = Crew(agents=[agent1, agent2], tasks=[task1, task2]) diff --git a/docs/edge/ko/concepts/flows.mdx b/docs/edge/ko/concepts/flows.mdx index e168b7e3f..5c2e21d5a 100644 --- a/docs/edge/ko/concepts/flows.mdx +++ b/docs/edge/ko/concepts/flows.mdx @@ -261,6 +261,24 @@ print(flow.usage_metrics) **전체** 토큰 집계가 필요할 때는 항상 `flow.usage_metrics`를 사용하십시오. +### UsageMetrics 필드 의미 + +반환되는 [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) 객체는 제공자 중립 계약을 사용합니다: + +| 필드 | 의미 | +| --- | --- | +| `total_tokens` | 청구된 총합: `prompt_tokens + completion_tokens` | +| `prompt_tokens` | 요청에 대해 청구된 전체 입력/프롬프트 토큰 | +| `completion_tokens` | 요청에 대해 청구된 출력/완료 토큰 | +| `cached_prompt_tokens` | 프롬프트 토큰 중 캐시 읽기 부분 집합 (breakdown 전용) | +| `cache_creation_tokens` | 프롬프트 토큰 중 캐시 쓰기 부분 집합 (breakdown 전용, Anthropic) | +| `reasoning_tokens` | 제공자가 별도로 보고하는 추론/사고 부분 집합 (breakdown 전용) | +| `successful_requests` | 집계된 LLM 호출 수 | + +`cached_prompt_tokens`, `cache_creation_tokens`, `reasoning_tokens`와 같은 breakdown 필드는 `total_tokens` **위에 추가되지 않습니다** — 이미 `prompt_tokens` 또는 `completion_tokens`에 포함된 부분을 설명합니다. + +Anthropic의 경우 캐시 읽기 및 쓰기 카운터가 `prompt_tokens`에 포함되므로, 캐시된 워크로드가 `total_tokens`에 완전히 반영됩니다. OpenAI 스타일 제공자는 캐시된 입력을 이미 `prompt_tokens`에 포함합니다. CrewAI는 가시성을 위해 캐시된 부분을 별도로 표시합니다. + 반환되는 [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py)의 각 항목은 단일 `flow.kickoff()` 실행 동안 발생한 모든 LLM 호출의 합계입니다. 다음 `kickoff()` 호출(및 `kickoff_for_each`의 각 반복)에서 카운터가 초기화되므로 연속 실행이 이중으로 집계되지 않습니다. 이 속성은 `kickoff()` 완료 후 언제든지 안전하게 읽을 수 있으며, 실행 중에 읽으면 그 시점까지 누적된 부분 합계를 반환합니다. ## 플로우 상태 관리 diff --git a/docs/edge/ko/concepts/llms.mdx b/docs/edge/ko/concepts/llms.mdx index 0945c6d50..760377ac1 100644 --- a/docs/edge/ko/concepts/llms.mdx +++ b/docs/edge/ko/concepts/llms.mdx @@ -270,6 +270,21 @@ CrewAI는 고유한 기능, 인증 방법, 모델 역량을 제공하는 다양 ) ``` + **토큰 사용량 및 프롬프트 캐싱:** + + Anthropic은 청구된 입력을 별도 카운터로 보고합니다 — `input_tokens`(캐시되지 않은 입력), `cache_read_input_tokens`, `cache_creation_input_tokens`. CrewAI는 세 값을 모두 `prompt_tokens`(및 제공자 응답의 네이티브 `input_tokens`)에 포함시켜 캐시된 워크로드에서 `total_tokens`가 전체 청구 사용량을 반영하도록 합니다. + + `cached_prompt_tokens`는 캐시 읽기 부분을 breakdown으로만 기록합니다. 이미 `prompt_tokens`에 포함되어 있으므로 `total_tokens`에 다시 더하면 안 됩니다. `cache_creation_tokens`도 캐시 쓰기를 같은 방식으로 기록합니다. + + ```python Code + usage = llm.get_token_usage_summary() + # total_tokens == prompt_tokens + completion_tokens + # prompt_tokens includes cache read + cache write for Anthropic + ``` + + `crew.usage_metrics` 및 `flow.usage_metrics`에 사용되는 제공자 중립 계약은 + Flows 개념 문서의 **UsageMetrics field semantics** 섹션을 참조하세요. + 현재 모델 ID와 기능은 Anthropic의 [모델 개요](https://platform.claude.com/docs/en/about-claude/models/overview)를 확인하고, 프로덕션에서 모델을 고정하기 전에 [모델 지원 중단 표](https://platform.claude.com/docs/en/about-claude/model-deprecations)를 검토하세요. diff --git a/docs/edge/pt-BR/concepts/crews.mdx b/docs/edge/pt-BR/concepts/crews.mdx index bb8298cde..a8317e525 100644 --- a/docs/edge/pt-BR/concepts/crews.mdx +++ b/docs/edge/pt-BR/concepts/crews.mdx @@ -322,6 +322,8 @@ Caches podem ser utilizados para armazenar resultados de execuções de ferramen Após a execução da crew, você pode acessar o atributo `usage_metrics` para visualizar as métricas de uso do modelo de linguagem (LLM) para todas as tasks executadas pela crew. Isso fornece insights sobre eficiência operacional e oportunidades de melhoria. +`total_tokens` é o total faturado (`prompt_tokens + completion_tokens`). Campos de breakdown como `cached_prompt_tokens` e `cache_creation_tokens` descrevem subconjuntos já incluídos nesses totais e não são somados novamente a `total_tokens`. Consulte a seção **UsageMetrics field semantics** na documentação do conceito Flows para o contrato completo. + ```python Code # Acessar as métricas de uso da crew crew = Crew(agents=[agent1, agent2], tasks=[task1, task2]) diff --git a/docs/edge/pt-BR/concepts/flows.mdx b/docs/edge/pt-BR/concepts/flows.mdx index 8879edca8..d70bb9774 100644 --- a/docs/edge/pt-BR/concepts/flows.mdx +++ b/docs/edge/pt-BR/concepts/flows.mdx @@ -260,6 +260,24 @@ print(flow.usage_metrics) rollup **completo** de tokens da execução do Flow. +### Semântica dos campos UsageMetrics + +O objeto [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) retornado usa um contrato neutro em relação ao provedor: + +| Campo | Significado | +| --- | --- | +| `total_tokens` | Total faturado: `prompt_tokens + completion_tokens` | +| `prompt_tokens` | Total de tokens de entrada/prompt faturados para a requisição | +| `completion_tokens` | Tokens de saída/conclusão faturados para a requisição | +| `cached_prompt_tokens` | Subconjunto de leitura de cache dos tokens de prompt (apenas breakdown) | +| `cache_creation_tokens` | Subconjunto de escrita de cache dos tokens de prompt (apenas breakdown, Anthropic) | +| `reasoning_tokens` | Subconjunto de raciocínio/pensamento quando o provedor reporta separadamente (apenas breakdown) | +| `successful_requests` | Número de chamadas LLM agregadas | + +Campos de breakdown como `cached_prompt_tokens`, `cache_creation_tokens` e `reasoning_tokens` **não** são somados sobre `total_tokens` — eles descrevem porções já incluídas em `prompt_tokens` ou `completion_tokens`. + +Para Anthropic, os contadores de leitura e escrita de cache são incorporados em `prompt_tokens`, de modo que workloads em cache são totalmente refletidos em `total_tokens`. Provedores no estilo OpenAI já incluem a entrada em cache dentro de `prompt_tokens`; o CrewAI expõe a porção em cache separadamente para visibilidade. + Cada campo do [`UsageMetrics`](https://github.com/crewAIInc/crewAI/blob/main/lib/crewai/src/crewai/types/usage_metrics.py) retornado representa a soma de todas as chamadas de LLM feitas em uma única invocação de `flow.kickoff()`. Os contadores são resetados a cada novo `kickoff()` (e em cada iteração de `kickoff_for_each`), de modo que execuções sucessivas não duplicam o total. A propriedade é segura para ser lida em qualquer momento após o `kickoff()`; lê-la durante a execução retorna o total parcial acumulado até aquele instante. ## Gerenciamento de Estado em Flows diff --git a/docs/edge/pt-BR/concepts/llms.mdx b/docs/edge/pt-BR/concepts/llms.mdx index efc12ed82..c4cf18ecf 100644 --- a/docs/edge/pt-BR/concepts/llms.mdx +++ b/docs/edge/pt-BR/concepts/llms.mdx @@ -270,6 +270,22 @@ Nesta seção, você encontrará exemplos detalhados que ajudam a selecionar, co ) ``` + **Uso de tokens e prompt caching:** + + A Anthropic reporta a entrada faturada em contadores separados — `input_tokens` (não em cache), `cache_read_input_tokens` e `cache_creation_input_tokens`. O CrewAI incorpora os três em `prompt_tokens` (e no `input_tokens` nativo nas respostas do provedor) para que `total_tokens` reflita o uso faturado completo em workloads em cache. + + `cached_prompt_tokens` registra a porção de leitura de cache apenas como breakdown; ela já está incluída em `prompt_tokens` e não deve ser somada novamente a `total_tokens`. `cache_creation_tokens` registra escritas de cache da mesma forma. + + ```python Code + usage = llm.get_token_usage_summary() + # total_tokens == prompt_tokens + completion_tokens + # prompt_tokens includes cache read + cache write for Anthropic + ``` + + Consulte a seção **UsageMetrics field semantics** na documentação do + conceito Flows para o contrato neutro em relação ao provedor usado por + `crew.usage_metrics` e `flow.usage_metrics`. + Consulte a [visão geral dos modelos](https://platform.claude.com/docs/en/about-claude/models/overview) da Anthropic para obter IDs e capacidades atuais e revise a [tabela de descontinuação](https://platform.claude.com/docs/en/about-claude/model-deprecations) antes de fixar um modelo em produção. diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index b0b5cd3a1..81807c920 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -2225,14 +2225,14 @@ class LLM(BaseLLM): ) return messages - provider = self.provider or self.model + formatter = self._multimodal_formatter_name() for msg in messages: files = msg.get("files") if not files: continue - content_blocks = format_multimodal_content(files, provider) + content_blocks = format_multimodal_content(files, formatter) if not content_blocks: msg.pop("files", None) continue @@ -2250,6 +2250,13 @@ class LLM(BaseLLM): return messages + def _multimodal_formatter_name(self) -> str: + # Identity (`self.provider`) stays e.g. anthropic. LiteLLM's completion() + # API is OpenAI-shaped and translates blocks to the vendor on the wire. + if self.is_litellm: + return "openai" + return self.provider or self.model + async def _aprocess_message_files( self, messages: list[LLMMessage] ) -> list[LLMMessage]: @@ -2276,14 +2283,14 @@ class LLM(BaseLLM): ) return messages - provider = self.provider or self.model + formatter = self._multimodal_formatter_name() for msg in messages: files = msg.get("files") if not files: continue - content_blocks = await aformat_multimodal_content(files, provider) + content_blocks = await aformat_multimodal_content(files, formatter) if not content_blocks: msg.pop("files", None) continue diff --git a/lib/crewai/src/crewai/llms/base_llm.py b/lib/crewai/src/crewai/llms/base_llm.py index a71126f58..3b40601bc 100644 --- a/lib/crewai/src/crewai/llms/base_llm.py +++ b/lib/crewai/src/crewai/llms/base_llm.py @@ -274,7 +274,10 @@ class BaseLLM(BaseModel, ABC): data["stop"] = list(stop) if not data.get("provider"): - data["provider"] = "openai" + model = data.get("model") or "" + data["provider"] = ( + cls._extract_provider(model) if isinstance(model, str) else "openai" + ) known_fields = set(cls.model_fields.keys()) extras = {k: v for k, v in data.items() if k not in known_fields} @@ -507,6 +510,10 @@ class BaseLLM(BaseModel, ABC): """ return False + def _multimodal_formatter_name(self) -> str: + # Content-block schema key for crewai_files. Identity stays on self.provider. + return self.provider or self.model + def format_text_content(self, text: str) -> dict[str, Any]: """Format text as a content block for the LLM. @@ -866,7 +873,7 @@ class BaseLLM(BaseModel, ABC): ) return messages - provider = getattr(self, "provider", None) or getattr(self, "model", "openai") + formatter = self._multimodal_formatter_name() api = getattr(self, "api", None) for msg in messages: @@ -878,7 +885,7 @@ class BaseLLM(BaseModel, ABC): text = existing_content if isinstance(existing_content, str) else None content_blocks = format_multimodal_content( - files, provider, api=api, prefer_upload=self.prefer_upload, text=text + files, formatter, api=api, prefer_upload=self.prefer_upload, text=text ) if not content_blocks: msg.pop("files", None) diff --git a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py index 7deecbf60..cdeef692a 100644 --- a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py +++ b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py @@ -12,6 +12,7 @@ from crewai.llms.base_llm import BaseLLM, JsonResponseFormat, llm_call_context from crewai.llms.hooks.base import BaseInterceptor from crewai.llms.hooks.transport import AsyncHTTPTransport, HTTPTransport from crewai.llms.providers.utils.common import safe_tool_conversion +from crewai.types.usage_metrics import _coerce_int from crewai.utilities.agent_utils import is_context_length_exceeded from crewai.utilities.exceptions.context_window_exceeding_exception import ( LLMContextLengthExceededError, @@ -1965,12 +1966,15 @@ class AnthropicCompletion(BaseLLM): """Extract token usage and response metadata from Anthropic response.""" if hasattr(response, "usage") and response.usage: usage = response.usage - input_tokens = getattr(usage, "input_tokens", 0) - output_tokens = getattr(usage, "output_tokens", 0) - cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 - cache_creation_tokens = ( - getattr(usage, "cache_creation_input_tokens", 0) or 0 + input_tokens = _coerce_int(getattr(usage, "input_tokens", 0)) + output_tokens = _coerce_int(getattr(usage, "output_tokens", 0)) + cache_read_tokens = _coerce_int( + getattr(usage, "cache_read_input_tokens", 0) ) + cache_creation_tokens = _coerce_int( + getattr(usage, "cache_creation_input_tokens", 0) + ) + input_tokens = input_tokens + cache_read_tokens + cache_creation_tokens result: dict[str, Any] = { "input_tokens": input_tokens, "output_tokens": output_tokens, diff --git a/lib/crewai/src/crewai/types/usage_metrics.py b/lib/crewai/src/crewai/types/usage_metrics.py index 891795d8a..7d58a47b3 100644 --- a/lib/crewai/src/crewai/types/usage_metrics.py +++ b/lib/crewai/src/crewai/types/usage_metrics.py @@ -108,6 +108,37 @@ class UsageMetrics(BaseModel): ), ) + @staticmethod + def _has_unreconciled_anthropic_cache_keys(usage_data: dict[str, Any]) -> bool: + """Detect raw Anthropic usage that still splits cache from ``input_tokens``. + + The native ``AnthropicCompletion`` provider folds cache read/creation + counters into ``input_tokens`` before usage reaches this normalizer. + LiteLLM and flow-level event aggregation can still deliver the raw + Anthropic API shape, where ``input_tokens`` is only the uncached + portion and cache counters arrive as separate keys. Without + reconciling here, ``prompt_tokens`` and ``total_tokens`` undercount + billed usage on cached Anthropic workloads. + """ + return "input_tokens" in usage_data and ( + "cache_read_input_tokens" in usage_data + or "cache_creation_input_tokens" in usage_data + ) + + @staticmethod + def _resolve_billed_prompt_tokens(usage_data: dict[str, Any]) -> int: + """Return the full billed prompt/input token count for a usage dict.""" + if UsageMetrics._has_unreconciled_anthropic_cache_keys(usage_data): + return ( + _coerce_int(usage_data.get("input_tokens")) + + _coerce_int(usage_data.get("cache_read_input_tokens")) + + _coerce_int(usage_data.get("cache_creation_input_tokens")) + ) + + return _first_int( + usage_data, "prompt_tokens", "prompt_token_count", "input_tokens" + ) + @classmethod def from_provider_dict(cls, usage_data: dict[str, Any] | None) -> Self | None: """Normalize a provider's raw usage dict into a ``UsageMetrics``. @@ -125,9 +156,7 @@ class UsageMetrics(BaseModel): if not usage_data: return None - prompt_tokens = _first_int( - usage_data, "prompt_tokens", "prompt_token_count", "input_tokens" - ) + prompt_tokens = cls._resolve_billed_prompt_tokens(usage_data) completion_tokens = _first_int( usage_data, "completion_tokens", @@ -145,12 +174,16 @@ class UsageMetrics(BaseModel): if isinstance(details, dict): cached_prompt_tokens = _coerce_int(details.get("cached_tokens")) + cache_creation_tokens = _coerce_int( + usage_data.get("cache_creation_tokens") + ) or _coerce_int(usage_data.get("cache_creation_input_tokens")) + return cls( total_tokens=prompt_tokens + completion_tokens, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cached_prompt_tokens=cached_prompt_tokens, reasoning_tokens=_coerce_int(usage_data.get("reasoning_tokens")), - cache_creation_tokens=_coerce_int(usage_data.get("cache_creation_tokens")), + cache_creation_tokens=cache_creation_tokens, successful_requests=1, ) diff --git a/lib/crewai/src/crewai/utilities/internal_instructor.py b/lib/crewai/src/crewai/utilities/internal_instructor.py index 36472482c..074f3a824 100644 --- a/lib/crewai/src/crewai/utilities/internal_instructor.py +++ b/lib/crewai/src/crewai/utilities/internal_instructor.py @@ -105,7 +105,12 @@ class InternalInstructor(Generic[T]): if value is not None: extra_kwargs[attr] = value - return instructor.from_provider(f"{provider}/{model_string}", **extra_kwargs) + qualified_model = ( + model_string + if not provider or model_string.startswith(f"{provider}/") + else f"{provider}/{model_string}" + ) + return instructor.from_provider(qualified_model, **extra_kwargs) def _extract_provider(self) -> str: """Extract provider from LLM model name. diff --git a/lib/crewai/tests/cassettes/utilities/test_llm_emits_call_failed_event.yaml b/lib/crewai/tests/cassettes/utilities/test_llm_emits_call_failed_event.yaml deleted file mode 100644 index e572b5b5d..000000000 --- a/lib/crewai/tests/cassettes/utilities/test_llm_emits_call_failed_event.yaml +++ /dev/null @@ -1,105 +0,0 @@ -interactions: -- request: - body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": - "gpt-4o-mini", "stop": []}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '102' - content-type: - - application/json - cookie: - - _cfuvid=IY8ppO70AMHr2skDSUsGh71zqHHdCQCZ3OvkPi26NBc-1740424913267-0.0.1.1-604800000; - __cf_bm=fU6K5KZoDmgcEuF8_yWAYKUO5fKHh6q5.wDPnna393g-1740424913-1.0.1.1-2iOaq3JVGWs439V0HxJee0IC9HdJm7dPkeJorD.AGw0YwkngRPM8rrTzn_7ht1BkbOauEezj.wPKcBz18gIYUg - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.61.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.61.0 - x-stainless-raw-response: - - 'true' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.12.8 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: "{\n \"id\": \"chatcmpl-B4YLA2SrC2rwdVQ3U87G5a0P5lsLw\",\n \"object\"\ - : \"chat.completion\",\n \"created\": 1740425016,\n \"model\": \"gpt-4o-mini-2024-07-18\"\ - ,\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \ - \ \"role\": \"assistant\",\n \"content\": \"Hello! I'm just a\ - \ computer program, so I don't have feelings, but I'm here and ready to help\ - \ you. How can I assist you today?\",\n \"refusal\": null\n },\n\ - \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n\ - \ \"usage\": {\n \"prompt_tokens\": 13,\n \"completion_tokens\": 30,\n\ - \ \"total_tokens\": 43,\n \"prompt_tokens_details\": {\n \"cached_tokens\"\ - : 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\"\ - : {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"\ - accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n\ - \ }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\":\ - \ \"fp_709714d124\"\n}\n" - headers: - CF-RAY: - - 9171d4c0ed44236e-SJC - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Mon, 24 Feb 2025 19:23:38 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - crewai-iuxna1 - openai-processing-ms: - - '1954' - openai-version: - - '2020-10-01' - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999978' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_ea2703502b8827e4297cd2a7bae9d9c8 - status: - code: 200 - message: OK -version: 1 diff --git a/lib/crewai/tests/events/test_llm_finish_reason_response_id.py b/lib/crewai/tests/events/test_llm_finish_reason_response_id.py index 091875fdf..5a1d4892d 100644 --- a/lib/crewai/tests/events/test_llm_finish_reason_response_id.py +++ b/lib/crewai/tests/events/test_llm_finish_reason_response_id.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch import pytest -from crewai.events.event_bus import CrewAIEventsBus +from crewai.events.event_bus import crewai_event_bus from crewai.events.types.llm_events import ( LLMCallCompletedEvent, LLMCallStartedEvent, @@ -31,7 +31,7 @@ class _StubLLM(BaseLLM): @pytest.fixture def mock_emit(): - with patch.object(CrewAIEventsBus, "emit") as mock: + with patch.object(crewai_event_bus, "emit") as mock: yield mock diff --git a/lib/crewai/tests/events/test_llm_usage_event.py b/lib/crewai/tests/events/test_llm_usage_event.py index d0b29c863..4e8f93461 100644 --- a/lib/crewai/tests/events/test_llm_usage_event.py +++ b/lib/crewai/tests/events/test_llm_usage_event.py @@ -4,7 +4,6 @@ from unittest.mock import patch import pytest from pydantic import BaseModel -from crewai.events.event_bus import CrewAIEventsBus from crewai.events.types.llm_events import LLMCallCompletedEvent, LLMCallType from crewai.llm import LLM from crewai.llms.base_llm import BaseLLM @@ -203,7 +202,9 @@ class _StubLLM(BaseLLM): class TestEmitCallCompletedEventPassesUsage: @pytest.fixture def mock_emit(self): - with patch.object(CrewAIEventsBus, "emit") as mock: + from crewai.events.event_bus import crewai_event_bus + + with patch.object(crewai_event_bus, "emit") as mock: yield mock @pytest.fixture @@ -297,3 +298,128 @@ class TestUsageMetricsNewFields: dumped = metrics.model_dump() assert dumped["reasoning_tokens"] == 10 assert dumped["cache_creation_tokens"] == 5 + + +class TestFromProviderDictAnthropicCacheTokens: + def test_cache_read_tokens_included_in_prompt_and_total(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + { + "input_tokens": 3, + "output_tokens": 44, + "cache_read_input_tokens": 2061, + } + ) + + assert metrics is not None + assert metrics.prompt_tokens == 2064 + assert metrics.completion_tokens == 44 + assert metrics.total_tokens == 2108 + assert metrics.cached_prompt_tokens == 2061 + + def test_cache_creation_tokens_included_in_prompt_and_total(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 20, + } + ) + + assert metrics is not None + assert metrics.prompt_tokens == 120 + assert metrics.total_tokens == 170 + assert metrics.cache_creation_tokens == 20 + + def test_cache_read_and_creation_tokens_both_included(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + { + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 30, + "cache_creation_input_tokens": 20, + } + ) + + assert metrics is not None + assert metrics.prompt_tokens == 150 + assert metrics.total_tokens == 200 + assert metrics.cached_prompt_tokens == 30 + assert metrics.cache_creation_tokens == 20 + + def test_missing_cache_fields_preserve_non_cached_totals(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + {"input_tokens": 100, "output_tokens": 50} + ) + + assert metrics is not None + assert metrics.prompt_tokens == 100 + assert metrics.total_tokens == 150 + assert metrics.cached_prompt_tokens == 0 + assert metrics.cache_creation_tokens == 0 + + def test_reconciled_native_dict_is_not_double_counted(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + { + "input_tokens": 150, + "output_tokens": 50, + "cached_prompt_tokens": 30, + "cache_creation_tokens": 20, + } + ) + + assert metrics is not None + assert metrics.prompt_tokens == 150 + assert metrics.total_tokens == 200 + assert metrics.cache_creation_tokens == 20 + + def test_openai_cached_prompt_tokens_are_not_added_twice(self): + from crewai.types.usage_metrics import UsageMetrics + + metrics = UsageMetrics.from_provider_dict( + { + "prompt_tokens": 100, + "completion_tokens": 50, + "prompt_tokens_details": {"cached_tokens": 30}, + } + ) + + assert metrics is not None + assert metrics.prompt_tokens == 100 + assert metrics.total_tokens == 150 + assert metrics.cached_prompt_tokens == 30 + + def test_cumulative_usage_via_add_usage_metrics(self): + from crewai.types.usage_metrics import UsageMetrics + + first = UsageMetrics.from_provider_dict( + { + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 30, + } + ) + second = UsageMetrics.from_provider_dict( + { + "input_tokens": 40, + "output_tokens": 20, + } + ) + + assert first is not None and second is not None + first.add_usage_metrics(second) + + assert first.prompt_tokens == 170 + assert first.completion_tokens == 70 + assert first.total_tokens == 240 + assert first.cached_prompt_tokens == 30 + assert first.successful_requests == 2 diff --git a/lib/crewai/tests/llms/anthropic/test_anthropic.py b/lib/crewai/tests/llms/anthropic/test_anthropic.py index fd21d3b8a..375246bb4 100644 --- a/lib/crewai/tests/llms/anthropic/test_anthropic.py +++ b/lib/crewai/tests/llms/anthropic/test_anthropic.py @@ -549,7 +549,12 @@ def test_anthropic_token_usage_tracking(): with patch.object(llm._client.messages, 'create') as mock_create: mock_response = MagicMock() mock_response.content = [MagicMock(text="test response")] - mock_response.usage = MagicMock(input_tokens=50, output_tokens=25) + mock_response.usage = MagicMock( + input_tokens=50, + output_tokens=25, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + ) mock_create.return_value = mock_response result = llm.call("Hello") @@ -1653,9 +1658,9 @@ def test_anthropic_cache_creation_tokens_extraction(): mock_response.model = None usage = llm._extract_anthropic_token_usage(mock_response) - assert usage["input_tokens"] == 100 + assert usage["input_tokens"] == 150 assert usage["output_tokens"] == 50 - assert usage["total_tokens"] == 150 + assert usage["total_tokens"] == 200 assert usage["cached_prompt_tokens"] == 30 assert usage["cache_creation_tokens"] == 20 diff --git a/lib/crewai/tests/llms/google/test_google.py b/lib/crewai/tests/llms/google/test_google.py index 0213eb525..2c03dcd23 100644 --- a/lib/crewai/tests/llms/google/test_google.py +++ b/lib/crewai/tests/llms/google/test_google.py @@ -123,12 +123,12 @@ def test_gemini_completion_initialization_parameters(): def test_gemini_started_event_surfaces_max_output_tokens(): - from crewai.events.event_bus import CrewAIEventsBus + from crewai.events.event_bus import crewai_event_bus from crewai.events.types.llm_events import LLMCallStartedEvent llm = LLM(model="google/gemini-2.0-flash-001", max_output_tokens=2000, api_key="test-key") - with patch.object(CrewAIEventsBus, "emit") as mock_emit: + with patch.object(crewai_event_bus, "emit") as mock_emit: llm._emit_call_started_event(messages="hi") event = mock_emit.call_args[1]["event"] diff --git a/lib/crewai/tests/llms/test_multimodal_integration.py b/lib/crewai/tests/llms/test_multimodal_integration.py index 9957be8c1..eb9ca7914 100644 --- a/lib/crewai/tests/llms/test_multimodal_integration.py +++ b/lib/crewai/tests/llms/test_multimodal_integration.py @@ -77,8 +77,8 @@ startxref def _build_multimodal_message(llm: LLM, prompt: str, files: dict) -> list[dict]: """Build a multimodal message with text and file content.""" - provider = getattr(llm, "provider", None) or llm.model - content_blocks = format_multimodal_content(files, provider) + formatter = llm._multimodal_formatter_name() + content_blocks = format_multimodal_content(files, formatter) return [ { "role": "user", diff --git a/lib/crewai/tests/llms/test_tool_call_streaming.py b/lib/crewai/tests/llms/test_tool_call_streaming.py index 03bc0fd9b..54cc2fd1d 100644 --- a/lib/crewai/tests/llms/test_tool_call_streaming.py +++ b/lib/crewai/tests/llms/test_tool_call_streaming.py @@ -38,18 +38,10 @@ def get_temperature_tool_schema() -> dict[str, Any]: @pytest.fixture def mock_emit() -> MagicMock: - """Mock the singleton event bus emit used by LLM providers. + from crewai.events.event_bus import crewai_event_bus - Patch the singleton instance (not only the class) so a leftover - instance-level ``emit`` from other tests cannot shadow the mock. - """ - from crewai.events.event_bus import CrewAIEventsBus, crewai_event_bus - - with ( - patch.object(CrewAIEventsBus, "emit") as class_mock, - patch.object(crewai_event_bus, "emit", new=class_mock), - ): - yield class_mock + with patch.object(crewai_event_bus, "emit") as mock: + yield mock def _event_from_emit_call(call: Any) -> Any: diff --git a/lib/crewai/tests/test_llm.py b/lib/crewai/tests/test_llm.py index 1c98d751e..08d80bbae 100644 --- a/lib/crewai/tests/test_llm.py +++ b/lib/crewai/tests/test_llm.py @@ -534,9 +534,9 @@ def assert_event_count( @pytest.fixture def mock_emit() -> MagicMock: - from crewai.events.event_bus import CrewAIEventsBus + from crewai.events.event_bus import crewai_event_bus - with patch.object(CrewAIEventsBus, "emit") as mock_emit: + with patch.object(crewai_event_bus, "emit") as mock_emit: yield mock_emit @@ -860,16 +860,19 @@ def test_prefixed_models_with_invalid_constants_use_litellm(): llm = LLM(model="openai/gemini-2.5-flash", is_litellm=False) assert llm.is_litellm is True assert llm.model == "openai/gemini-2.5-flash" + assert llm.provider == "openai" # Test openai/ prefix with model that doesn't match patterns (e.g. no gpt- prefix) → LiteLLM llm2 = LLM(model="openai/custom-finetune-model", is_litellm=False) assert llm2.is_litellm is True assert llm2.model == "openai/custom-finetune-model" + assert llm2.provider == "openai" # Test anthropic/ prefix with non-Anthropic model → LiteLLM llm3 = LLM(model="anthropic/gpt-4o", is_litellm=False) assert llm3.is_litellm is True assert llm3.model == "anthropic/gpt-4o" + assert llm3.provider == "anthropic" def test_prefixed_models_with_valid_patterns_use_native_sdk(): @@ -893,11 +896,38 @@ def test_prefixed_models_with_non_native_providers_use_litellm(): llm = LLM(model="groq/llama-3.3-70b", is_litellm=False) assert llm.is_litellm is True assert llm.model == "groq/llama-3.3-70b" + assert llm.provider == "groq" # Test together/ prefix (not a native provider) → LiteLLM llm2 = LLM(model="together/qwen-2.5-72b", is_litellm=False) assert llm2.is_litellm is True assert llm2.model == "together/qwen-2.5-72b" + assert llm2.provider == "together" + + +@pytest.mark.parametrize( + ("model", "expected_provider"), + [ + ("groq/llama-3.3-70b", "groq"), + ("cohere/command-r", "cohere"), + ("sambanova/Meta-Llama-3.1-70B-Instruct", "sambanova"), + ("mistral/mistral-large", "mistral"), + ("vertex_ai/gemini-1.5-pro", "vertex_ai"), + ("openai/custom-finetune-model", "openai"), + ("anthropic/gpt-4o", "anthropic"), + ], +) +def test_litellm_path_preserves_provider_from_model_prefix(model, expected_provider): + llm = LLM(model=model, is_litellm=False) + assert llm.is_litellm is True + assert llm.provider == expected_provider + assert llm.model == model + + +def test_litellm_keeps_provider_but_formats_multimodal_as_openai_schema(): + llm = LLM(model="anthropic/claude-3-5-haiku-20241022", is_litellm=True) + assert llm.provider == "anthropic" + assert llm._multimodal_formatter_name() == "openai" def test_unprefixed_models_use_native_sdk(): diff --git a/lib/crewai/tests/test_llm_streaming_finish_reason.py b/lib/crewai/tests/test_llm_streaming_finish_reason.py index ff8a94d4e..c90fe77a4 100644 --- a/lib/crewai/tests/test_llm_streaming_finish_reason.py +++ b/lib/crewai/tests/test_llm_streaming_finish_reason.py @@ -11,14 +11,14 @@ from unittest.mock import patch import pytest -from crewai.events.event_bus import CrewAIEventsBus +from crewai.events.event_bus import crewai_event_bus from crewai.events.types.llm_events import LLMCallCompletedEvent from crewai.llm import LLM @pytest.fixture def mock_emit(): - with patch.object(CrewAIEventsBus, "emit") as mock: + with patch.object(crewai_event_bus, "emit") as mock: yield mock diff --git a/lib/crewai/tests/utilities/test_converter.py b/lib/crewai/tests/utilities/test_converter.py index ed6429dac..f46cf8008 100644 --- a/lib/crewai/tests/utilities/test_converter.py +++ b/lib/crewai/tests/utilities/test_converter.py @@ -1004,3 +1004,21 @@ def test_internal_instructor_omits_unset_base_url_and_api_key() -> None: InternalInstructor(content="x", model=SimpleModel, llm=mock_llm) mock_from_provider.assert_called_once_with("openai/gpt-4o") + + +def test_internal_instructor_does_not_double_prefix_qualified_models() -> None: + from crewai.utilities.internal_instructor import InternalInstructor + + mock_llm = Mock() + mock_llm.is_litellm = False + mock_llm.model = "groq/llama-3.3-70b" + mock_llm.provider = "groq" + mock_llm.base_url = None + mock_llm.api_key = None + + with patch("instructor.from_provider") as mock_from_provider: + mock_from_provider.return_value = Mock() + + InternalInstructor(content="x", model=SimpleModel, llm=mock_llm) + + mock_from_provider.assert_called_once_with("groq/llama-3.3-70b") diff --git a/lib/crewai/tests/utilities/test_events.py b/lib/crewai/tests/utilities/test_events.py index d9c9d5529..a5e22fe81 100644 --- a/lib/crewai/tests/utilities/test_events.py +++ b/lib/crewai/tests/utilities/test_events.py @@ -1244,7 +1244,6 @@ def test_llm_completed_event_includes_usage(): assert event.usage.get("total_tokens", 0) > 0 -@pytest.mark.vcr() def test_llm_emits_call_failed_event(): received_events = [] event_received = threading.Event() @@ -1256,12 +1255,10 @@ def test_llm_emits_call_failed_event(): error_message = "OpenAI API call failed: Simulated API failure" - with patch( - "crewai.llms.providers.openai.completion.OpenAICompletion._handle_completion" - ) as mock_handle_completion: - mock_handle_completion.side_effect = Exception("Simulated API failure") - - llm = LLM(model="gpt-4o-mini") + llm = LLM(model="gpt-4o-mini") + with patch.object( + llm, "_handle_completion", side_effect=Exception("Simulated API failure") + ): with pytest.raises(Exception) as exc_info: llm.call("Hello, how are you?") diff --git a/pyproject.toml b/pyproject.toml index 1b991a022..4b64275e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -216,6 +216,9 @@ exclude-newer-package = { pypdf = "2026-06-24T00:00:00Z", msgpack = "2026-06-20T # 3.9.4, so that ignore is no longer needed. Transitive via # crewai-tools[xml] -> unstructured. # pydantic-settings <2.14.2 has GHSA-4xgf-cpjx-pc3j. +# h2 <=4.4.0 has GHSA-6hr6-w5qg-qmwg (CVE-2026-71554): duplicate Host headers +# can facilitate request smuggling; fixed in 4.4.1. Transitive via +# qdrant-client -> httpx[http2]. # Keep OpenAI on the SDK range required by CrewAI when transitive dependencies # loosen or pin their own lower versions. override-dependencies = [ @@ -246,6 +249,7 @@ override-dependencies = [ "pydantic-settings>=2.14.2", "setuptools>=83.0.0", # PYSEC-2026-3447 "nltk>=3.10.0", + "h2>=4.4.1", ] [tool.uv.workspace] diff --git a/uv.lock b/uv.lock index 7142151fe..92440253e 100644 --- a/uv.lock +++ b/uv.lock @@ -38,6 +38,7 @@ overrides = [ { name = "cryptography", specifier = ">=50.0.0" }, { name = "docling-core", extras = ["chunking"], specifier = ">=2.74.1" }, { name = "gitpython", specifier = ">=3.1.57,<4" }, + { name = "h2", specifier = ">=4.4.1" }, { name = "langchain-core", specifier = ">=1.3.3,<2" }, { name = "langchain-text-splitters", specifier = ">=1.1.2,<2" }, { name = "langsmith", specifier = ">=0.8.18,<1" }, @@ -3057,15 +3058,15 @@ wheels = [ [[package]] name = "h2" -version = "4.3.0" +version = "4.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "hpack" }, { name = "hyperframe" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, ] [[package]] @@ -3094,11 +3095,11 @@ wheels = [ [[package]] name = "hpack" -version = "4.1.0" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, ] [[package]]