From 0bdc5a093eaa42d2a0bfdb5b97d6d7d2d017d199 Mon Sep 17 00:00:00 2001 From: Matt Aitchison Date: Wed, 25 Feb 2026 16:14:38 -0600 Subject: [PATCH] =?UTF-8?q?ci:=20optimize=20test=20workflows=20=E2=80=94?= =?UTF-8?q?=20reduce=20jobs,=20share=20venv=20via=20artifact?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restructure tests.yml: install once per Python version, share .venv via artifact instead of 32 independent installs - Reduce test groups from 8 to 4 (tests only take ~60s per group) - Only test Python 3.12+3.13 on PRs; full matrix on push to main - Switch all workflows from manual actions/cache to setup-uv built-in caching, eliminating cache race conditions - Add --frozen flag to uv sync for deterministic CI installs - Re-enable duration-based test splitting with least_duration algorithm (was disabled due to a bug in the path filter) - Fix update-test-durations path filter (tests/**/*.py never matched actual test dirs under lib/) - Add concurrency group with cancel-in-progress for PR runs - Add gate jobs to satisfy existing branch protection required checks --- .github/workflows/build-uv-cache.yml | 20 +-- .github/workflows/linter.yml | 26 +--- .github/workflows/tests.yml | 156 +++++++++++++------- .github/workflows/type-checker.yml | 30 +--- .github/workflows/update-test-durations.yml | 34 +---- 5 files changed, 121 insertions(+), 145 deletions(-) diff --git a/.github/workflows/build-uv-cache.yml b/.github/workflows/build-uv-cache.yml index 3e5028eb7..564a77b0e 100644 --- a/.github/workflows/build-uv-cache.yml +++ b/.github/workflows/build-uv-cache.yml @@ -25,24 +25,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install uv + - name: Install uv and populate cache uses: astral-sh/setup-uv@v6 with: version: "0.8.4" python-version: ${{ matrix.python-version }} - enable-cache: false + enable-cache: true - - name: Install dependencies and populate cache - run: | - echo "Building global UV cache for Python ${{ matrix.python-version }}..." - uv sync --all-groups --all-extras --no-install-project - echo "Cache populated successfully" - - - name: Save uv caches - uses: actions/cache/save@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + - name: Install dependencies + run: uv sync --all-groups --all-extras --frozen --no-install-project diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index ae26c4209..6e7e7ed85 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -18,27 +18,15 @@ jobs: - name: Fetch Target Branch run: git fetch origin $TARGET_BRANCH --depth=1 - - name: Restore global uv cache - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py3.11-${{ hashFiles('uv.lock') }} - restore-keys: | - uv-main-py3.11- - - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "0.8.4" python-version: "3.11" - enable-cache: false + enable-cache: true - name: Install dependencies - run: uv sync --all-groups --all-extras --no-install-project + run: uv sync --all-groups --all-extras --frozen --no-install-project - name: Get Changed Python Files id: changed-files @@ -57,13 +45,3 @@ jobs: | grep -v 'src/crewai/cli/templates/' \ | grep -v '/tests/' \ | xargs -I{} uv run ruff check "{}" - - - name: Save uv caches - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py3.11-${{ hashFiles('uv.lock') }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6d8054ff4..8f858b995 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,37 +1,79 @@ name: Run Tests -on: [pull_request] +on: + pull_request: + push: + branches: [main] permissions: contents: read +concurrency: + group: tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: - tests: - name: tests (${{ matrix.python-version }}) + configure: + name: Configure matrix runs-on: ubuntu-latest - timeout-minutes: 15 + outputs: + python-versions: ${{ steps.matrix.outputs.python-versions }} + steps: + - id: matrix + run: | + if [ "${{ github.event_name }}" = "push" ]; then + echo 'python-versions=["3.10","3.11","3.12","3.13"]' >> "$GITHUB_OUTPUT" + else + echo 'python-versions=["3.12","3.13"]' >> "$GITHUB_OUTPUT" + fi + + install: + name: install (py${{ matrix.python-version }}) + needs: configure + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ${{ fromJSON(needs.configure.outputs.python-versions) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.4" + python-version: ${{ matrix.python-version }} + enable-cache: true + + - name: Install the project + run: uv sync --all-groups --all-extras --frozen + + - name: Package virtualenv + run: tar czf /tmp/venv.tar.gz .venv + + - name: Upload virtualenv + uses: actions/upload-artifact@v4 + with: + name: venv-py${{ matrix.python-version }} + path: /tmp/venv.tar.gz + retention-days: 1 + compression-level: 0 + + tests: + name: tests (py${{ matrix.python-version }}, ${{ matrix.group }}/4) + needs: [configure, install] + runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: true matrix: - python-version: ['3.10', '3.11', '3.12', '3.13'] - group: [1, 2, 3, 4, 5, 6, 7, 8] + python-version: ${{ fromJSON(needs.configure.outputs.python-versions) }} + group: [1, 2, 3, 4] steps: - name: Checkout code uses: actions/checkout@v4 with: - fetch-depth: 0 # Fetch all history for proper diff - - - name: Restore global uv cache - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} - restore-keys: | - uv-main-py${{ matrix.python-version }}- + fetch-depth: 0 - name: Install uv uses: astral-sh/setup-uv@v6 @@ -40,8 +82,14 @@ jobs: python-version: ${{ matrix.python-version }} enable-cache: false - - name: Install the project - run: uv sync --all-groups --all-extras + - name: Download virtualenv + uses: actions/download-artifact@v4 + with: + name: venv-py${{ matrix.python-version }} + path: /tmp + + - name: Restore virtualenv + run: tar xzf /tmp/venv.tar.gz - name: Restore test durations uses: actions/cache/restore@v4 @@ -49,52 +97,56 @@ jobs: path: .test_durations_py* key: test-durations-py${{ matrix.python-version }} - - name: Run tests (group ${{ matrix.group }} of 8) + - name: Run tests (group ${{ matrix.group }} of 4) run: | PYTHON_VERSION_SAFE=$(echo "${{ matrix.python-version }}" | tr '.' '_') DURATION_FILE="../../.test_durations_py${PYTHON_VERSION_SAFE}" - # Temporarily always skip cached durations to fix test splitting - # When durations don't match, pytest-split runs duplicate tests instead of splitting - echo "Using even test splitting (duration cache disabled until fix merged)" DURATIONS_ARG="" + if [ -f "$DURATION_FILE" ]; then + if git diff origin/${{ github.base_ref }}...HEAD --name-only 2>/dev/null | grep -q "^lib/.*/tests/.*\.py$"; then + echo "::notice::Test files changed — using even splitting" + else + echo "::notice::Using cached test durations for optimal splitting" + DURATIONS_ARG="--durations-path=${DURATION_FILE}" + fi + else + echo "::notice::No cached durations — using even splitting" + fi - # Original logic (disabled temporarily): - # if [ ! -f "$DURATION_FILE" ]; then - # echo "No cached durations found, tests will be split evenly" - # DURATIONS_ARG="" - # elif git diff origin/${{ github.base_ref }}...HEAD --name-only 2>/dev/null | grep -q "^tests/.*\.py$"; then - # echo "Test files have changed, skipping cached durations to avoid mismatches" - # DURATIONS_ARG="" - # else - # echo "No test changes detected, using cached test durations for optimal splitting" - # DURATIONS_ARG="--durations-path=${DURATION_FILE}" - # fi - - cd lib/crewai && uv run pytest \ + cd lib/crewai && uv run --frozen pytest \ -vv \ - --splits 8 \ + --splits 4 \ --group ${{ matrix.group }} \ $DURATIONS_ARG \ + --splitting-algorithm least_duration \ --durations=10 \ --maxfail=3 - - name: Run tool tests (group ${{ matrix.group }} of 8) + - name: Run tool tests (group ${{ matrix.group }} of 4) run: | - cd lib/crewai-tools && uv run pytest \ + cd lib/crewai-tools && uv run --frozen pytest \ -vv \ - --splits 8 \ + --splits 4 \ --group ${{ matrix.group }} \ --durations=10 \ --maxfail=3 - - - name: Save uv caches - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + # Gate jobs matching required status checks in branch protection + tests-gate: + name: tests (${{ matrix.python-version }}) + needs: [tests] + if: always() + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13'] + steps: + - name: Check test results + run: | + if [ "${{ needs.tests.result }}" = "success" ]; then + echo "All tests passed" + else + echo "Tests failed: ${{ needs.tests.result }}" + exit 1 + fi diff --git a/.github/workflows/type-checker.yml b/.github/workflows/type-checker.yml index 03a5841a0..6a7be4751 100644 --- a/.github/workflows/type-checker.yml +++ b/.github/workflows/type-checker.yml @@ -20,27 +20,15 @@ jobs: with: fetch-depth: 0 # Fetch all history for proper diff - - name: Restore global uv cache - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} - restore-keys: | - uv-main-py${{ matrix.python-version }}- - - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "0.8.4" python-version: ${{ matrix.python-version }} - enable-cache: false + enable-cache: true - name: Install dependencies - run: uv sync --all-groups --all-extras + run: uv sync --all-groups --all-extras --frozen - name: Get changed Python files id: changed-files @@ -74,16 +62,6 @@ jobs: if: steps.changed-files.outputs.has_changes == 'false' run: echo "No Python files in src/ were modified - skipping type checks" - - name: Save uv caches - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} - # Summary job to provide single status for branch protection type-checker: name: type-checker @@ -94,8 +72,8 @@ jobs: - name: Check matrix results run: | if [ "${{ needs.type-checker-matrix.result }}" == "success" ] || [ "${{ needs.type-checker-matrix.result }}" == "skipped" ]; then - echo "✅ All type checks passed" + echo "All type checks passed" else - echo "❌ Type checks failed" + echo "Type checks failed" exit 1 fi diff --git a/.github/workflows/update-test-durations.yml b/.github/workflows/update-test-durations.yml index 13f1ecd69..3f25659c8 100644 --- a/.github/workflows/update-test-durations.yml +++ b/.github/workflows/update-test-durations.yml @@ -5,7 +5,9 @@ on: branches: - main paths: - - 'tests/**/*.py' + - 'lib/crewai/tests/**/*.py' + - 'lib/crewai-tools/tests/**/*.py' + - 'lib/crewai-files/tests/**/*.py' workflow_dispatch: permissions: @@ -20,37 +22,25 @@ jobs: env: OPENAI_API_KEY: fake-api-key PYTHONUNBUFFERED: 1 - + steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Restore global uv cache - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} - restore-keys: | - uv-main-py${{ matrix.python-version }}- - - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "0.8.4" python-version: ${{ matrix.python-version }} - enable-cache: false + enable-cache: true - name: Install the project - run: uv sync --all-groups --all-extras + run: uv sync --all-groups --all-extras --frozen - name: Run all tests and store durations run: | PYTHON_VERSION_SAFE=$(echo "${{ matrix.python-version }}" | tr '.' '_') - uv run pytest --store-durations --durations-path=.test_durations_py${PYTHON_VERSION_SAFE} -n auto + uv run --frozen pytest --store-durations --durations-path=.test_durations_py${PYTHON_VERSION_SAFE} -n auto continue-on-error: true - name: Save durations to cache @@ -59,13 +49,3 @@ jobs: with: path: .test_durations_py* key: test-durations-py${{ matrix.python-version }} - - - name: Save uv caches - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - ~/.cache/uv - ~/.local/share/uv - .venv - key: uv-main-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} \ No newline at end of file