mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-07-03 22:19:27 +00:00
0cc43b2720
New vulnerability scan process: 1. Run pip-audit without ignores on every PR 2. Classify vulns as direct or transitive (checks against all monorepo pyproject.toml files) 3. Direct vulns: auto-fix with pip-audit --fix and commit the bump to the PR branch 4. Transitive vulns: add to ignore list and create a GitHub issue for tracking 5. Re-run pip-audit with transitive ignores — PR passes only if direct vulns are resolved 6. Scheduled runs also validate that previously ignored vulns are still unfixable Removes continue-on-error: true so the action actually blocks.
271 lines
9.8 KiB
YAML
271 lines
9.8 KiB
YAML
name: Vulnerability Scan
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches: [main]
|
|
schedule:
|
|
# Run weekly on Monday at 9:00 UTC
|
|
- cron: '0 9 * * 1'
|
|
|
|
permissions:
|
|
contents: write
|
|
pull-requests: write
|
|
issues: write
|
|
|
|
jobs:
|
|
pip-audit:
|
|
name: pip-audit
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
|
with:
|
|
persist-credentials: ${{ github.event_name == 'pull_request' }}
|
|
|
|
- name: Restore global uv cache
|
|
id: cache-restore
|
|
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: |
|
|
~/.cache/uv
|
|
~/.local/share/uv
|
|
.venv
|
|
key: uv-main-py3.11-${{ hashFiles('uv.lock') }}
|
|
restore-keys: |
|
|
uv-main-py3.11-
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
|
with:
|
|
version: "0.11.3"
|
|
python-version: "3.11"
|
|
enable-cache: false
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --all-groups --all-extras --no-install-project
|
|
|
|
- name: Install pip-audit
|
|
run: uv pip install pip-audit
|
|
|
|
- name: Run pip-audit
|
|
id: audit
|
|
run: |
|
|
uv run pip-audit --desc --aliases --skip-editable --format json --output pip-audit-report.json || true
|
|
# Intentionally ignore exit code — we parse the JSON ourselves below.
|
|
|
|
- name: Classify vulnerabilities
|
|
id: classify
|
|
run: |
|
|
set -euo pipefail
|
|
python3 << 'PYEOF'
|
|
import json, sys, glob, re
|
|
from pathlib import Path
|
|
|
|
# Collect direct deps from all pyproject.toml files in the monorepo
|
|
try:
|
|
import tomllib
|
|
except ImportError:
|
|
import tomli as tomllib
|
|
|
|
direct_deps = set()
|
|
for toml_path in glob.glob("**/pyproject.toml", recursive=True):
|
|
if "templates/" in toml_path or "node_modules/" in toml_path:
|
|
continue
|
|
try:
|
|
with open(toml_path, "rb") as f:
|
|
data = tomllib.load(f)
|
|
except Exception:
|
|
continue
|
|
project = data.get("project", {})
|
|
for dep_str in project.get("dependencies", []):
|
|
name = re.split(r"[><=!~\[]", dep_str)[0].strip().lower()
|
|
direct_deps.add(name)
|
|
for group_deps in project.get("optional-dependencies", {}).values():
|
|
for dep_str in group_deps:
|
|
name = re.split(r"[><=!~\[]", dep_str)[0].strip().lower()
|
|
direct_deps.add(name)
|
|
for group_deps in data.get("dependency-groups", {}).values():
|
|
if isinstance(group_deps, list):
|
|
for dep_str in group_deps:
|
|
if isinstance(dep_str, str):
|
|
name = re.split(r"[><=!~\[]", dep_str)[0].strip().lower()
|
|
direct_deps.add(name)
|
|
|
|
# Load pip-audit report
|
|
try:
|
|
with open("pip-audit-report.json") as f:
|
|
report = json.load(f)
|
|
except FileNotFoundError:
|
|
print("::error::pip-audit report not found")
|
|
sys.exit(1)
|
|
|
|
deps = report.get("dependencies", [])
|
|
vulns = [d for d in deps if d.get("vulns")]
|
|
|
|
if not vulns:
|
|
print("No vulnerabilities found")
|
|
Path("direct_vulns.txt").write_text("")
|
|
Path("transitive_vulns.txt").write_text("")
|
|
Path("transitive_ids.txt").write_text("")
|
|
sys.exit(0)
|
|
|
|
direct_vulns = []
|
|
transitive_vulns = []
|
|
transitive_ids = []
|
|
|
|
for dep in vulns:
|
|
name = dep["name"]
|
|
version = dep["version"]
|
|
is_direct = name.lower() in direct_deps
|
|
for v in dep["vulns"]:
|
|
entry = f"{name}=={version} ({v['id']})"
|
|
if is_direct:
|
|
direct_vulns.append(entry)
|
|
else:
|
|
transitive_vulns.append(entry)
|
|
transitive_ids.append(v['id'])
|
|
|
|
Path("direct_vulns.txt").write_text("\n".join(direct_vulns) if direct_vulns else "")
|
|
Path("transitive_vulns.txt").write_text("\n".join(transitive_vulns) if transitive_vulns else "")
|
|
Path("transitive_ids.txt").write_text("\n".join(transitive_ids) if transitive_ids else "")
|
|
|
|
print(f"Direct: {len(direct_vulns)}, Transitive: {len(transitive_vulns)}")
|
|
for v in direct_vulns:
|
|
print(f" DIRECT: {v}")
|
|
for v in transitive_vulns:
|
|
print(f" TRANSITIVE: {v}")
|
|
PYEOF
|
|
|
|
# Set outputs
|
|
if [ -s direct_vulns.txt ]; then
|
|
echo "has_direct=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_direct=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
if [ -s transitive_vulns.txt ]; then
|
|
echo "has_transitive=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_transitive=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Attempt fix for direct vulnerabilities
|
|
if: github.event_name == 'pull_request' && steps.classify.outputs.has_direct == 'true'
|
|
id: fix
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
echo "Attempting to fix direct vulnerabilities..."
|
|
cat direct_vulns.txt
|
|
|
|
# Try pip-audit --fix to bump direct deps
|
|
uv run pip-audit --fix --skip-editable 2>&1 || true
|
|
|
|
# Check if uv.lock changed
|
|
if git diff --quiet uv.lock; then
|
|
echo "fixed=false" >> "$GITHUB_OUTPUT"
|
|
echo "::warning::Could not auto-fix direct vulnerabilities. Manual intervention required."
|
|
else
|
|
echo "fixed=true" >> "$GITHUB_OUTPUT"
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
git add uv.lock
|
|
git commit -m "fix: bump dependencies to resolve security vulnerabilities
|
|
|
|
Auto-fixed by vulnerability-scan workflow.
|
|
Resolved: $(cat direct_vulns.txt | tr '\n' ', ')"
|
|
git push
|
|
fi
|
|
|
|
- name: Add transitive vulns to ignore list and create issues
|
|
if: steps.classify.outputs.has_transitive == 'true'
|
|
id: ignore
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# Build --ignore-vuln flags from transitive vuln IDs
|
|
IGNORE_FLAGS=""
|
|
while IFS= read -r vuln_id; do
|
|
if [ -n "$vuln_id" ]; then
|
|
IGNORE_FLAGS="$IGNORE_FLAGS --ignore-vuln $vuln_id"
|
|
fi
|
|
done < transitive_ids.txt
|
|
echo "ignore_flags=$IGNORE_FLAGS" >> "$GITHUB_OUTPUT"
|
|
|
|
# Create GitHub issues for transitive vulns
|
|
while IFS= read -r line; do
|
|
if [ -z "$line" ]; then continue; fi
|
|
VULN_ID=$(echo "$line" | grep -oE '[A-Z]+-[0-9]+-[0-9]+|GHSA-[a-z0-9-]+' || true)
|
|
PKG=$(echo "$line" | cut -d'=' -f1)
|
|
|
|
# Check if issue already exists
|
|
EXISTING=$(gh issue list --label "security,transitive-vuln" --state open --json title \
|
|
--jq ".[] | select(.title | contains(\"$VULN_ID\"))" || true)
|
|
|
|
if [ -z "$EXISTING" ]; then
|
|
gh issue create \
|
|
--title "🔒 Transitive vulnerability: $VULN_ID in $PKG" \
|
|
--label "security,transitive-vuln" \
|
|
--body "## Transitive Dependency Vulnerability
|
|
|
|
**Package:** \`$line\`
|
|
**Vulnerability:** $VULN_ID
|
|
**Status:** No fix available upstream
|
|
|
|
This vulnerability is in a transitive dependency and cannot be fixed directly. It has been added to the pip-audit ignore list until an upstream fix is available.
|
|
|
|
### Action Required
|
|
- [ ] Monitor upstream for a fix
|
|
- [ ] Remove from ignore list once fixed
|
|
- [ ] Close this issue when resolved
|
|
|
|
_Auto-created by vulnerability-scan workflow._"
|
|
fi
|
|
done < <(cat transitive_vulns.txt)
|
|
|
|
- name: Re-run pip-audit with transitive ignores
|
|
if: steps.classify.outputs.has_transitive == 'true'
|
|
id: audit-final
|
|
run: |
|
|
IGNORE_FLAGS="${{ steps.ignore.outputs.ignore_flags }}"
|
|
eval uv run pip-audit --desc --aliases --skip-editable --format json \
|
|
--output pip-audit-report.json \
|
|
$IGNORE_FLAGS
|
|
|
|
- name: Fail if direct vulnerabilities remain unfixed
|
|
if: steps.classify.outputs.has_direct == 'true' && steps.fix.outputs.fixed != 'true'
|
|
run: |
|
|
echo "::error::Direct vulnerabilities found that could not be auto-fixed:"
|
|
cat direct_vulns.txt
|
|
echo ""
|
|
echo "Fix these manually or run: pip-audit --fix"
|
|
exit 1
|
|
|
|
- name: Display results
|
|
if: always()
|
|
run: |
|
|
if [ -f pip-audit-report.json ]; then
|
|
echo "## pip-audit Results" >> $GITHUB_STEP_SUMMARY
|
|
echo '```json' >> $GITHUB_STEP_SUMMARY
|
|
cat pip-audit-report.json | python3 -m json.tool >> $GITHUB_STEP_SUMMARY
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "::error::pip-audit failed to produce a report."
|
|
fi
|
|
|
|
- name: Upload pip-audit report
|
|
if: always()
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: pip-audit-report
|
|
path: pip-audit-report.json
|
|
|
|
- name: Save uv caches
|
|
if: steps.cache-restore.outputs.cache-hit != 'true'
|
|
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: |
|
|
~/.cache/uv
|
|
~/.local/share/uv
|
|
.venv
|
|
key: uv-main-py3.11-${{ hashFiles('uv.lock') }}
|