Successfully created the essential workflows , CI, security, and codequality, etc #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🚀 CI/CD Pipeline | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main, develop ] | |
| workflow_dispatch: | |
| env: | |
| POETRY_VERSION: "2.2.1" | |
| NODE_VERSION: "22.x" | |
| PYTHON_VERSION: "3.13" | |
| jobs: | |
| # ============================================================================ | |
| # SETUP & VALIDATION | |
| # ============================================================================ | |
| setup: | |
| name: 🔧 Setup & Validate Environment | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend-changed: ${{ steps.changes.outputs.backend }} | |
| frontend-changed: ${{ steps.changes.outputs.frontend }} | |
| docker-changed: ${{ steps.changes.outputs.docker }} | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: 🔍 Detect Changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| filters: | | |
| backend: | |
| - 'backend/**' | |
| - 'pyproject.toml' | |
| - 'poetry.lock' | |
| - 'tests/**' | |
| frontend: | |
| - 'frontend/**' | |
| - 'landing/**' | |
| - 'package*.json' | |
| docker: | |
| - 'backend/docker-compose.yml' | |
| - '**/Dockerfile*' | |
| - name: 📊 Repository Stats | |
| run: | | |
| echo "### 📊 Repository Overview" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Python files**: $(find . -name '*.py' | wc -l)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **TypeScript/JavaScript files**: $(find . -name '*.ts' -o -name '*.tsx' -o -name '*.js' -o -name '*.jsx' | wc -l)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Test files**: $(find . -name 'test_*.py' -o -name '*.test.ts' -o -name '*.test.js' | wc -l)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Changed paths**: Backend: ${{ steps.changes.outputs.backend }}, Frontend: ${{ steps.changes.outputs.frontend }}" >> $GITHUB_STEP_SUMMARY | |
| # ============================================================================ | |
| # BACKEND TESTING & QUALITY | |
| # ============================================================================ | |
| backend-test: | |
| name: 🐍 Backend Tests | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| if: needs.setup.outputs.backend-changed == 'true' || github.event_name == 'workflow_dispatch' | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| os: [ubuntu-latest, macos-latest] | |
| fail-fast: false | |
| services: | |
| weaviate: | |
| image: cr.weaviate.io/semitechnologies/weaviate:1.31.0 | |
| ports: | |
| - 8080:8080 | |
| env: | |
| AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' | |
| PERSISTENCE_DATA_PATH: '/var/lib/weaviate' | |
| options: >- | |
| --health-cmd "wget --no-verbose --tries=3 --spider http://localhost:8080/v1/meta || exit 1" | |
| --health-interval 30s | |
| --health-timeout 10s | |
| --health-retries 5 | |
| rabbitmq: | |
| image: rabbitmq:3-management | |
| ports: | |
| - 5672:5672 | |
| env: | |
| RABBITMQ_DEFAULT_USER: guest | |
| RABBITMQ_DEFAULT_PASS: guest | |
| options: >- | |
| --health-cmd "rabbitmq-diagnostics status" | |
| --health-interval 30s | |
| --health-timeout 10s | |
| --health-retries 5 | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: 📦 Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: 🔄 Load Cached Dependencies | |
| uses: actions/cache@v4 | |
| id: cached-poetry-dependencies | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: 📥 Install Dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: | | |
| cd backend | |
| poetry install --with dev | |
| - name: 🔧 Setup Test Environment | |
| run: | | |
| # Create test .env file | |
| cat > .env << EOF | |
| GEMINI_API_KEY=test_key_placeholder | |
| TAVILY_API_KEY=test_key_placeholder | |
| DISCORD_BOT_TOKEN=test_token_placeholder | |
| GITHUB_TOKEN=test_token_placeholder | |
| SUPABASE_URL=https://test.supabase.co | |
| SUPABASE_KEY=test_key_placeholder | |
| BACKEND_URL=http://localhost:8001 | |
| RABBITMQ_URL=amqp://localhost:5672/ | |
| LANGSMITH_TRACING=false | |
| EOF | |
| - name: 🧪 Run Backend Tests | |
| working-directory: backend | |
| run: | | |
| poetry run pytest tests/ \ | |
| --cov=app \ | |
| --cov-report=xml \ | |
| --cov-report=html \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=20 \ | |
| --junit-xml=test-results.xml \ | |
| -v | |
| - name: 📊 Upload Coverage to Codecov | |
| if: matrix.python-version == env.PYTHON_VERSION && matrix.os == 'ubuntu-latest' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: backend/coverage.xml | |
| flags: backend | |
| name: backend-coverage | |
| - name: 📋 Upload Test Results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: backend-test-results-${{ matrix.python-version }}-${{ matrix.os }} | |
| path: | | |
| backend/test-results.xml | |
| backend/htmlcov/ | |
| # ============================================================================ | |
| # FRONTEND TESTING & QUALITY | |
| # ============================================================================ | |
| frontend-test: | |
| name: ⚛️ Frontend Tests | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| if: needs.setup.outputs.frontend-changed == 'true' || github.event_name == 'workflow_dispatch' | |
| strategy: | |
| matrix: | |
| node-version: ["18.x", "20.x", "22.x"] | |
| fail-fast: false | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🟢 Setup Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| cache: 'npm' | |
| cache-dependency-path: 'frontend/package-lock.json' | |
| - name: 📦 Install Frontend Dependencies | |
| working-directory: frontend | |
| run: npm ci | |
| - name: 🔍 TypeScript Check | |
| working-directory: frontend | |
| run: npx tsc --noEmit | |
| - name: 🧪 Run Frontend Tests | |
| working-directory: frontend | |
| run: | | |
| npm run test -- --coverage --watchAll=false | |
| env: | |
| CI: true | |
| - name: 🏗️ Build Frontend | |
| working-directory: frontend | |
| run: npm run build | |
| - name: 📊 Upload Coverage | |
| if: matrix.node-version == env.NODE_VERSION | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: frontend/coverage/lcov.info | |
| flags: frontend | |
| name: frontend-coverage | |
| - name: 📦 Upload Build Artifacts | |
| if: matrix.node-version == env.NODE_VERSION | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: frontend-build | |
| path: frontend/dist/ | |
| # ============================================================================ | |
| # CODE QUALITY & LINTING | |
| # ============================================================================ | |
| code-quality: | |
| name: 🔍 Code Quality & Linting | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: 📦 Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| - name: 🟢 Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'npm' | |
| cache-dependency-path: 'frontend/package-lock.json' | |
| - name: 📥 Install Dependencies | |
| run: | | |
| cd backend && poetry install --with dev | |
| cd ../frontend && npm ci | |
| # Backend Quality Checks | |
| - name: 🐍 Python Code Formatting (Black) | |
| working-directory: backend | |
| run: poetry run black --check --diff . | |
| - name: 🔢 Import Sorting (isort) | |
| working-directory: backend | |
| run: poetry run isort --check-only --diff . | |
| - name: 🔍 Python Linting (flake8) | |
| working-directory: backend | |
| run: poetry run flake8 . | |
| - name: 🏷️ Type Checking (mypy) | |
| working-directory: backend | |
| run: poetry run mypy . || true # Allow to continue for now | |
| # Frontend Quality Checks | |
| - name: ⚛️ Frontend Linting (ESLint) | |
| working-directory: frontend | |
| run: npx eslint . --ext .ts,.tsx,.js,.jsx --max-warnings 0 | |
| - name: 💅 Frontend Formatting (Prettier) | |
| working-directory: frontend | |
| run: npx prettier --check . | |
| - name: 🔍 TypeScript Strict Check | |
| working-directory: frontend | |
| run: npx tsc --noEmit --strict | |
| # ============================================================================ | |
| # SECURITY SCANNING | |
| # ============================================================================ | |
| security-scan: | |
| name: 🔒 Security Scanning | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: 📦 Install Poetry | |
| uses: snok/install-poetry@v1 | |
| - name: 🔍 Python Security Scan (Safety) | |
| working-directory: backend | |
| run: | | |
| poetry run pip install safety | |
| poetry export -f requirements.txt --output requirements.txt | |
| poetry run safety check -r requirements.txt || true | |
| - name: 🔒 Code Security Analysis (Bandit) | |
| working-directory: backend | |
| run: | | |
| poetry run pip install bandit[toml] | |
| poetry run bandit -r . -f json -o bandit-report.json || true | |
| - name: 🔍 Frontend Security Audit | |
| working-directory: frontend | |
| run: npm audit --audit-level high || true | |
| - name: 📋 Upload Security Reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: security-reports | |
| path: | | |
| backend/bandit-report.json | |
| backend/requirements.txt | |
| # ============================================================================ | |
| # DOCKER & INTEGRATION TESTS | |
| # ============================================================================ | |
| docker-test: | |
| name: 🐳 Docker & Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: [setup, backend-test] | |
| if: needs.setup.outputs.docker-changed == 'true' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🐳 Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: 🏗️ Test Docker Compose Build | |
| working-directory: backend | |
| run: | | |
| docker-compose -f docker-compose.yml build --no-cache | |
| docker-compose -f docker-compose.yml up -d | |
| sleep 30 # Wait for services to be ready | |
| - name: 🧪 Integration Health Checks | |
| run: | | |
| echo "Testing service endpoints..." | |
| # Test Weaviate | |
| curl -f http://localhost:8080/v1/meta || echo "Weaviate check failed" | |
| # Test RabbitMQ | |
| curl -f http://localhost:15672 || echo "RabbitMQ check failed" | |
| # Test FalkorDB | |
| curl -f http://localhost:3000 || echo "FalkorDB check failed" | |
| - name: 🧹 Cleanup Docker | |
| if: always() | |
| working-directory: backend | |
| run: | | |
| docker-compose -f docker-compose.yml down -v | |
| docker system prune -f | |
| # ============================================================================ | |
| # FINAL STATUS CHECK | |
| # ============================================================================ | |
| ci-success: | |
| name: ✅ CI Pipeline Success | |
| runs-on: ubuntu-latest | |
| needs: [backend-test, frontend-test, code-quality, security-scan, docker-test] | |
| if: always() | |
| steps: | |
| - name: 🎉 All Checks Passed | |
| if: ${{ needs.backend-test.result == 'success' && needs.frontend-test.result == 'success' && needs.code-quality.result == 'success' && needs.security-scan.result == 'success' && (needs.docker-test.result == 'success' || needs.docker-test.result == 'skipped') }} | |
| run: | | |
| echo "🎉 All CI checks passed successfully!" | |
| echo "### ✅ CI Pipeline Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Backend Tests: ✅ Passed" >> $GITHUB_STEP_SUMMARY | |
| echo "- Frontend Tests: ✅ Passed" >> $GITHUB_STEP_SUMMARY | |
| echo "- Code Quality: ✅ Passed" >> $GITHUB_STEP_SUMMARY | |
| echo "- Security Scan: ✅ Passed" >> $GITHUB_STEP_SUMMARY | |
| echo "- Docker Tests: ✅ Passed" >> $GITHUB_STEP_SUMMARY | |
| - name: ❌ Some Checks Failed | |
| if: ${{ needs.backend-test.result != 'success' || needs.frontend-test.result != 'success' || needs.code-quality.result != 'success' || needs.security-scan.result != 'success' || (needs.docker-test.result != 'success' && needs.docker-test.result != 'skipped') }} | |
| run: | | |
| echo "❌ Some CI checks failed!" | |
| echo "### ❌ CI Pipeline Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Backend Tests: ${{ needs.backend-test.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Frontend Tests: ${{ needs.frontend-test.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Code Quality: ${{ needs.code-quality.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Security Scan: ${{ needs.security-scan.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Docker Tests: ${{ needs.docker-test.result }}" >> $GITHUB_STEP_SUMMARY | |
| exit 1 |