chore: init gsd

.agents/skills/deployment-pipeline-design/references/advanced-strategies.md

@@ -0,0 +1,629 @@
+# Advanced Deployment Strategies Reference
+
+Extended configurations, platform-specific patterns, and advanced rollback strategies.
+Core patterns and decision tables live in [`../SKILL.md`](../SKILL.md).
+
+---
+
+## Platform-Specific Pipeline Configurations
+
+### GitHub Actions — Full Production Pipeline with Reusable Workflows
+
+```yaml
+# .github/workflows/production.yml
+name: Production Pipeline
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      skip_tests:
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  id-token: write   # for OIDC auth to cloud providers
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      image_tag: ${{ steps.meta.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=sha,prefix=,format=short
+
+      - name: Build and push (with layer cache)
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  security-scan:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run Trivy vulnerability scan
+        uses: aquasecurity/trivy-action@0.28.0
+        with:
+          image-ref: ghcr.io/${{ github.repository }}:${{ needs.build.outputs.image_tag }}
+          exit-code: 1
+          severity: CRITICAL,HIGH
+
+      - name: SAST with Semgrep
+        uses: semgrep/semgrep-action@v1
+        with:
+          config: auto
+
+  test:
+    needs: build
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run test suite
+        run: make test-ci
+        env:
+          DATABASE_URL: postgres://postgres:test@localhost/test
+
+  deploy-staging:
+    needs: [test, security-scan]
+    environment:
+      name: staging
+      url: https://staging.example.com
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::123456789:role/deploy-staging
+          aws-region: us-east-1
+
+      - name: Deploy to EKS staging
+        run: |
+          aws eks update-kubeconfig --name my-cluster-staging
+          kubectl set image deployment/my-app \
+            app=ghcr.io/${{ github.repository }}:${{ needs.build.outputs.image_tag }}
+          kubectl rollout status deployment/my-app --timeout=5m
+
+  e2e-tests:
+    needs: deploy-staging
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Playwright E2E suite
+        run: npx playwright test --reporter=github
+        env:
+          BASE_URL: https://staging.example.com
+
+  deploy-production:
+    needs: e2e-tests
+    environment:
+      name: production
+      url: https://app.example.com
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::123456789:role/deploy-production
+          aws-region: us-east-1
+
+      - name: Deploy canary to production
+        run: |
+          aws eks update-kubeconfig --name my-cluster-prod
+          kubectl argo rollouts set image my-app \
+            app=ghcr.io/${{ github.repository }}:${{ needs.build.outputs.image_tag }}
+
+      - name: Monitor canary promotion
+        run: |
+          kubectl argo rollouts status my-app --timeout=30m
+
+      - name: Rollback on failure
+        if: failure()
+        run: kubectl argo rollouts abort my-app
+```
+
+---
+
+### GitLab CI — Multi-Environment Pipeline with Dynamic Environments
+
+```yaml
+# .gitlab-ci.yml
+stages:
+  - build
+  - test
+  - staging
+  - production
+
+variables:
+  DOCKER_DRIVER: overlay2
+  IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+
+build:
+  stage: build
+  image: docker:24
+  services:
+    - docker:24-dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $IMAGE .
+    - docker push $IMAGE
+    - docker tag $IMAGE $CI_REGISTRY_IMAGE:latest
+    - docker push $CI_REGISTRY_IMAGE:latest
+
+test:unit:
+  stage: test
+  image: $IMAGE
+  script:
+    - make test
+  coverage: '/coverage: \d+\.\d+%/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+
+test:security:
+  stage: test
+  image:
+    name: aquasec/trivy:latest
+    entrypoint: [""]
+  script:
+    - trivy image --exit-code 1 --severity CRITICAL,HIGH $IMAGE
+
+deploy:staging:
+  stage: staging
+  environment:
+    name: staging
+    url: https://staging.example.com
+    on_stop: stop:staging
+  script:
+    - kubectl apply -f k8s/staging/
+    - kubectl set image deployment/my-app app=$IMAGE -n staging
+    - kubectl rollout status deployment/my-app -n staging
+  only:
+    - main
+
+stop:staging:
+  stage: staging
+  environment:
+    name: staging
+    action: stop
+  script:
+    - kubectl delete namespace staging --ignore-not-found
+  when: manual
+  only:
+    - main
+
+deploy:production:
+  stage: production
+  environment:
+    name: production
+    url: https://app.example.com
+  script:
+    - kubectl set image deployment/my-app app=$IMAGE -n production
+    - kubectl rollout status deployment/my-app -n production --timeout=10m
+  when: manual
+  only:
+    - main
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      when: manual
+      allow_failure: false
+```
+
+---
+
+### Azure Pipelines — Multi-Stage with Approvals and Environments
+
+```yaml
+# azure-pipelines.yml
+trigger:
+  branches:
+    include:
+      - main
+
+variables:
+  imageRepository: 'myapp'
+  containerRegistry: 'myregistry.azurecr.io'
+  tag: '$(Build.BuildId)'
+
+stages:
+  - stage: Build
+    displayName: 'Build & Test'
+    jobs:
+      - job: BuildAndTest
+        pool:
+          vmImage: ubuntu-latest
+        steps:
+          - task: Docker@2
+            displayName: Build image
+            inputs:
+              command: build
+              repository: $(imageRepository)
+              dockerfile: Dockerfile
+              tags: $(tag)
+
+          - task: Docker@2
+            displayName: Push image
+            inputs:
+              command: push
+              repository: $(imageRepository)
+              tags: $(tag)
+
+          - script: make test
+            displayName: Run tests
+
+  - stage: Staging
+    displayName: 'Deploy to Staging'
+    dependsOn: Build
+    jobs:
+      - deployment: DeployStaging
+        environment: staging
+        pool:
+          vmImage: ubuntu-latest
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - task: KubernetesManifest@0
+                  inputs:
+                    action: deploy
+                    manifests: k8s/staging/*.yaml
+                    containers: $(containerRegistry)/$(imageRepository):$(tag)
+
+  - stage: Production
+    displayName: 'Deploy to Production'
+    dependsOn: Staging
+    jobs:
+      - deployment: DeployProduction
+        environment:
+          name: production
+          resourceType: Kubernetes
+        pool:
+          vmImage: ubuntu-latest
+        strategy:
+          canary:
+            increments: [10, 25, 50]
+            preDeploy:
+              steps:
+                - task: ManualValidation@0
+                  inputs:
+                    notifyUsers: 'release-managers@example.com'
+                    instructions: 'Verify staging metrics. Approve to start canary.'
+                    onTimeout: reject
+            deploy:
+              steps:
+                - task: KubernetesManifest@0
+                  inputs:
+                    action: deploy
+                    manifests: k8s/production/*.yaml
+                    containers: $(containerRegistry)/$(imageRepository):$(tag)
+            postRouteTraffic:
+              steps:
+                - script: ./scripts/verify-deployment.sh https://app.example.com
+            on:
+              failure:
+                steps:
+                  - task: KubernetesManifest@0
+                    inputs:
+                      action: reject
+```
+
+---
+
+## Multi-Region Canary Promotion
+
+Deploy to a pilot region first, validate, then promote to remaining regions in parallel:
+
+```yaml
+# deploy-multiregion.yml (GitHub Actions)
+jobs:
+  deploy-pilot:
+    environment: production-us-east-1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy canary to pilot region
+        run: |
+          aws eks update-kubeconfig --name cluster-us-east-1 --region us-east-1
+          kubectl argo rollouts set image my-app app=$IMAGE
+          kubectl argo rollouts status my-app --timeout=20m
+
+  deploy-secondary:
+    needs: deploy-pilot
+    strategy:
+      matrix:
+        region: [us-west-2, eu-west-1, ap-southeast-1]
+    environment: production-${{ matrix.region }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to ${{ matrix.region }}
+        run: |
+          aws eks update-kubeconfig --name cluster-${{ matrix.region }} \
+            --region ${{ matrix.region }}
+          kubectl argo rollouts set image my-app app=$IMAGE
+          kubectl argo rollouts status my-app --timeout=20m
+```
+
+---
+
+## Advanced Argo Rollouts Patterns
+
+### Experiment-Based Canary (A/B Analysis)
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: my-app
+spec:
+  replicas: 20
+  strategy:
+    canary:
+      canaryService: my-app-canary
+      stableService: my-app-stable
+      trafficRouting:
+        istio:
+          virtualService:
+            name: my-app-vsvc
+      analysis:
+        templates:
+          - templateName: success-rate
+          - templateName: latency-p99
+        startingStep: 1
+        args:
+          - name: service-name
+            value: my-app-canary
+      steps:
+        - setWeight: 5
+        - experiment:
+            templates:
+              - name: baseline
+                specRef: stable
+              - name: canary
+                specRef: canary
+            analyses:
+              - templateName: ab-test
+                requiredForCompletion: true
+        - setWeight: 20
+        - pause: { duration: 10m }
+        - setWeight: 50
+        - pause: { duration: 15m }
+        - setWeight: 100
+```
+
+### Header-Based Canary (Internal Testing Before Traffic Split)
+
+```yaml
+# Route users with X-Canary: true header to canary pods
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: my-app-vsvc
+spec:
+  http:
+    - match:
+        - headers:
+            x-canary:
+              exact: "true"
+      route:
+        - destination:
+            host: my-app-canary
+    - route:
+        - destination:
+            host: my-app-stable
+          weight: 90
+        - destination:
+            host: my-app-canary
+          weight: 10
+```
+
+---
+
+## Database Migration Rollback Strategies
+
+### Expand/Contract Pattern (Safe Schema Changes)
+
+Never break backward compatibility within a single deployment cycle. Use the expand/contract pattern across three releases:
+
+```
+Release N:   Add nullable column (expand)
+Release N+1: Backfill data, deploy new code that reads new column
+Release N+2: Drop old column (contract) — safe because no code references it
+```
+
+```bash
+# Release N migration — backward compatible
+cat > migrations/V20240315__expand_add_email_v2.sql <<'EOF'
+ALTER TABLE users ADD COLUMN email_v2 VARCHAR(255);
+CREATE INDEX CONCURRENTLY idx_users_email_v2 ON users(email_v2);
+EOF
+
+# Release N+2 migration — contract (only after old code retired)
+cat > migrations/V20240415__contract_drop_email.sql <<'EOF'
+ALTER TABLE users DROP COLUMN email;
+ALTER TABLE users RENAME COLUMN email_v2 TO email;
+EOF
+```
+
+### Flyway with Undo Scripts
+
+```bash
+# Forward migration
+flyway migrate
+
+# Undo the most recent migration (requires Flyway Teams)
+flyway undo
+
+# Undo a specific version
+flyway undo -target=20240315
+```
+
+Undo script naming convention: `U20240315__expand_add_email_v2.sql` (prefix `U` instead of `V`).
+
+### Zero-Downtime Index Creation (PostgreSQL)
+
+```sql
+-- Blocking (avoid in production):
+CREATE INDEX idx_users_email ON users(email);
+
+-- Non-blocking (safe for live systems):
+CREATE INDEX CONCURRENTLY idx_users_email ON users(email);
+```
+
+Use `CREATE INDEX CONCURRENTLY` in all production migrations. Add it to a pre-deploy migration step that runs before the application update.
+
+---
+
+## Blue-Green with Database (Full Example)
+
+```yaml
+# blue-green-deploy.yml
+steps:
+  - name: Deploy green environment
+    run: |
+      kubectl apply -f k8s/green/
+
+  - name: Run database migrations (forward, backward-compatible)
+    run: |
+      kubectl exec -n production deploy/migration-job -- flyway migrate
+
+  - name: Smoke test green
+    run: |
+      kubectl port-forward -n production svc/my-app-green 8080:80 &
+      sleep 3
+      curl -sf http://localhost:8080/health/ready
+
+  - name: Switch traffic to green
+    run: |
+      kubectl patch service my-app \
+        -p '{"spec":{"selector":{"slot":"green"}}}'
+
+  - name: Verify green is live
+    run: ./scripts/verify-deployment.sh https://app.example.com
+
+  - name: Scale down blue
+    run: |
+      kubectl scale deployment my-app-blue --replicas=0
+
+  - name: Rollback to blue on failure
+    if: failure()
+    run: |
+      kubectl patch service my-app \
+        -p '{"spec":{"selector":{"slot":"blue"}}}'
+      kubectl scale deployment my-app-blue --replicas=5
+```
+
+---
+
+## Deployment Freeze Automation
+
+Automatically block production deployments during freeze windows:
+
+```python
+#!/usr/bin/env python3
+# scripts/check-freeze-window.py
+import sys
+from datetime import datetime, timezone
+
+FREEZE_WINDOWS = [
+    # (month, day_start, day_end, description)
+    (11, 25, 30, "US Thanksgiving"),
+    (12, 20, 31, "Year-end freeze"),
+    (1, 1, 2, "New Year"),
+]
+
+now = datetime.now(timezone.utc)
+month, day = now.month, now.day
+
+for fm, d_start, d_end, label in FREEZE_WINDOWS:
+    if fm == month and d_start <= day <= d_end:
+        print(f"BLOCKED: Deployment freeze active — {label}")
+        print("Override with FORCE_DEPLOY=true environment variable if critical.")
+        if not os.environ.get("FORCE_DEPLOY"):
+            sys.exit(1)
+
+print("No active freeze window — deployment allowed.")
+```
+
+Use in pipeline:
+
+```yaml
+- name: Check deployment freeze window
+  run: python scripts/check-freeze-window.py
+  env:
+    FORCE_DEPLOY: ${{ vars.FORCE_DEPLOY }}
+```
+
+---
+
+## Notification Templates
+
+### Slack Deployment Notification
+
+```bash
+#!/usr/bin/env bash
+# scripts/notify-slack.sh
+STATUS="${1:?pass 'success' or 'failure'}"
+WEBHOOK="${SLACK_WEBHOOK:?SLACK_WEBHOOK not set}"
+REPO="${GITHUB_REPOSITORY:-unknown}"
+SHA="${GITHUB_SHA:-unknown}"
+ACTOR="${GITHUB_ACTOR:-unknown}"
+
+if [ "$STATUS" = "success" ]; then
+  COLOR="good"
+  EMOJI=":white_check_mark:"
+  TEXT="Production deploy succeeded"
+else
+  COLOR="danger"
+  EMOJI=":red_circle:"
+  TEXT="Production deploy FAILED — rollback triggered"
+fi
+
+curl -sf -X POST "$WEBHOOK" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"attachments\": [{
+      \"color\": \"$COLOR\",
+      \"text\": \"$EMOJI $TEXT\",
+      \"fields\": [
+        {\"title\": \"Repo\", \"value\": \"$REPO\", \"short\": true},
+        {\"title\": \"SHA\", \"value\": \"${SHA:0:7}\", \"short\": true},
+        {\"title\": \"Deployed by\", \"value\": \"$ACTOR\", \"short\": true}
+      ]
+    }]
+  }"
+```

.agents/skills/deployment-pipeline-design/references/details.md

@@ -0,0 +1,399 @@
+# deployment-pipeline-design — detailed patterns and worked examples
+
+## Pipeline Stages
+
+### Standard Pipeline Flow
+
+```
+┌─────────┐   ┌──────┐   ┌─────────┐   ┌────────┐   ┌──────────┐
+│  Build  │ → │ Test │ → │ Staging │ → │ Approve│ → │Production│
+└─────────┘   └──────┘   └─────────┘   └────────┘   └──────────┘
+```
+
+### Detailed Stage Breakdown
+
+1. **Source** - Code checkout, dependency graph resolution
+2. **Build** - Compile, package, containerize, sign artifacts
+3. **Test** - Unit, integration, SAST/SCA security scans
+4. **Staging Deploy** - Deploy to staging environment with smoke tests
+5. **Integration Tests** - E2E, contract tests, performance baselines
+6. **Approval Gate** - Manual or automated metric-based gate
+7. **Production Deploy** - Canary, blue-green, or rolling strategy
+8. **Verification** - Deep health checks, synthetic monitoring
+9. **Rollback** - Automated rollback on failure signals
+
+## Approval Gate Patterns
+
+### Pattern 1: Manual Approval (GitHub Actions)
+
+```yaml
+production-deploy:
+  needs: staging-deploy
+  environment:
+    name: production
+    url: https://app.example.com
+  runs-on: ubuntu-latest
+  steps:
+    - name: Deploy to production
+      run: kubectl apply -f k8s/production/
+```
+
+Environment protection rules in GitHub enforce required reviewers before this job starts. Configure reviewers at **Settings → Environments → production → Required reviewers**.
+
+### Pattern 2: Time-Based Approval (GitLab CI)
+
+```yaml
+deploy:production:
+  stage: deploy
+  script:
+    - deploy.sh production
+  environment:
+    name: production
+  when: delayed
+  start_in: 30 minutes
+  only:
+    - main
+```
+
+### Pattern 3: Multi-Approver (Azure Pipelines)
+
+```yaml
+stages:
+  - stage: Production
+    dependsOn: Staging
+    jobs:
+      - deployment: Deploy
+        environment:
+          name: production
+          resourceType: Kubernetes
+        strategy:
+          runOnce:
+            preDeploy:
+              steps:
+                - task: ManualValidation@0
+                  inputs:
+                    notifyUsers: "team-leads@example.com"
+                    instructions: "Review staging metrics before approving"
+```
+
+### Pattern 4: Automated Metric Gate
+
+Use an AnalysisTemplate (Argo Rollouts) or a custom gate script to block promotion when error rates exceed a threshold:
+
+```yaml
+# Argo Rollouts AnalysisTemplate — blocks canary promotion automatically
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  metrics:
+  - name: success-rate
+    interval: 60s
+    successCondition: "result[0] >= 0.95"
+    failureCondition: "result[0] < 0.90"
+    inconclusiveLimit: 3
+    provider:
+      prometheus:
+        address: http://prometheus:9090
+        query: |
+          sum(rate(http_requests_total{status!~"5..",job="my-app"}[2m]))
+          / sum(rate(http_requests_total{job="my-app"}[2m]))
+```
+
+## Deployment Strategies
+
+### Decision Table
+
+| Strategy     | Downtime | Rollback Speed | Cost Impact     | Best For                        |
+|-------------|----------|----------------|-----------------|----------------------------------|
+| Rolling      | None     | ~minutes       | None            | Most stateless services          |
+| Blue-Green   | None     | Instant        | 2x infra (temp) | High-risk or database migrations |
+| Canary       | None     | Instant        | Minimal         | High-traffic, metric-driven      |
+| Recreate     | Yes      | Fast           | None            | Dev/test, batch jobs             |
+| Feature Flag | None     | Instant        | None            | Gradual feature exposure         |
+
+### 1. Rolling Deployment
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-app
+spec:
+  replicas: 10
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 2         # at most 12 pods during rollout
+      maxUnavailable: 1   # at least 9 pods always serving
+```
+
+Characteristics: gradual rollout, zero downtime, easy rollback, best for most applications.
+
+### 2. Blue-Green Deployment
+
+```bash
+# Switch traffic from blue to green
+kubectl apply -f k8s/green-deployment.yaml
+kubectl rollout status deployment/my-app-green
+
+# Flip the service selector
+kubectl patch service my-app -p '{"spec":{"selector":{"version":"green"}}}'
+
+# Rollback instantly if needed
+kubectl patch service my-app -p '{"spec":{"selector":{"version":"blue"}}}'
+```
+
+Characteristics: instant switchover, easy rollback, doubles infrastructure cost temporarily, good for high-risk deployments with long warm-up times.
+
+### 3. Canary Deployment (Argo Rollouts)
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: my-app
+spec:
+  replicas: 10
+  strategy:
+    canary:
+      analysis:
+        templates:
+          - templateName: success-rate
+        startingStep: 2
+      steps:
+        - setWeight: 10
+        - pause: { duration: 5m }
+        - setWeight: 25
+        - pause: { duration: 5m }
+        - setWeight: 50
+        - pause: { duration: 10m }
+        - setWeight: 100
+```
+
+Characteristics: gradual traffic shift, real-user metric validation, automated promotion or rollback, requires Argo Rollouts or a service mesh.
+
+### 4. Feature Flags
+
+```python
+from flagsmith import Flagsmith
+
+flagsmith = Flagsmith(environment_key="API_KEY")
+
+if flagsmith.has_feature("new_checkout_flow"):
+    process_checkout_v2()
+else:
+    process_checkout_v1()
+```
+
+Characteristics: deploy without releasing, A/B testing, instant rollback per user segment, granular control independent of deployment.
+
+## Pipeline Orchestration
+
+### Multi-Stage Pipeline Example (GitHub Actions)
+
+```yaml
+name: Production Pipeline
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.build.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and push Docker image
+        id: build
+        run: |
+          IMAGE=myapp:${{ github.sha }}
+          docker build -t $IMAGE .
+          docker push $IMAGE
+          echo "image=$IMAGE" >> $GITHUB_OUTPUT
+
+  test:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Unit tests
+        run: make test
+      - name: Security scan
+        run: trivy image ${{ needs.build.outputs.image }}
+
+  deploy-staging:
+    needs: test
+    environment:
+      name: staging
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to staging
+        run: kubectl apply -f k8s/staging/
+
+  integration-test:
+    needs: deploy-staging
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run E2E tests
+        run: npm run test:e2e
+
+  deploy-production:
+    needs: integration-test
+    environment:
+      name: production        # blocks here until required reviewers approve
+    runs-on: ubuntu-latest
+    steps:
+      - name: Canary deployment
+        run: |
+          kubectl apply -f k8s/production/
+          kubectl argo rollouts promote my-app
+
+  verify:
+    needs: deploy-production
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deep health check
+        run: |
+          for i in {1..12}; do
+            STATUS=$(curl -sf https://app.example.com/health/ready | jq -r '.status')
+            [ "$STATUS" = "ok" ] && exit 0
+            sleep 10
+          done
+          exit 1
+      - name: Notify on success
+        run: |
+          curl -X POST ${{ secrets.SLACK_WEBHOOK }} \
+            -d '{"text":"Production deployment successful: ${{ github.sha }}"}'
+```
+
+## Health Checks
+
+### Shallow vs Deep Health Endpoints
+
+A shallow `/ping` returns 200 even when downstream dependencies are broken. Use a deep readiness endpoint that verifies actual dependencies before promoting traffic.
+
+```python
+# /health/ready — checks real dependencies, used by pipeline gate
+@app.get("/health/ready")
+async def readiness():
+    checks = {
+        "database": await check_db_connection(),
+        "cache":    await check_redis_connection(),
+        "queue":    await check_queue_connection(),
+    }
+    status = "ok" if all(checks.values()) else "degraded"
+    code = 200 if status == "ok" else 503
+    return JSONResponse({"status": status, "checks": checks}, status_code=code)
+```
+
+### Post-Deployment Verification Script
+
+```bash
+#!/usr/bin/env bash
+# verify-deployment.sh — run after every production deploy
+set -euo pipefail
+
+ENDPOINT="${1:?usage: verify-deployment.sh <base-url>}"
+MAX_ATTEMPTS=12
+SLEEP_SECONDS=10
+
+for i in $(seq 1 $MAX_ATTEMPTS); do
+  STATUS=$(curl -sf "$ENDPOINT/health/ready" | jq -r '.status' 2>/dev/null || echo "unreachable")
+  if [ "$STATUS" = "ok" ]; then
+    echo "Health check passed after $((i * SLEEP_SECONDS))s"
+    exit 0
+  fi
+  echo "Attempt $i/$MAX_ATTEMPTS: status=$STATUS — retrying in ${SLEEP_SECONDS}s"
+  sleep "$SLEEP_SECONDS"
+done
+
+echo "Health check failed after $((MAX_ATTEMPTS * SLEEP_SECONDS))s"
+exit 1
+```
+
+## Rollback Strategies
+
+### Automated Rollback in Pipeline
+
+```yaml
+deploy-and-verify:
+  steps:
+    - name: Deploy new version
+      run: kubectl apply -f k8s/
+
+    - name: Wait for rollout
+      run: kubectl rollout status deployment/my-app --timeout=5m
+
+    - name: Post-deployment health check
+      id: health
+      run: ./scripts/verify-deployment.sh https://app.example.com
+
+    - name: Rollback on failure
+      if: failure()
+      run: |
+        kubectl rollout undo deployment/my-app
+        echo "Rolled back to previous revision"
+```
+
+### Manual Rollback Commands
+
+```bash
+# List revision history with change-cause annotations
+kubectl rollout history deployment/my-app
+
+# Rollback to previous version
+kubectl rollout undo deployment/my-app
+
+# Rollback to a specific revision
+kubectl rollout undo deployment/my-app --to-revision=3
+
+# Verify rollback completed
+kubectl rollout status deployment/my-app
+```
+
+For advanced rollback strategies including database migration rollbacks and Argo Rollouts abort flows, see [`references/advanced-strategies.md`](references/advanced-strategies.md).
+
+## Monitoring and Metrics
+
+### Key DORA Metrics to Track
+
+| Metric                    | Target (Elite) | How to Measure                           |
+|--------------------------|----------------|------------------------------------------|
+| Deployment Frequency      | Multiple/day   | Pipeline run count per day               |
+| Lead Time for Changes     | < 1 hour       | Commit timestamp → production deploy     |
+| Change Failure Rate       | < 5%           | Failed deploys / total deploys           |
+| Mean Time to Recovery     | < 1 hour       | Incident open → service restored         |
+
+### Post-Deployment Metric Verification
+
+```yaml
+- name: Verify error rate post-deployment
+  run: |
+    sleep 60  # allow metrics to accumulate
+
+    ERROR_RATE=$(curl -sf "$PROMETHEUS_URL/api/v1/query" \
+      --data-urlencode 'query=sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))' \
+      | jq '.data.result[0].value[1]')
+
+    echo "Current error rate: $ERROR_RATE"
+    if (( $(echo "$ERROR_RATE > 0.01" | bc -l) )); then
+      echo "Error rate $ERROR_RATE exceeds 1% threshold — triggering rollback"
+      exit 1
+    fi
+```
+
+## Pipeline Best Practices
+
+1. **Fail fast** — Run quick checks (lint, unit tests) before slow ones (E2E, security scans)
+2. **Parallel execution** — Run independent jobs concurrently to minimize total pipeline time
+3. **Caching** — Cache dependency layers and build artifacts between runs
+4. **Artifact promotion** — Build once, promote the same artifact through all environments
+5. **Environment parity** — Keep staging infrastructure as close to production as possible
+6. **Secrets management** — Use secret stores (Vault, AWS Secrets Manager, GitHub encrypted secrets) — never hardcode
+7. **Deployment windows** — Prefer low-traffic windows; enforce change freeze periods via gate policies
+8. **Idempotent deploys** — Ensure re-running a deploy produces the same result
+9. **Rollback automation** — Trigger rollback automatically on health check or metric threshold failure
+10. **Annotate deployments** — Send deployment markers to monitoring tools (Datadog, Grafana) for correlation