#!/usr/bin/env bash # Phase 3 failure-injection checklist helper (staging only). # Does not mutate production. Prints steps and optional compose service names. set -euo pipefail TARGET="${1:-help}" COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.dev.yml}" PROJECT="${COMPOSE_PROJECT_NAME:-rentaldrivego-dev}" usage() { cat <<'EOF' Usage: bash scripts/chaos/failure-injection.sh Scenarios (run against staging / local compose only): stop-redis Stop Redis; expect /ready 503 and alert on readiness start-redis Start Redis again stop-worker Stop api-worker; watch notification_outbox_pending climb start-worker Start api-worker again stop-api Stop one API replica (if scaled) start-api Start API again checklist Print full drill checklist (no compose actions) Record: time-to-detect, time-to-recover, alert name, /metrics before/after. EOF } compose() { docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@" } case "${TARGET}" in help|-h|--help) usage ;; checklist) cat <<'EOF' Failure injection drill checklist 1. Confirm scrape of /metrics and alerts for 5xx, /ready, outbox pending 2. stop-redis → /ready fails → alert → start-redis → recover 3. stop-worker → enqueue notification → pending gauge rises → start-worker → drain 4. Optionally kill one API replica under load; traffic continues on remaining replica 5. Attach dated evidence (screenshots + times) to Phase 3 exit log EOF ;; stop-redis) compose stop redis ;; start-redis) compose start redis ;; stop-worker) compose stop api-worker 2>/dev/null || compose stop worker 2>/dev/null || echo "No worker service found — adjust COMPOSE_FILE" ;; start-worker) compose start api-worker 2>/dev/null || compose start worker 2>/dev/null || echo "No worker service found" ;; stop-api) compose stop api ;; start-api) compose start api ;; *) usage; exit 2 ;; esac