carmanagement/scripts/chaos/failure-injection.sh

#!/usr/bin/env bash
# Phase 3 failure-injection checklist helper (staging only).
# Does not mutate production. Prints steps and optional compose service names.
set -euo pipefail

TARGET="${1:-help}"
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.dev.yml}"
PROJECT="${COMPOSE_PROJECT_NAME:-rentaldrivego-dev}"

usage() {
  cat <<'EOF'
Usage: bash scripts/chaos/failure-injection.sh <scenario>

Scenarios (run against staging / local compose only):
  stop-redis      Stop Redis; expect /ready 503 and alert on readiness
  start-redis     Start Redis again
  stop-worker     Stop api-worker; watch notification_outbox_pending climb
  start-worker    Start api-worker again
  stop-api        Stop one API replica (if scaled)
  start-api       Start API again
  checklist       Print full drill checklist (no compose actions)

Record: time-to-detect, time-to-recover, alert name, /metrics before/after.
EOF
}

compose() {
  docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"
}

case "${TARGET}" in
  help|-h|--help) usage ;;
  checklist)
    cat <<'EOF'
Failure injection drill checklist
1. Confirm scrape of /metrics and alerts for 5xx, /ready, outbox pending
2. stop-redis → /ready fails → alert → start-redis → recover
3. stop-worker → enqueue notification → pending gauge rises → start-worker → drain
4. Optionally kill one API replica under load; traffic continues on remaining replica
5. Attach dated evidence (screenshots + times) to Phase 3 exit log
EOF
    ;;
  stop-redis)  compose stop redis ;;
  start-redis) compose start redis ;;
  stop-worker) compose stop api-worker 2>/dev/null || compose stop worker 2>/dev/null || echo "No worker service found — adjust COMPOSE_FILE" ;;
  start-worker) compose start api-worker 2>/dev/null || compose start worker 2>/dev/null || echo "No worker service found" ;;
  stop-api)    compose stop api ;;
  start-api)   compose start api ;;
  *) usage; exit 2 ;;
esac