fix production issues

scripts/chaos/failure-injection.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Phase 3 failure-injection checklist helper (staging only).
+# Does not mutate production. Prints steps and optional compose service names.
+set -euo pipefail
+
+TARGET="${1:-help}"
+COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.dev.yml}"
+PROJECT="${COMPOSE_PROJECT_NAME:-rentaldrivego-dev}"
+
+usage() {
+  cat <<'EOF'
+Usage: bash scripts/chaos/failure-injection.sh <scenario>
+
+Scenarios (run against staging / local compose only):
+  stop-redis      Stop Redis; expect /ready 503 and alert on readiness
+  start-redis     Start Redis again
+  stop-worker     Stop api-worker; watch notification_outbox_pending climb
+  start-worker    Start api-worker again
+  stop-api        Stop one API replica (if scaled)
+  start-api       Start API again
+  checklist       Print full drill checklist (no compose actions)
+
+Record: time-to-detect, time-to-recover, alert name, /metrics before/after.
+EOF
+}
+
+compose() {
+  docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"
+}
+
+case "${TARGET}" in
+  help|-h|--help) usage ;;
+  checklist)
+    cat <<'EOF'
+Failure injection drill checklist
+1. Confirm scrape of /metrics and alerts for 5xx, /ready, outbox pending
+2. stop-redis → /ready fails → alert → start-redis → recover
+3. stop-worker → enqueue notification → pending gauge rises → start-worker → drain
+4. Optionally kill one API replica under load; traffic continues on remaining replica
+5. Attach dated evidence (screenshots + times) to Phase 3 exit log
+EOF
+    ;;
+  stop-redis)  compose stop redis ;;
+  start-redis) compose start redis ;;
+  stop-worker) compose stop api-worker 2>/dev/null || compose stop worker 2>/dev/null || echo "No worker service found — adjust COMPOSE_FILE" ;;
+  start-worker) compose start api-worker 2>/dev/null || compose start worker 2>/dev/null || echo "No worker service found" ;;
+  stop-api)    compose stop api ;;
+  start-api)   compose start api ;;
+  *) usage; exit 2 ;;
+esac