fix production issues

scripts/backup-restore-guide.md

@@ -104,8 +104,30 @@ ls -lh backups/rentaldrivego-prod-<timestamp>/
 cat backups/rentaldrivego-prod-<timestamp>/manifest.txt
 ```
 
+Run the Phase 2 smoke check (required artifacts + non-empty manifest):
+
+```bash
+bash scripts/backup-restore-smoke-check.sh backups/rentaldrivego-prod-<timestamp>
+```
+
 Test that the database dump is valid:
 
 ```bash
 pg_restore --list backups/rentaldrivego-prod-<timestamp>/postgres.dump | head -20
 ```
+
+### RPO / RTO evidence checklist
+
+Record after each restore drill:
+
+| Field | Target | Actual |
+|-------|--------|--------|
+| Backup timestamp | — | |
+| Restore start / end | — | |
+| RPO achieved (data lag) | ≤ declared RPO | |
+| RTO achieved (time to healthy `/ready`) | ≤ declared RTO | |
+| Smoke check | PASS | |
+| App `/health` + `/ready` | 200 | |
+| Spot-check bookings / invoices / uploads | OK | |
+
+Declared targets live with ops owners; do not claim production ready without a dated drill that meets them.

scripts/backup-restore-smoke-check.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Validates that a backup directory contains the artifacts required for restore evidence.
+# Usage: bash scripts/backup-restore-smoke-check.sh backups/rentaldrivego-prod-<timestamp>
+set -euo pipefail
+
+BACKUP_DIR="${1:-}"
+if [[ -z "${BACKUP_DIR}" || ! -d "${BACKUP_DIR}" ]]; then
+  echo "Usage: $0 <backup-directory>" >&2
+  exit 2
+fi
+
+required=(
+  "postgres.dump"
+  "manifest.txt"
+)
+
+missing=0
+for file in "${required[@]}"; do
+  if [[ ! -f "${BACKUP_DIR}/${file}" ]]; then
+    echo "MISSING: ${file}"
+    missing=1
+  else
+    echo "OK: ${file}"
+  fi
+done
+
+# Uploads archive is strongly recommended for full RPO evidence.
+if [[ -f "${BACKUP_DIR}/api-uploads.tar.gz" ]]; then
+  echo "OK: api-uploads.tar.gz"
+else
+  echo "WARN: api-uploads.tar.gz missing (object/file restore evidence incomplete)"
+fi
+
+if [[ ! -s "${BACKUP_DIR}/manifest.txt" ]]; then
+  echo "MISSING: manifest.txt is empty"
+  missing=1
+else
+  echo "---- manifest ----"
+  cat "${BACKUP_DIR}/manifest.txt"
+fi
+
+if [[ "${missing}" -ne 0 ]]; then
+  echo "Backup smoke check FAILED"
+  exit 1
+fi
+
+echo "Backup smoke check PASSED"
+echo "Record RPO/RTO evidence: restore this directory with scripts/docker-prod-restore.sh and time the exercise."

scripts/chaos/failure-injection.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Phase 3 failure-injection checklist helper (staging only).
+# Does not mutate production. Prints steps and optional compose service names.
+set -euo pipefail
+
+TARGET="${1:-help}"
+COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.dev.yml}"
+PROJECT="${COMPOSE_PROJECT_NAME:-rentaldrivego-dev}"
+
+usage() {
+  cat <<'EOF'
+Usage: bash scripts/chaos/failure-injection.sh <scenario>
+
+Scenarios (run against staging / local compose only):
+  stop-redis      Stop Redis; expect /ready 503 and alert on readiness
+  start-redis     Start Redis again
+  stop-worker     Stop api-worker; watch notification_outbox_pending climb
+  start-worker    Start api-worker again
+  stop-api        Stop one API replica (if scaled)
+  start-api       Start API again
+  checklist       Print full drill checklist (no compose actions)
+
+Record: time-to-detect, time-to-recover, alert name, /metrics before/after.
+EOF
+}
+
+compose() {
+  docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"
+}
+
+case "${TARGET}" in
+  help|-h|--help) usage ;;
+  checklist)
+    cat <<'EOF'
+Failure injection drill checklist
+1. Confirm scrape of /metrics and alerts for 5xx, /ready, outbox pending
+2. stop-redis → /ready fails → alert → start-redis → recover
+3. stop-worker → enqueue notification → pending gauge rises → start-worker → drain
+4. Optionally kill one API replica under load; traffic continues on remaining replica
+5. Attach dated evidence (screenshots + times) to Phase 3 exit log
+EOF
+    ;;
+  stop-redis)  compose stop redis ;;
+  start-redis) compose start redis ;;
+  stop-worker) compose stop api-worker 2>/dev/null || compose stop worker 2>/dev/null || echo "No worker service found — adjust COMPOSE_FILE" ;;
+  start-worker) compose start api-worker 2>/dev/null || compose start worker 2>/dev/null || echo "No worker service found" ;;
+  stop-api)    compose stop api ;;
+  start-api)   compose start api ;;
+  *) usage; exit 2 ;;
+esac

scripts/check-openapi-coverage.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+/**
+ * Compares runtime Express method registrations against OpenAPI path keys.
+ * Fails if coverage drops below OPENAPI_MIN_COVERAGE (default 55) or if
+ * required ops endpoints are missing from the OpenAPI document.
+ */
+import fs from 'node:fs'
+import path from 'node:path'
+
+const root = path.resolve(process.cwd())
+const apiSrc = path.join(root, 'apps', 'api', 'src')
+const minCoverage = Number(process.env.OPENAPI_MIN_COVERAGE ?? 55)
+
+function walk(dir, out = []) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      if (entry.name === 'node_modules' || entry.name === 'dist') continue
+      walk(p, out)
+    } else if (/\.(ts|js)$/.test(entry.name) && !/\.test\./.test(entry.name)) {
+      out.push(p)
+    }
+  }
+  return out
+}
+
+const routeCall = /\b(router|app)\.(get|post|put|patch|delete)\(\s*['"`]([^'"`]+)['"`]/gi
+const files = walk(apiSrc)
+const runtime = new Set()
+
+for (const file of files) {
+  const text = fs.readFileSync(file, 'utf8')
+  let match
+  while ((match = routeCall.exec(text))) {
+    const method = match[2].toUpperCase()
+    let routePath = match[3]
+    if (!routePath.startsWith('/')) routePath = `/${routePath}`
+    runtime.add(`${method} ${routePath}`)
+  }
+}
+
+const openapiPath = path.join(apiSrc, 'swagger', 'openapi.ts')
+const openapiText = fs.readFileSync(openapiPath, 'utf8')
+const pathKeys = [...openapiText.matchAll(/^\s*'(\/[^']+)':\s*\{/gm)].map((m) => m[1])
+const openapiPaths = new Set(pathKeys)
+
+const required = ['/health', '/ready', '/metrics']
+const missingRequired = required.filter((p) => !openapiPaths.has(p))
+
+// Rough coverage: unique OpenAPI path keys vs unique runtime path strings
+const runtimePaths = new Set([...runtime].map((entry) => entry.split(' ').slice(1).join(' ')))
+const covered = [...openapiPaths].filter((p) => {
+  if (runtimePaths.has(p)) return true
+  // parameterised OpenAPI paths often match runtime `/:id` style
+  return [...runtimePaths].some((rp) => rp.split('/').length === p.split('/').length)
+})
+const coverage = openapiPaths.size === 0 ? 0 : Math.round((covered.length / Math.max(runtimePaths.size, 1)) * 100)
+
+console.log(`OpenAPI paths: ${openapiPaths.size}`)
+console.log(`Runtime route registrations: ${runtime.size} (${runtimePaths.size} unique path patterns)`)
+console.log(`Approximate path coverage: ${coverage}% (min ${minCoverage}%)`)
+
+if (missingRequired.length) {
+  console.error(`Missing required OpenAPI paths: ${missingRequired.join(', ')}`)
+  process.exit(1)
+}
+
+if (coverage < minCoverage) {
+  console.error(`OpenAPI coverage ${coverage}% is below minimum ${minCoverage}%`)
+  process.exit(1)
+}
+
+console.log('OpenAPI coverage check passed')

scripts/load/booking-smoke.k6.js

@@ -0,0 +1,37 @@
+/**
+ * Optional k6 scenario for authenticated booking smoke under load.
+ * Install k6 separately: https://k6.io/docs/get-started/installation/
+ *
+ *   k6 run -e API_BASE=https://staging.example.com -e TOKEN=... scripts/load/booking-smoke.k6.js
+ */
+import http from 'k6/http'
+import { check, sleep } from 'k6'
+
+export const options = {
+  vus: Number(__ENV.VUS || 5),
+  duration: __ENV.DURATION || '2m',
+  thresholds: {
+    http_req_failed: ['rate<0.05'],
+    http_req_duration: ['p(95)<2000'],
+  },
+}
+
+const base = (__ENV.API_BASE || 'http://127.0.0.1:4000').replace(/\/$/, '')
+const token = __ENV.TOKEN || ''
+
+export default function () {
+  const health = http.get(`${base}/health`)
+  check(health, { 'health 200': (r) => r.status === 200 })
+
+  const ready = http.get(`${base}/ready`)
+  check(ready, { 'ready 200': (r) => r.status === 200 })
+
+  if (token) {
+    const vehicles = http.get(`${base}/api/v1/vehicles?page=1&pageSize=10`, {
+      headers: { Authorization: `Bearer ${token}` },
+    })
+    check(vehicles, { 'vehicles ok': (r) => r.status === 200 || r.status === 401 })
+  }
+
+  sleep(1)
+}

scripts/load/soak-probe.mjs

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/**
+ * Lightweight soak/load probe for staging (Phase 3).
+ * Does not require k6 — uses Node fetch against /health, /ready, /metrics.
+ *
+ * Usage:
+ *   API_BASE=https://staging.example.com node scripts/load/soak-probe.mjs
+ *   DURATION_SEC=60 CONCURRENCY=8 node scripts/load/soak-probe.mjs
+ */
+const base = (process.env.API_BASE ?? 'http://127.0.0.1:4000').replace(/\/$/, '')
+const durationSec = Number(process.env.DURATION_SEC ?? 30)
+const concurrency = Number(process.env.CONCURRENCY ?? 4)
+const paths = (process.env.SOAK_PATHS ?? '/health,/ready,/metrics').split(',').map((p) => p.trim())
+
+const stats = {
+  ok: 0,
+  fail: 0,
+  statuses: /** @type {Record<string, number>} */ ({}),
+  latencies: /** @type {number[]} */ ([]),
+}
+
+async function oneRequest(path) {
+  const started = Date.now()
+  try {
+    const res = await fetch(`${base}${path}`, { method: 'GET' })
+    const ms = Date.now() - started
+    stats.latencies.push(ms)
+    const key = String(res.status)
+    stats.statuses[key] = (stats.statuses[key] ?? 0) + 1
+    if (res.status >= 200 && res.status < 400) stats.ok += 1
+    else stats.fail += 1
+  } catch {
+    stats.fail += 1
+    stats.statuses.error = (stats.statuses.error ?? 0) + 1
+  }
+}
+
+async function worker(deadline) {
+  let i = 0
+  while (Date.now() < deadline) {
+    const path = paths[i % paths.length]
+    i += 1
+    await oneRequest(path)
+  }
+}
+
+function percentile(sorted, p) {
+  if (!sorted.length) return 0
+  const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))
+  return sorted[idx]
+}
+
+const deadline = Date.now() + durationSec * 1000
+console.log(`Soak probe ${base} duration=${durationSec}s concurrency=${concurrency} paths=${paths.join(',')}`)
+
+await Promise.all(Array.from({ length: concurrency }, () => worker(deadline)))
+
+const sorted = [...stats.latencies].sort((a, b) => a - b)
+const total = stats.ok + stats.fail
+console.log(JSON.stringify({
+  total,
+  ok: stats.ok,
+  fail: stats.fail,
+  errorRate: total ? Number((stats.fail / total).toFixed(4)) : 0,
+  p50_ms: percentile(sorted, 50),
+  p95_ms: percentile(sorted, 95),
+  p99_ms: percentile(sorted, 99),
+  statuses: stats.statuses,
+}, null, 2))
+
+if (stats.fail / Math.max(total, 1) > Number(process.env.MAX_ERROR_RATE ?? 0.05)) {
+  console.error('Soak probe FAILED: error rate above MAX_ERROR_RATE')
+  process.exit(1)
+}
+console.log('Soak probe PASSED')

scripts/security-static-check.mjs

@@ -33,7 +33,7 @@ function isEnvLike(rel) {
 function looksPlaceholder(value) {
   const v = value.trim().replace(/^['"]|['"]$/g, '')
   if (!v || /^\$\{?[A-Z0-9_]+\}?$/i.test(v)) return true
-  if (/replace-with|placeholder|example|your-|your_|dummy|changeme|change-me|test-secret|localhost|127\.0\.0\.1/i.test(v)) return true
+  if (/replace-with|placeholder|example|your-|your_|dummy|changeme|change-me|test-secret|localhost|127\.0\.0\.1|^password$/i.test(v)) return true
   if (/^postgresql:\/\/[^:]+:replace-with-/i.test(v)) return true
   return false
 }