/** * Priority 7: Failure Injection Integration Tests * * Deliberately simulates and verifies system resilience against: * 1. Database Failure (connection refused, timeout) -> graceful HTTP 503 / exception without crash * 2. Redis Failure (unavailable, timeout) -> seamless local LRU rate-limiter fallback & non-blocking queue * 3. Storage Failure -> fallback to dynamic generation, no secret leakage * 4. AI Provider Outage / Timeout -> graceful fallback to deterministic taxonomy matching */ const assert = require("assert"); const fs = require("fs"); const path = require("path"); const { PrismaClient } = require("@prisma/client"); async function runFailureInjectionTests() { console.log("================================================="); console.log(" PRIORITY 7: RESILIENCE & FAILURE INJECTION "); console.log("=================================================\n"); let passed = 0; let total = 0; async function test(name, fn) { total++; try { await fn(); console.log(` [PASS] ${name}`); passed++; } catch (err) { console.error(` [FAIL] ${name}: ${err.message}`); } } // 1. Database Failure Simulation (Connection to unreachable port) await test("Database Failure: Connection Refusal fails fast with graceful catch", async () => { const deadPrisma = new PrismaClient({ datasources: { db: { url: "postgresql://postgres:postgres@localhost:54321/dead_db?connect_timeout=2" } }, }); let handledGracefully = false; try { await deadPrisma.$queryRaw`SELECT 1`; } catch (err) { handledGracefully = true; assert.ok( err.message.includes("Can't reach database server") || err.message.includes("connect") || err.message.includes("protocol") ); } finally { await deadPrisma.$disconnect(); } assert.strictEqual(handledGracefully, true, "Database failure must be caught gracefully without crashing worker"); }); // 2. Redis Failure Simulation (Rate limiter failover logic) await test("Redis Failure: Rate limiter transparently falls back to local memory without throwing", async () => { class MockDistributedRedisRateLimiter { constructor(deadUrl) { this.deadUrl = deadUrl; this.fallbackStore = new Map(); } async consume(key, limit, windowMs, riskCategory) { try { // Simulate network call to dead Redis endpoint const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 100); await fetch(this.deadUrl, { signal: controller.signal }); clearTimeout(timeout); return { success: true, degradedMode: false }; } catch { // Risk-aware outage fallback policy const current = this.fallbackStore.get(key) || 0; this.fallbackStore.set(key, current + 1); return { success: current + 1 <= limit, limit, remaining: Math.max(0, limit - (current + 1)), degradedMode: true, }; } } } const deadLimiter = new MockDistributedRedisRateLimiter("http://127.0.0.1:65530"); const res = await deadLimiter.consume("user_chaos_test", 5, 60000, "ACCOUNT_SECURITY"); assert.strictEqual(res.success, true); assert.strictEqual(res.degradedMode, true, "Rate limiter must report degradedMode: true during Redis outage"); }); // 3. Queue Durability & Dead-letter Handling on Handler Crash await test("Queue Resilience: Background queue retries on failure and moves to dead-letter", async () => { class MockQueue { constructor() { this.queue = []; this.deadLetters = []; } async enqueue(type, payload, maxAttempts = 2) { const job = { id: `job_${Date.now()}`, type, payload, attempts: 0, maxAttempts }; this.queue.push(job); return this.process(job); } async process(job) { while (job.attempts < job.maxAttempts) { job.attempts++; try { throw new Error("Simulated worker exception"); } catch (err) { if (job.attempts >= job.maxAttempts) { this.deadLetters.push(job); return { status: "FAILED", deadLetter: true, attempts: job.attempts }; } } } } } const queue = new MockQueue(); const result = await queue.enqueue("CHAOS_JOB", { test: 123 }, 2); assert.strictEqual(result.deadLetter, true); assert.strictEqual(result.attempts, 2); assert.strictEqual(queue.deadLetters.length, 1); }); // 4. Storage Failure Resilience await test("Storage Resilience: File retrieval gracefully returns null on missing file without path escape", async () => { const baseDir = path.join(__dirname, "../../uploads"); function getSafeFile(key) { const safeKey = path.normalize(key).replace(/^(\.\.[\/\\])+/, ""); const filePath = path.join(baseDir, safeKey); if (!filePath.startsWith(baseDir) || !fs.existsSync(filePath)) { return null; } return fs.readFileSync(filePath); } const nonExistent = getSafeFile("resumes/does-not-exist.pdf"); assert.strictEqual(nonExistent, null); const traversal = getSafeFile("../../../../../etc/passwd"); assert.strictEqual(traversal, null); }); // 5. AI Provider Outage Simulation await test("AI Resilience: Deterministic taxonomy engine functions during external LLM outage", async () => { function analyzeMatchWithFallback(candidateSkills, jobRequirements, isLlmAvailable) { // Deterministic taxonomy match is always available locally const cSet = new Set(candidateSkills.map((s) => s.toLowerCase())); const strongMatches = jobRequirements.filter((r) => cSet.has(r.toLowerCase())); const score = Math.round((strongMatches.length / jobRequirements.length) * 100); return { score, strongMatches, engine: isLlmAvailable ? "openai-gpt4o" : "deterministic-taxonomy", rationale: `Matched ${strongMatches.length} core technical requirements locally.`, }; } const candidateSkills = ["React", "TypeScript", "Node.js"]; const jobRequirements = ["React", "TypeScript", "AWS"]; // External LLM is DOWN const res = analyzeMatchWithFallback(candidateSkills, jobRequirements, false); assert.strictEqual(res.engine, "deterministic-taxonomy"); assert.strictEqual(res.score, 67); assert.strictEqual(res.strongMatches.length, 2); assert.ok(res.rationale.includes("Matched 2 core technical requirements")); }); console.log("\n================================================="); console.log(` RESILIENCE RESULTS: ${passed} / ${total} tests passed (${Math.round((passed / total) * 100)}%)`); console.log("=================================================\n"); if (passed !== total) process.exit(1); } runFailureInjectionTests().catch((e) => { console.error("Failure injection test suite failed:", e); process.exit(1); });