From Production Incidents to JUnit 5 Annotations: A Three-Tier Chaos Testing Framework

                      // The post-mortem's action item, as a test:
@Test
@IncidentChaosK8sRollingUpdateRst(toxicity = 0.3)
void service_survives_rolling_update_rst_storm() {
    // 30% of RECV calls return ECONNRESET
    // This is exactly what a Kubernetes rolling update looks like
    assertThat(callService("/api/health")).isEqualTo(200);
}
                    

                      // Network — connect/recv/send
@ChaosConnectEconnrefused(probability = 0.10)
@ChaosConnectEtimedout(probability = 0.03)
@ChaosRecvEconnreset(probability = 0.05)
@ChaosSendEpipe(probability = 0.02)

// DNS — getaddrinfo
@ChaosDnsGetaddrinfoEaiAgain(hostPattern = "*.internal", probability = 0.20)
@ChaosDnsGetaddrinfoEaiFail(hostPattern = "legacy.svc", probability = 1.0)

// File I/O — writes and reads
@ChaosWriteTorn(pathPrefix = "/data/wal", probability = 0.05)
@ChaosReadEio(pathPrefix = "/etc/app", probability = 0.01)
@ChaosWriteEnospc(pathPrefix = "/data", probability = 0.001)

// Memory
@ChaosMmapEnomem(probability = 0.005)

// Timing — breaks JWT expiry checks, caching TTLs
@ChaosClock(effect = ClockEffect.OFFSET, offsetMs = 30_000)  // +30 second skew

// All annotations support:
//   probability  — 0.0 (never) to 1.0 (always)
//   id           — target a specific container by ID
//   onMissingEnv — ERROR (fail test) or ABORT (skip test) if chaos env unavailable
                    

                      // L3 Incident: Kubernetes Rolling Update RST Storm
// Source: Kubernetes GitHub Issue #56903, multiple production incidents
// Severity: CRITICAL
// What happens: During rolling updates, iptables rules update asynchronously.
// Old pods receive connection attempts for ~15-30s after replacement.
// Active connections to old pods get ECONNRESET when the pod terminates.
@IncidentChaosK8sRollingUpdateRst(toxicity = 0.3)

// L3 Incident: Feign Retry Amplification Storm
// Source: Documented in multiple high-traffic Java service incidents
// Severity: CRITICAL
// What happens: Feign default retry (5x) × replicas (3) × backends (N)
// creates multiplicative amplification. A 50% failure rate becomes 9x load.
@IncidentChaosFeignRetryAmplification(toxicity = 0.5)

// L3 Incident: Spring @Transactional(REQUIRES_NEW) Deadlock
// Source: Spring Framework known issue pattern
// Severity: SEVERE
// What happens: REQUIRES_NEW suspends outer transaction and borrows new
// connection. Under load, pool exhausts — outer tx holds one connection,
// REQUIRES_NEW waits for another, deadlock.
@IncidentChaosSpringTransactionalPoolDeadlock

// L3 Incident: Kubernetes DNS ndots=5 Storm
// Source: CoreDNS performance analysis, ndots search path behavior
// Severity: SEVERE
// What happens: With ndots:5 (K8s default), each hostname generates 5+
// DNS queries (search path expansion). Under CoreDNS load, EAI_AGAIN
// spikes. Services that don't handle DNS transient failures cascade.
@IncidentChaosK8sDnsNdots5Storm

// L3 Incident: JVM Code Cache Full
// Source: JVM JIT compilation documentation, production profiling
// Severity: SEVERE
// What happens: When code cache fills, JIT stops compiling new methods.
// Throughput drops 10–50x as hot paths deoptimize to interpreted mode.
@IncidentChaosJvmCodeCacheFull

// L3 Incident: Redis Sentinel Failover Storm
// Source: Redis Sentinel documentation, observed failover cascades
// Severity: MEDIUM
// What happens: Sentinel failover triggers connection flap on all clients.
// Applications that don't handle LOADING errors loop reconnecting,
// overwhelming the new master.
@IncidentChaosRedisNetworkFlap
                    

                      // @SyscallLevelChaos declares which libraries to inject
// The framework handles the rest: image detection, binary selection, injection

@Testcontainers
@ExtendWith(ChaosTestingExtension.class)
@SyscallLevelChaos({LibchaosLib.NET, LibchaosLib.DNS})  // Inject both
class InventoryServiceChaosTest {

    @Container @AppContainer
    static GenericContainer<?> inventory =
        new GenericContainer<>("inventory-service:latest")  // Any image works
            .withExposedPorts(8080);

    @Container
    static PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:16");

    @Test
    @CompositeChaosTransientDnsFailure  // L2: 15% EAI_AGAIN on DNS
    void inventory_lookup_resilient_to_dns_hiccups(String inventoryUrl) {
        // libchaos-dns.so is active inside the inventory-service container
        // DNS calls from the service will see 15% EAI_AGAIN failures
        var response = given().get(inventoryUrl + "/api/product/123");
        assertThat(response.statusCode()).isEqualTo(200);
    }
}
                    

                      // Test your service under the same resource constraints as production
@Testcontainers
@ExtendWith(ChaosTestingExtension.class)
@SyscallLevelChaos(LibchaosLib.NET)
@Resources(cpuQuota = 0.5, memoryMb = 256)  // Same limits as production K8s pod
class ResourceConstrainedChaosTest {

    @Container @AppContainer
    static GenericContainer<?> app =
        new GenericContainer<>("my-service:latest")
            .withExposedPorts(8080);

    @Test
    @IncidentChaosJvmCodeCacheFull  // Code cache fills → JIT stops
    void service_degrades_gracefully_under_resource_pressure(String appUrl) {
        // CPU-throttled + code cache pressure = realistic production stress test
        var response = given().get(appUrl + "/api/health");
        // Service should return 200 or 503 — not hang forever
        assertThat(response.statusCode()).isIn(200, 503);
        assertThat(response.time()).isLessThan(5000);  // Under 5s even degraded
    }
}
                    

                      // Registering a custom container type
public class MyServiceChaosPlugin implements ChaosPlugin {

    @Override
    public boolean supportsContainer(GenericContainer<?> container) {
        return container.getDockerImageName().startsWith("my-service:");
    }

    @Override
    public ConnectionInfo extractConnectionInfo(GenericContainer<?> container,
                                               Annotation annotation) {
        return ConnectionInfo.of(
            "http://" + container.getHost() + ":" + container.getMappedPort(8080)
        );
    }

    @Override
    public Class<? extends Annotation> containerAnnotation() {
        return AppContainer.class;  // @AppContainer marks this container
    }
}
// META-INF/services/com.macstab.chaos.core.spi.ChaosPlugin:
// com.example.chaos.MyServiceChaosPlugin