Implement retry, backoff

pom.xml

@@ -28,7 +28,19 @@
     </scm>
     <properties>
         <java.version>25</java.version>
+        <spring-cloud.version>2025.1.1</spring-cloud.version>
     </properties>
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.cloud</groupId>
+                <artifactId>spring-cloud-dependencies</artifactId>
+                <version>${spring-cloud.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
     <dependencies>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -43,7 +55,11 @@
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
             <version>3.0.2</version>
         </dependency>
-
+        <!-- Source: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aspectj -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aspectj</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-devtools</artifactId>
@@ -86,6 +102,11 @@
             <artifactId>spring-restdocs-mockmvc</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.wiremock.integrations</groupId>
+            <artifactId>wiremock-spring-boot</artifactId>
+            <version>4.0.9</version>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers-junit-jupiter</artifactId>
@@ -95,6 +116,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-validation</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-circuitbreaker-reactor-resilience4j</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -117,11 +142,6 @@
                     </execution>
                 </executions>
                 <dependencies>
-                    <dependency>
-                        <groupId>org.springframework.restdocs</groupId>
-                        <artifactId>spring-restdocs-asciidoctor</artifactId>
-                        <version>${spring-restdocs.version}</version>
-                    </dependency>
                 </dependencies>
             </plugin>
             <plugin>

src/main/java/org/example/projectbifrost/BifrostController.java

@@ -5,7 +5,6 @@
 import org.example.projectbifrost.domain.ChatSession;
 import org.example.projectbifrost.dto.ChatRequestDTO;
 import org.example.projectbifrost.service.ChatService;
-import org.slf4j.Logger;
 import org.springframework.web.bind.annotation.*;
 
 @Slf4j
@@ -28,7 +27,7 @@ public String bifrost() {
     @PostMapping("/v1/chat")
     public String sendChatRequest(@Valid @RequestBody ChatRequestDTO dto) {
         log.info("Received chat request: Personality={}, SessionId={}", dto.personality(), maskSessionId(dto.sessionId()));
-        return chatService.sendRequestToLLM(dto);
+        return chatService.chatWithLLM(dto);
     }
 
     @GetMapping("/v1/chat/{sessionId}")

src/main/java/org/example/projectbifrost/configuration/RestClientConfiguration.java

@@ -1,21 +1,34 @@
 package org.example.projectbifrost.configuration;
 
+import org.apache.hc.client5.http.impl.classic.HttpClients;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.resilience.annotation.EnableResilientMethods;
 import org.springframework.web.client.RestClient;
 
 import java.net.URI;
 
 @Configuration
+@EnableResilientMethods
 public class RestClientConfiguration {
     @Value("${openrouter.api.key}")
     private String apiKey;
+    @Value("${openrouter.base-url}")
+    private String baseUrl;
 
     @Bean
-    public RestClient openAIWebClient() {
-        return RestClient.builder()
-                .baseUrl(URI.create("https://openrouter.ai/api/v1"))
+    public RestClient openRouterRestClient(RestClient.Builder builder) {
+        var httpClient = HttpClients.custom()
+                .disableAutomaticRetries()
+                .build();
+
+        var requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
+
+        return builder
+                .requestFactory(requestFactory)
+                .baseUrl(URI.create(baseUrl))
                 .defaultHeader("Authorization", "Bearer " + apiKey)
                 .defaultHeader("Content-Type", "application/json")
                 .build();

src/main/java/org/example/projectbifrost/exception/GlobalExceptionHandler.java

@@ -17,8 +17,9 @@
 public class GlobalExceptionHandler {
     private static final String TIMESTAMP_PROPERTY = "timestamp";
 
-    @ExceptionHandler(LLMException.class)
-    public ResponseEntity<ProblemDetail> handleLLMException(LLMException ex) {
+    //General LLM errors - captures any error response from the LLM service and translates it to a client-friendly format
+    @ExceptionHandler(InvalidLLMResponseException.class)
+    public ResponseEntity<ProblemDetail> handleInvalidLLMResponseException(InvalidLLMResponseException ex) {
         HttpStatus status;
         try {
             status = HttpStatus.valueOf(ex.getStatusCode()); //Get the error from OpenRouter
@@ -34,6 +35,18 @@ public ResponseEntity<ProblemDetail> handleLLMException(LLMException ex) {
         return ResponseEntity.status(status).body(problem);
     }
 
+    @ExceptionHandler(RetryableHttpException.class)
+    public ResponseEntity<ProblemDetail> handleRetryableException(RetryableHttpException ex) {
+        log.warn("LLM service unavailable after retries: {}", ex.getMessage());
+        ProblemDetail problem = ProblemDetail.forStatusAndDetail(
+                HttpStatus.SERVICE_UNAVAILABLE,
+                "The Gods are silent - service temporarily unavailable"
+        );
+        problem.setProperty(TIMESTAMP_PROPERTY, Instant.now().toString());
+        problem.setTitle("Service Unavailable");
+        return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body(problem);
+    }
+
     @ExceptionHandler(ResourceAccessException.class)
     public ResponseEntity<ProblemDetail> handleTimeoutException(ResourceAccessException ex) {
         log.warn("Timeout connecting to LLM: {}", ex.getMessage());

src/main/java/org/example/projectbifrost/exception/LLMException.java → src/main/java/org/example/projectbifrost/exception/InvalidLLMResponseException.java

@@ -3,11 +3,11 @@
 import lombok.Getter;
 
 @Getter
-public class LLMException extends RuntimeException {
+public class InvalidLLMResponseException extends RuntimeException {
     private final int statusCode;
     private final String model;
 
-    public LLMException(String message, String model, int statusCode) {
+    public InvalidLLMResponseException(String message, String model, int statusCode) {
         super(message);
         this.statusCode = statusCode;
         this.model = model;

src/main/java/org/example/projectbifrost/exception/RetryableHttpException.java

@@ -0,0 +1,7 @@
+package org.example.projectbifrost.exception;
+
+public class RetryableHttpException extends RuntimeException {
+    public RetryableHttpException(String message) {
+        super(message);
+    }
+}

src/main/java/org/example/projectbifrost/service/ChatService.java

@@ -1,21 +1,25 @@
 package org.example.projectbifrost.service;
 
+import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
+import io.github.resilience4j.retry.annotation.Retry;
+import lombok.extern.slf4j.Slf4j;
 import org.example.projectbifrost.domain.ChatMessage;
 import org.example.projectbifrost.domain.ChatSession;
 import org.example.projectbifrost.dto.ChatRequestDTO;
 import org.example.projectbifrost.dto.OpenRouterRequestDTO;
 import org.example.projectbifrost.dto.OpenRouterResponseDTO;
-import org.example.projectbifrost.exception.LLMException;
+import org.example.projectbifrost.exception.InvalidLLMResponseException;
+import org.example.projectbifrost.exception.RetryableHttpException;
 import org.example.projectbifrost.storage.ChatSessionStorage;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpStatus;
-import org.springframework.http.HttpStatusCode;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestClient;
 
 import java.util.ArrayList;
 import java.util.List;
 
+@Slf4j
 @Service
 public class ChatService {
 
@@ -30,15 +34,18 @@ public ChatService(ChatSessionStorage chatSessionStorage, RestClient restClient)
         this.restClient = restClient;
     }
 
+
     /**
-     * Sends a request to the large language model (LLM) with the user's message and personality context
-     * and retrieves the response.
-     * <p>
-     * This method constructs a chat session using the provided session ID or creates a new one if it
-     * does not exist. It adds the user's message to the chat session and builds a context-aware request
-     * to be sent to the LLM. The LLM's response is then returned as a string.
+     * Sends a chat request to a large language model (LLM) using the supplied chat request data.
+     * The method manages the chat session, compiles the request for the LLM, and processes the response.
+     * It also maintains the history of chat messages within the session.
+     *
+     * @param dto the {@link ChatRequestDTO} containing the user message, chat session ID, and personality configuration.
+     * @return the response from the LLM as a string.
+     * @throws InvalidLLMResponseException if an error occurs during communication with the LLM,
+     *                                     or if the LLM response is empty or malformed.
      */
-    public String sendRequestToLLM(ChatRequestDTO dto) {
+    public String chatWithLLM(ChatRequestDTO dto) {
         ChatSession chatSession = chatSessionStorage.getOrCreateChatSession(dto.sessionId());
 
         List<OpenRouterRequestDTO.Message> apiMessages = new ArrayList<>();
@@ -49,32 +56,53 @@ public String sendRequestToLLM(ChatRequestDTO dto) {
         );
 
         apiMessages.add(new OpenRouterRequestDTO.Message("user", dto.message()));
+        String content = fetchResponseFromLLM(apiMessages);
+
+        // Don't pollute chat history with the circuit-breaker sentinel.
+        if (!"Fallback!".equals(content)) {
+            chatSession.addMessage(new ChatMessage("user", dto.message()));
+            chatSession.addMessage(new ChatMessage("assistant", content));
+        }
+
+        return content;
 
+    }
+
+    @CircuitBreaker(name = "chatService", fallbackMethod = "fallback")
+    //Break stream of tries if too many failures, and call fallback method
+    @Retry(name = "chatService") //Try again if fails, up to max-attempts
+    public String fetchResponseFromLLM(List<OpenRouterRequestDTO.Message> apiMessages) {
         var openRouterRequest = new OpenRouterRequestDTO(model, apiMessages);
 
         OpenRouterResponseDTO result = restClient.post()
                 .uri("/chat/completions")//Start of URI configured in RestClientConfiguration.java
                 .body(openRouterRequest) //Send JSON body of messages and model
                 .retrieve()
-                .onStatus(HttpStatusCode::isError, (request, response) -> {
-                    throw new LLMException("The Gods are silent", model, response.getStatusCode().value());
-                })
+                .onStatus(s -> s.value() == 429 || s.value() == 500,
+                        (req, resp) -> {
+                            throw new RetryableHttpException(
+                                    "Upstream LLM returned " + resp.getStatusCode().value()
+                            );
+                        })
                 .body(OpenRouterResponseDTO.class);
 
         if (result == null || result.choices() == null || result.choices().isEmpty()) {
-            throw new LLMException(
+            throw new InvalidLLMResponseException(
                     "The Gods sent an empty omen (Invalid response from LLM)",
                     model,
                     HttpStatus.BAD_GATEWAY.value()
             );
         }
-        String content = result.choices().getFirst().message().content();
+        return result.choices().getFirst().message().content();
 
-        chatSession.addMessage(new ChatMessage("user", dto.message()));
-        chatSession.addMessage(new ChatMessage("assistant", content));
-        return content;
     }
 
+    public String fallback(Exception e) {
+        // Fallback handles all errors from circuit breaker
+        // Log the circuit breaker event and return graceful fallback response
+        log.warn("Circuit breaker fallback triggered for LLM call. Cause: {}", e.getMessage(), e);
+        return "Fallback!";
+    }
 
     public ChatSession getSessionHistory(String sessionId) {
         return chatSessionStorage.getOrCreateChatSession(sessionId);

src/main/resources/application.properties

@@ -1,4 +1,19 @@
 spring.application.name=ProjectBifrost
 spring.mvc.problemdetails.enabled=true
 openrouter.api.key=${OPENROUTER_API_KEY}
+openrouter.base-url=https://openrouter.ai/api/v1
 openrouter.model=poolside/laguna-xs.2:free
+
+resilience4j.circuitbreaker.instances.chatService.sliding-window-size=10
+resilience4j.circuitbreaker.instances.chatService.minimum-number-of-calls=5
+resilience4j.circuitbreaker.instances.chatService.failure-rate-threshold=50
+
+resilience4j.circuitbreaker.instances.chatService.wait-duration-in-open-state=5s
+resilience4j.retry.instances.chatService.max-attempts=3
+resilience4j.retry.instances.chatService.retry-exceptions=org.example.projectbifrost.exception.RetryableHttpException
+resilience4j.retry.instances.chatService.wait-duration=500ms
+resilience4j.retry.instances.chatService.enable-exponential-backoff=true
+resilience4j.retry.instances.chatService.exponential-backoff-multiplier=2
+resilience4j.circuitbreaker.circuitBreakerAspectOrder=1
+resilience4j.retry.retryAspectOrder=2
+logging.level.io.github.resilience4j=DEBUG