diff --git a/core/src/main/java/com/opencontext/common/CommonResponse.java b/core/src/main/java/com/opencontext/common/CommonResponse.java index 61de039..ce57a1f 100644 --- a/core/src/main/java/com/opencontext/common/CommonResponse.java +++ b/core/src/main/java/com/opencontext/common/CommonResponse.java @@ -82,4 +82,17 @@ public static CommonResponse error(String message, String errorCode) { LocalDateTime.now() ); } + + /** + * Static factory method for creating error responses from ErrorCode enum. + */ + public static CommonResponse error(com.opencontext.enums.ErrorCode errorCode, String message) { + return new CommonResponse<>( + false, + null, + message, + errorCode.getCode(), + LocalDateTime.now() + ); + } } diff --git a/core/src/main/java/com/opencontext/config/ApiKeyAuthenticationFilter.java b/core/src/main/java/com/opencontext/config/ApiKeyAuthenticationFilter.java new file mode 100644 index 0000000..d71f07d --- /dev/null +++ b/core/src/main/java/com/opencontext/config/ApiKeyAuthenticationFilter.java @@ -0,0 +1,92 @@ +package com.opencontext.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencontext.common.CommonResponse; +import com.opencontext.enums.ErrorCode; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * Filter for API Key authentication on admin endpoints. + * + * This filter validates the X-API-KEY header for endpoints that require + * admin access, specifically the document management APIs. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ApiKeyAuthenticationFilter extends OncePerRequestFilter { + + private final ObjectMapper objectMapper; + + @Value("${opencontext.api.key:dev-api-key-123}") + private String validApiKey; + + // Endpoints that require API Key authentication + private static final List PROTECTED_ENDPOINTS = Arrays.asList( + "/api/v1/sources" + ); + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain) throws ServletException, IOException { + + String requestPath = request.getRequestURI(); + + // Check if this endpoint requires API Key authentication + boolean requiresAuth = PROTECTED_ENDPOINTS.stream() + .anyMatch(requestPath::startsWith); + + if (requiresAuth) { + String apiKey = request.getHeader("X-API-KEY"); + + if (!StringUtils.hasText(apiKey)) { + log.warn("API Key missing for protected endpoint: {}", requestPath); + sendErrorResponse(response, ErrorCode.INSUFFICIENT_PERMISSION, + "API Key is required. Please provide X-API-KEY header."); + return; + } + + if (!validApiKey.equals(apiKey)) { + log.warn("Invalid API Key provided for endpoint: {}", requestPath); + sendErrorResponse(response, ErrorCode.INSUFFICIENT_PERMISSION, + "Invalid API Key provided."); + return; + } + + log.debug("API Key authentication successful for endpoint: {}", requestPath); + } + + filterChain.doFilter(request, response); + } + + /** + * Sends an error response in JSON format. + */ + private void sendErrorResponse(HttpServletResponse response, ErrorCode errorCode, String message) + throws IOException { + + response.setStatus(errorCode.getHttpStatus().value()); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + + CommonResponse errorResponse = CommonResponse.error(errorCode, message); + + String jsonResponse = objectMapper.writeValueAsString(errorResponse); + response.getWriter().write(jsonResponse); + } +} diff --git a/core/src/main/java/com/opencontext/config/MinIOConfig.java b/core/src/main/java/com/opencontext/config/MinIOConfig.java new file mode 100644 index 0000000..c6592b6 --- /dev/null +++ b/core/src/main/java/com/opencontext/config/MinIOConfig.java @@ -0,0 +1,41 @@ +package com.opencontext.config; + +import io.minio.MinioClient; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * MinIO client configuration for object storage operations. + * + * This configuration creates a MinIO client bean that can be used throughout + * the application for file upload, download, and management operations. + */ +@Slf4j +@Data +@Configuration +@ConfigurationProperties(prefix = "minio") +public class MinIOConfig { + + private String endpoint; + private String accessKey; + private String secretKey; + private String bucketName; + + /** + * Creates and configures the MinIO client bean. + * + * @return configured MinioClient instance + */ + @Bean + public MinioClient minioClient() { + log.info("Initializing MinIO client with endpoint: {}", endpoint); + + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .build(); + } +} diff --git a/core/src/main/java/com/opencontext/config/SecurityConfig.java b/core/src/main/java/com/opencontext/config/SecurityConfig.java index 9219300..1e21882 100644 --- a/core/src/main/java/com/opencontext/config/SecurityConfig.java +++ b/core/src/main/java/com/opencontext/config/SecurityConfig.java @@ -25,6 +25,7 @@ public class SecurityConfig { private final Environment environment; + private final ApiKeyAuthenticationFilter apiKeyAuthenticationFilter; @Bean public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { @@ -62,15 +63,15 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { // Allow development mock data endpoints (development only) .requestMatchers("/api/v1/dev/**").permitAll() - // Admin APIs will require API Key authentication (TODO: implement in next step) - .requestMatchers("/api/v1/sources/**").permitAll() // Temporary - will add API key auth + // Admin APIs require API Key authentication + .requestMatchers("/api/v1/sources/**").permitAll() // Authentication handled by filter // All other requests require authentication .anyRequest().authenticated() ); - // TODO: Add API Key authentication filter for Admin APIs - // http.addFilterBefore(apiKeyAuthenticationFilter, UsernamePasswordAuthenticationFilter.class); + // Add API Key authentication filter for Admin APIs + http.addFilterBefore(apiKeyAuthenticationFilter, UsernamePasswordAuthenticationFilter.class); log.info("Security configuration completed successfully"); return http.build(); diff --git a/core/src/main/java/com/opencontext/controller/SourceController/DocsSourceController.java b/core/src/main/java/com/opencontext/controller/SourceController/DocsSourceController.java new file mode 100644 index 0000000..0cacb81 --- /dev/null +++ b/core/src/main/java/com/opencontext/controller/SourceController/DocsSourceController.java @@ -0,0 +1,317 @@ +package com.opencontext.controller.SourceController; + +import com.opencontext.common.CommonResponse; +import com.opencontext.common.PageResponse; +import com.opencontext.dto.SourceDocumentDto; +import com.opencontext.dto.SourceUploadResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.constraints.NotNull; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.util.UUID; + +/** + * Interface defining source document management API endpoints with comprehensive Swagger documentation. + * + * This interface serves as the contract for document ingestion pipeline management APIs + * with detailed API documentation including examples and response schemas. + */ +@Tag( + name = "Source Document Management", + description = "Admin APIs for document ingestion pipeline management. Requires X-API-KEY authentication." +) +@SecurityRequirement(name = "X-API-KEY") +public interface DocsSourceController { + + /** + * Uploads a file and starts the ingestion pipeline. + */ + @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Upload file and start ingestion pipeline", + description = """ + Uploads a PDF or Markdown file to the system and starts the asynchronous ingestion pipeline. + + **Authentication Required:** X-API-KEY header must be provided. + + **Supported File Types:** + - PDF documents (application/pdf) + - Markdown files (text/markdown) + - Plain text files (text/plain) + + **File Size Limit:** 100MB + + The API accepts the file upload request immediately and returns 202 Accepted status, + indicating that the actual processing will be performed in the background. + """, + requestBody = @io.swagger.v3.oas.annotations.parameters.RequestBody( + description = "Multipart form data containing the file to upload", + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA_VALUE, + examples = @ExampleObject( + name = "File Upload", + description = "Upload a PDF document", + value = "file: [binary PDF content]" + ) + ) + ) + ) + @ApiResponses(value = { + @ApiResponse( + responseCode = "202", + description = "File uploaded successfully and queued for ingestion", + content = @Content( + mediaType = MediaType.APPLICATION_JSON_VALUE, + schema = @Schema(implementation = SourceUploadResponse.class), + examples = @ExampleObject( + name = "Upload Success", + value = """ + { + "success": true, + "data": { + "sourceDocumentId": "a1b2c3d4-e5f6-7890-1234-567890abcdef", + "originalFilename": "document.pdf", + "ingestionStatus": "PENDING", + "message": "File uploaded successfully and is now pending for ingestion." + }, + "message": "요청이 성공적으로 처리되었습니다.", + "errorCode": null, + "timestamp": "2025-08-07T11:50:00Z" + } + """ + ) + ) + ), + @ApiResponse( + responseCode = "400", + description = "Bad Request - file part missing or validation failed", + content = @Content( + examples = @ExampleObject( + name = "Validation Failed", + value = """ + { + "success": false, + "data": null, + "message": "File cannot be empty", + "errorCode": "COMMON_002", + "timestamp": "2025-08-07T11:50:00Z" + } + """ + ) + ) + ), + @ApiResponse( + responseCode = "403", + description = "Forbidden - invalid or missing API Key", + content = @Content( + examples = @ExampleObject( + name = "Invalid API Key", + value = """ + { + "success": false, + "data": null, + "message": "Invalid API Key provided.", + "errorCode": "AUTH_001", + "timestamp": "2025-08-07T11:50:00Z" + } + """ + ) + ) + ), + @ApiResponse( + responseCode = "409", + description = "Conflict - duplicate file already exists", + content = @Content( + examples = @ExampleObject( + name = "Duplicate File", + value = """ + { + "success": false, + "data": null, + "message": "A file with identical content already exists.", + "errorCode": "DOC_002", + "timestamp": "2025-08-07T11:50:00Z" + } + """ + ) + ) + ), + @ApiResponse(responseCode = "413", description = "Payload Too Large - file exceeds 100MB limit"), + @ApiResponse(responseCode = "415", description = "Unsupported Media Type - invalid file format") + }) + ResponseEntity> uploadFile( + @Parameter(description = "File to upload (PDF, Markdown, or plain text)", required = true) + @RequestParam("file") @NotNull MultipartFile file + ); + + /** + * Retrieves all uploaded documents with their current status. + */ + @GetMapping + @Operation( + summary = "Get all source documents", + description = """ + Retrieves a paginated list of all source documents in the system with their current ingestion status. + + **Authentication Required:** X-API-KEY header must be provided. + + This endpoint is primarily used by the Admin UI dashboard to periodically refresh and display + the processing status of uploaded documents. + """ + ) + @ApiResponses(value = { + @ApiResponse( + responseCode = "200", + description = "Source documents retrieved successfully", + content = @Content( + mediaType = MediaType.APPLICATION_JSON_VALUE, + examples = @ExampleObject( + name = "Document List", + value = """ + { + "success": true, + "data": { + "content": [ + { + "id": "a1b2c3d4-e5f6-7890-1234-567890abcdef", + "originalFilename": "spring-security-reference.pdf", + "fileType": "PDF", + "fileSize": 10485760, + "ingestionStatus": "COMPLETED", + "errorMessage": null, + "lastIngestedAt": "2025-08-07T12:00:00Z", + "createdAt": "2025-08-07T11:50:00Z", + "updatedAt": "2025-08-07T12:00:00Z" + } + ], + "page": 0, + "size": 10, + "totalElements": 48, + "totalPages": 5, + "first": true, + "last": false, + "hasNext": true, + "hasPrevious": false + }, + "message": "요청이 성공적으로 처리되었습니다.", + "timestamp": "2025-08-07T12:05:00Z" + } + """ + ) + ) + ), + @ApiResponse(responseCode = "403", description = "Forbidden - invalid or missing API Key") + }) + ResponseEntity>> getAllSourceDocuments( + @Parameter(description = "Page number (0-based)", example = "0") + @RequestParam(defaultValue = "0") int page, + + @Parameter(description = "Page size", example = "20") + @RequestParam(defaultValue = "20") int size, + + @Parameter(description = "Sort specification", example = "createdAt,desc") + @RequestParam(defaultValue = "createdAt,desc") String sort + ); + + /** + * Triggers re-ingestion of a specific document. + */ + @PostMapping("/{sourceId}/resync") + @Operation( + summary = "Re-ingest a source document", + description = """ + Forces re-ingestion of a specific source document by restarting the ingestion pipeline. + + **Authentication Required:** X-API-KEY header must be provided. + + The actual re-processing is performed asynchronously in the background, + so this endpoint returns 202 Accepted to indicate the request has been queued. + """ + ) + @ApiResponses(value = { + @ApiResponse( + responseCode = "202", + description = "Re-ingestion request accepted and queued", + content = @Content( + examples = @ExampleObject( + name = "Resync Accepted", + value = """ + { + "success": true, + "data": "Document re-ingestion has been queued successfully.", + "message": "요청이 성공적으로 처리되었습니다.", + "timestamp": "2025-08-07T12:05:00Z" + } + """ + ) + ) + ), + @ApiResponse(responseCode = "403", description = "Forbidden - invalid or missing API Key"), + @ApiResponse(responseCode = "404", description = "Not Found - document does not exist"), + @ApiResponse(responseCode = "409", description = "Conflict - document is currently being processed") + }) + ResponseEntity> resyncSourceDocument( + @Parameter(description = "Source document ID", required = true) + @PathVariable UUID sourceId + ); + + /** + * Deletes a source document and all associated data. + */ + @DeleteMapping("/{sourceId}") + @Operation( + summary = "Delete a source document", + description = """ + Permanently deletes a source document and all its associated data from the system. + + **Authentication Required:** X-API-KEY header must be provided. + + This operation removes: + - The original file from MinIO storage + - All generated chunks from PostgreSQL + - All embeddings and indices from Elasticsearch + - The source document metadata + + The actual deletion is performed asynchronously in the background, + so this endpoint returns 202 Accepted to indicate the request has been queued. + """ + ) + @ApiResponses(value = { + @ApiResponse( + responseCode = "202", + description = "Deletion request accepted and queued", + content = @Content( + examples = @ExampleObject( + name = "Deletion Accepted", + value = """ + { + "success": true, + "data": "Document deletion has been queued successfully.", + "message": "요청이 성공적으로 처리되었습니다.", + "timestamp": "2025-08-07T12:05:00Z" + } + """ + ) + ) + ), + @ApiResponse(responseCode = "403", description = "Forbidden - invalid or missing API Key"), + @ApiResponse(responseCode = "404", description = "Not Found - document does not exist"), + @ApiResponse(responseCode = "409", description = "Conflict - document is currently being processed") + }) + ResponseEntity> deleteSourceDocument( + @Parameter(description = "Source document ID", required = true) + @PathVariable UUID sourceId + ); +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/controller/SourceController/SourceController.java b/core/src/main/java/com/opencontext/controller/SourceController/SourceController.java new file mode 100644 index 0000000..4746cf0 --- /dev/null +++ b/core/src/main/java/com/opencontext/controller/SourceController/SourceController.java @@ -0,0 +1,96 @@ +package com.opencontext.controller.SourceController; + +import com.opencontext.common.CommonResponse; +import com.opencontext.common.PageResponse; +import com.opencontext.dto.SourceDocumentDto; +import com.opencontext.dto.SourceUploadResponse; +import com.opencontext.service.SourceDocumentService; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import java.util.UUID; + +/** + * REST controller implementing source document management APIs. + * + * This controller implements the DocsSourceController interface and provides + * clean, Lombok-optimized code for document ingestion pipeline management. + */ +@Slf4j +@RestController +@RequestMapping("/api/v1/sources") +@RequiredArgsConstructor +public class SourceController implements DocsSourceController { + + private final SourceDocumentService sourceDocumentService; + + @Override + public ResponseEntity> uploadFile(@NotNull MultipartFile file) { + log.info("Source document upload request: filename={}, size={}, contentType={}", + file.getOriginalFilename(), file.getSize(), file.getContentType()); + + SourceUploadResponse response = sourceDocumentService.uploadSourceDocument(file); + + log.info("Source document upload accepted: id={}, filename={}", + response.getSourceDocumentId(), response.getOriginalFilename()); + + return ResponseEntity.status(HttpStatus.ACCEPTED) + .body(CommonResponse.success(response, "요청이 성공적으로 처리되었습니다.")); + } + + @Override + public ResponseEntity>> getAllSourceDocuments( + int page, int size, String sort) { + + log.debug("Source documents list request: page={}, size={}, sort={}", page, size, sort); + + // Parse sort parameter + String[] sortParts = sort.split(","); + String sortProperty = sortParts[0]; + Sort.Direction direction = sortParts.length > 1 && "desc".equals(sortParts[1]) + ? Sort.Direction.DESC : Sort.Direction.ASC; + + Pageable pageable = PageRequest.of(page, size, Sort.by(direction, sortProperty)); + + Page documents = sourceDocumentService.getAllSourceDocuments(pageable); + PageResponse pageResponse = PageResponse.from(documents); + + return ResponseEntity.ok(CommonResponse.success(pageResponse, "요청이 성공적으로 처리되었습니다.")); + } + + @Override + public ResponseEntity> resyncSourceDocument(UUID sourceId) { + log.info("Source document resync request: id={}", sourceId); + + sourceDocumentService.resyncSourceDocument(sourceId); + + return ResponseEntity.status(HttpStatus.ACCEPTED) + .body(CommonResponse.success( + "Document re-ingestion has been queued successfully.", + "요청이 성공적으로 처리되었습니다." + )); + } + + @Override + public ResponseEntity> deleteSourceDocument(UUID sourceId) { + log.info("Source document deletion request: id={}", sourceId); + + sourceDocumentService.deleteSourceDocument(sourceId); + + return ResponseEntity.status(HttpStatus.ACCEPTED) + .body(CommonResponse.success( + "Document deletion has been queued successfully.", + "요청이 성공적으로 처리되었습니다." + )); + } +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/dto/SourceDocumentDto.java b/core/src/main/java/com/opencontext/dto/SourceDocumentDto.java new file mode 100644 index 0000000..8d2d338 --- /dev/null +++ b/core/src/main/java/com/opencontext/dto/SourceDocumentDto.java @@ -0,0 +1,69 @@ +package com.opencontext.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +/** + * DTO for source document information returned in list operations. + * + * Contains essential information about source documents for admin UI display. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SourceDocumentDto { + + /** + * Unique identifier of the source document. + */ + private String id; + + /** + * Original filename when uploaded. + */ + private String originalFilename; + + /** + * File type (PDF, MARKDOWN, etc.). + */ + private String fileType; + + /** + * File size in bytes. + */ + private Long fileSize; + + /** + * Current ingestion status. + */ + private String ingestionStatus; + + /** + * Error message if ingestion failed. + */ + private String errorMessage; + + /** + * Timestamp when document was last successfully ingested. + */ + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + private LocalDateTime lastIngestedAt; + + /** + * Timestamp when document was created. + */ + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + private LocalDateTime createdAt; + + /** + * Timestamp when document was last updated. + */ + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + private LocalDateTime updatedAt; +} diff --git a/core/src/main/java/com/opencontext/dto/SourceUploadResponse.java b/core/src/main/java/com/opencontext/dto/SourceUploadResponse.java new file mode 100644 index 0000000..8f4cd69 --- /dev/null +++ b/core/src/main/java/com/opencontext/dto/SourceUploadResponse.java @@ -0,0 +1,61 @@ +package com.opencontext.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +/** + * Response DTO for source document upload operations. + * + * This response is returned when a document is successfully uploaded + * and queued for ingestion processing, following PRD specifications. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SourceUploadResponse { + + /** + * UUID of the created SourceDocument entity. + */ + private String sourceDocumentId; + + /** + * Original filename as provided by the client. + */ + private String originalFilename; + + /** + * Current ingestion status (always PENDING for new uploads). + */ + private String ingestionStatus; + + /** + * Message indicating the upload result. + */ + private String message; + + /** + * Timestamp when the upload was processed. + */ + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + private LocalDateTime timestamp; + + /** + * Creates a successful upload response. + */ + public static SourceUploadResponse success(String sourceDocumentId, String originalFilename) { + return SourceUploadResponse.builder() + .sourceDocumentId(sourceDocumentId) + .originalFilename(originalFilename) + .ingestionStatus("PENDING") + .message("File uploaded successfully and is now pending for ingestion.") + .timestamp(LocalDateTime.now()) + .build(); + } +} diff --git a/core/src/main/java/com/opencontext/enums/ErrorCode.java b/core/src/main/java/com/opencontext/enums/ErrorCode.java index cc63386..c76cb77 100644 --- a/core/src/main/java/com/opencontext/enums/ErrorCode.java +++ b/core/src/main/java/com/opencontext/enums/ErrorCode.java @@ -32,6 +32,13 @@ public enum ErrorCode { TOKEN_LIMIT_EXCEEDED(HttpStatus.UNPROCESSABLE_ENTITY, "CTX_001", "Requested content token count exceeds maximum limit."), CHUNK_NOT_FOUND(HttpStatus.NOT_FOUND, "CTX_002", "Chunk with the specified ID not found."), + // --- FILE STORAGE --- + FILE_UPLOAD_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "FILE_001", "Failed to upload file to storage."), + FILE_NOT_FOUND(HttpStatus.NOT_FOUND, "FILE_002", "Requested file not found in storage."), + FILE_DELETE_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "FILE_003", "Failed to delete file from storage."), + STORAGE_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "FILE_004", "Storage system error occurred."), + FILE_ACCESS_DENIED(HttpStatus.FORBIDDEN, "FILE_005", "Access denied to the requested file."), + // --- INFRASTRUCTURE/EXTERNAL SERVICES --- INGESTION_PIPELINE_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "INFRA_001", "Internal error occurred during document processing."), EXTERNAL_SERVICE_UNAVAILABLE(HttpStatus.SERVICE_UNAVAILABLE, "INFRA_002", "External service is not responding."), diff --git a/core/src/main/java/com/opencontext/service/FileStorageService.java b/core/src/main/java/com/opencontext/service/FileStorageService.java new file mode 100644 index 0000000..d73573d --- /dev/null +++ b/core/src/main/java/com/opencontext/service/FileStorageService.java @@ -0,0 +1,151 @@ +package com.opencontext.service; + +import com.opencontext.config.MinIOConfig; +import com.opencontext.enums.ErrorCode; +import com.opencontext.exception.BusinessException; +import io.minio.*; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.time.LocalDateTime; +import java.util.UUID; + +/** + * Service for handling basic file storage operations with MinIO. + * + * This service provides essential methods for storing and managing + * files in MinIO object storage for the document ingestion pipeline. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FileStorageService { + + private final MinioClient minioClient; + private final MinIOConfig minioConfig; + + /** + * Uploads a file to MinIO storage and returns the object key. + * + * @param file the multipart file to upload + * @return the object key where the file was stored + */ + public String uploadFile(MultipartFile file) { + try { + // Ensure bucket exists + ensureBucketExists(); + + // Generate unique object key + String objectKey = generateObjectKey(file.getOriginalFilename()); + + // Upload file to MinIO + PutObjectArgs putObjectArgs = PutObjectArgs.builder() + .bucket(minioConfig.getBucketName()) + .object(objectKey) + .stream(file.getInputStream(), file.getSize(), -1) + .contentType(file.getContentType()) + .build(); + + minioClient.putObject(putObjectArgs); + + log.info("Successfully uploaded file: {} to bucket: {} with key: {}", + file.getOriginalFilename(), minioConfig.getBucketName(), objectKey); + + return objectKey; + + } catch (Exception e) { + log.error("Failed to upload file: {}", file.getOriginalFilename(), e); + throw new BusinessException(ErrorCode.FILE_UPLOAD_FAILED, + "Failed to upload file to storage: " + e.getMessage()); + } + } + + /** + * Deletes a file from MinIO storage. + * + * @param objectKey the object key of the file to delete + */ + public void deleteFile(String objectKey) { + try { + RemoveObjectArgs removeObjectArgs = RemoveObjectArgs.builder() + .bucket(minioConfig.getBucketName()) + .object(objectKey) + .build(); + + minioClient.removeObject(removeObjectArgs); + log.info("Successfully deleted file with key: {}", objectKey); + + } catch (Exception e) { + log.error("Failed to delete file with key: {}", objectKey, e); + throw new BusinessException(ErrorCode.FILE_DELETE_FAILED, + "Failed to delete file: " + e.getMessage()); + } + } + + /** + * Checks if a file exists in MinIO storage. + * + * @param objectKey the object key to check + * @return true if file exists, false otherwise + */ + public boolean fileExists(String objectKey) { + try { + StatObjectArgs statObjectArgs = StatObjectArgs.builder() + .bucket(minioConfig.getBucketName()) + .object(objectKey) + .build(); + + minioClient.statObject(statObjectArgs); + return true; + + } catch (Exception e) { + return false; + } + } + + /** + * Ensures that the configured bucket exists, creating it if necessary. + */ + private void ensureBucketExists() { + try { + BucketExistsArgs bucketExistsArgs = BucketExistsArgs.builder() + .bucket(minioConfig.getBucketName()) + .build(); + + boolean bucketExists = minioClient.bucketExists(bucketExistsArgs); + + if (!bucketExists) { + MakeBucketArgs makeBucketArgs = MakeBucketArgs.builder() + .bucket(minioConfig.getBucketName()) + .build(); + + minioClient.makeBucket(makeBucketArgs); + log.info("Created MinIO bucket: {}", minioConfig.getBucketName()); + } + + } catch (Exception e) { + log.error("Failed to ensure bucket exists: {}", minioConfig.getBucketName(), e); + throw new BusinessException(ErrorCode.STORAGE_ERROR, + "Failed to ensure bucket exists: " + e.getMessage()); + } + } + + /** + * Generates an object key for MinIO storage. + * + * @param originalFilename the original filename + * @return generated object key + */ + private String generateObjectKey(String originalFilename) { + LocalDateTime now = LocalDateTime.now(); + String uuid = UUID.randomUUID().toString().substring(0, 8); + String timestamp = String.valueOf(System.currentTimeMillis()); + + String filename = String.format("%s_%s_%s", timestamp, uuid, originalFilename); + + return String.format("documents/%d/%02d/%02d/%s", + now.getYear(), now.getMonthValue(), now.getDayOfMonth(), filename); + } +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/service/SourceDocumentService.java b/core/src/main/java/com/opencontext/service/SourceDocumentService.java new file mode 100644 index 0000000..f0b019f --- /dev/null +++ b/core/src/main/java/com/opencontext/service/SourceDocumentService.java @@ -0,0 +1,328 @@ +package com.opencontext.service; + +import com.opencontext.dto.SourceDocumentDto; +import com.opencontext.dto.SourceUploadResponse; +import com.opencontext.entity.SourceDocument; +import com.opencontext.enums.ErrorCode; +import com.opencontext.enums.IngestionStatus; +import com.opencontext.exception.BusinessException; +import com.opencontext.repository.SourceDocumentRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +/** + * Service for managing source document ingestion and lifecycle. + * + * This service handles the complete document ingestion pipeline from upload + * to indexing, integrating with MinIO storage and the async processing pipeline. + */ +@Slf4j +@Service +@RequiredArgsConstructor +@Transactional +public class SourceDocumentService { + + private final SourceDocumentRepository sourceDocumentRepository; + private final FileStorageService fileStorageService; + + // Supported file types for document processing + private static final List SUPPORTED_CONTENT_TYPES = List.of( + "application/pdf", + "text/markdown", + "text/plain" + ); + + /** + * Uploads a source document and starts the ingestion pipeline. + * + * @param file the multipart file to upload + * @return SourceUploadResponse containing the created document information + */ + public SourceUploadResponse uploadSourceDocument(MultipartFile file) { + log.info("Starting source document upload: filename={}, size={}, contentType={}", + file.getOriginalFilename(), file.getSize(), file.getContentType()); + + // Validate file + validateFile(file); + + // Calculate file checksum to prevent duplicates + String fileChecksum = calculateFileChecksum(file); + + // Check for duplicate files + if (sourceDocumentRepository.existsByFileChecksum(fileChecksum)) { + log.warn("Duplicate file upload attempt: checksum={}", fileChecksum); + throw new BusinessException(ErrorCode.DUPLICATE_FILE_UPLOADED, + "A file with identical content already exists."); + } + + try { + // Upload file to MinIO + String objectKey = fileStorageService.uploadFile(file); + + // Create SourceDocument entity + SourceDocument sourceDocument = SourceDocument.builder() + .originalFilename(file.getOriginalFilename()) + .fileStoragePath(objectKey) + .fileType(determineFileType(file.getContentType())) + .fileSize(file.getSize()) + .fileChecksum(fileChecksum) + .ingestionStatus(IngestionStatus.PENDING) + .build(); + + // Save to database + SourceDocument savedDocument = sourceDocumentRepository.save(sourceDocument); + + log.info("Source document created successfully: id={}, filename={}", + savedDocument.getId(), savedDocument.getOriginalFilename()); + + // Start async ingestion pipeline + startIngestionPipeline(savedDocument.getId()); + + // Return response + return SourceUploadResponse.success( + savedDocument.getId().toString(), + savedDocument.getOriginalFilename() + ); + + } catch (Exception e) { + log.error("Failed to upload source document: {}", file.getOriginalFilename(), e); + if (e instanceof BusinessException) { + throw e; + } + throw new BusinessException(ErrorCode.FILE_UPLOAD_FAILED, + "Unexpected error during file upload: " + e.getMessage()); + } + } + + /** + * Retrieves all source documents with pagination. + * + * @param pageable pagination parameters + * @return Page of SourceDocumentDto + */ + @Transactional(readOnly = true) + public Page getAllSourceDocuments(Pageable pageable) { + log.debug("Retrieving source documents with pagination: page={}, size={}", + pageable.getPageNumber(), pageable.getPageSize()); + + return sourceDocumentRepository.findAllByOrderByCreatedAtDesc(pageable) + .map(this::convertToDto); + } + + /** + * Retrieves a specific source document by ID. + * + * @param documentId the document ID + * @return SourceDocumentDto + */ + @Transactional(readOnly = true) + public SourceDocumentDto getSourceDocument(UUID documentId) { + log.debug("Retrieving source document: id={}", documentId); + + SourceDocument document = sourceDocumentRepository.findById(documentId) + .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, + "Document not found: " + documentId)); + + return convertToDto(document); + } + + /** + * Triggers re-ingestion of a source document. + * + * @param documentId the document ID to re-ingest + */ + public void resyncSourceDocument(UUID documentId) { + log.info("Starting source document resync: id={}", documentId); + + SourceDocument document = sourceDocumentRepository.findById(documentId) + .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, + "Document not found: " + documentId)); + + // Check if document is currently being processed + if (document.isProcessing()) { + throw new BusinessException(ErrorCode.RESOURCE_IS_BEING_PROCESSED, + "Document is currently being processed and cannot be resynced."); + } + + // Reset status to PENDING and clear error message + document.updateIngestionStatus(IngestionStatus.PENDING); + sourceDocumentRepository.save(document); + + // Start async ingestion pipeline + startIngestionPipeline(documentId); + + log.info("Source document resync initiated: id={}", documentId); + } + + /** + * Deletes a source document and all associated data. + * + * @param documentId the document ID to delete + */ + public void deleteSourceDocument(UUID documentId) { + log.info("Starting source document deletion: id={}", documentId); + + SourceDocument document = sourceDocumentRepository.findById(documentId) + .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, + "Document not found: " + documentId)); + + // Check if document is currently being processed + if (document.isProcessing()) { + throw new BusinessException(ErrorCode.RESOURCE_IS_BEING_PROCESSED, + "Document is currently being processed and cannot be deleted."); + } + + try { + // Mark document as being deleted + document.updateIngestionStatus(IngestionStatus.DELETING); + sourceDocumentRepository.save(document); + + // Start async deletion process + startDeletionPipeline(documentId); + + log.info("Source document deletion initiated: id={}", documentId); + + } catch (Exception e) { + log.error("Failed to initiate document deletion: id={}", documentId, e); + throw new BusinessException(ErrorCode.FILE_DELETE_FAILED, + "Failed to initiate document deletion: " + e.getMessage()); + } + } + + /** + * Starts the async ingestion pipeline for a document. + * + * @param documentId the document ID to process + */ + @Async + public void startIngestionPipeline(UUID documentId) { + log.info("Starting ingestion pipeline for document: id={}", documentId); + + // TODO: Implement actual ingestion pipeline + // This would typically involve: + // 1. Update status to PARSING + // 2. Parse document using Unstructured API + // 3. Update status to CHUNKING + // 4. Split into chunks + // 5. Update status to EMBEDDING + // 6. Generate embeddings + // 7. Update status to INDEXING + // 8. Store in Elasticsearch + // 9. Update status to COMPLETED + + // For now, just log the pipeline start + log.info("Ingestion pipeline queued for document: id={}", documentId); + } + + /** + * Starts the async deletion pipeline for a document. + * + * @param documentId the document ID to delete + */ + @Async + public void startDeletionPipeline(UUID documentId) { + log.info("Starting deletion pipeline for document: id={}", documentId); + + // TODO: Implement actual deletion pipeline + // This would typically involve: + // 1. Delete from Elasticsearch + // 2. Delete chunks from PostgreSQL + // 3. Delete file from MinIO + // 4. Delete SourceDocument record + + // For now, just log the pipeline start + log.info("Deletion pipeline queued for document: id={}", documentId); + } + + /** + * Validates the uploaded file. + */ + private void validateFile(MultipartFile file) { + if (file.isEmpty()) { + throw new BusinessException(ErrorCode.INVALID_REQUEST, "File cannot be empty"); + } + + if (file.getSize() > 100 * 1024 * 1024) { // 100MB + throw new BusinessException(ErrorCode.PAYLOAD_TOO_LARGE, + "File size exceeds maximum limit of 100MB"); + } + + String contentType = file.getContentType(); + if (contentType == null || !SUPPORTED_CONTENT_TYPES.contains(contentType)) { + throw new BusinessException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, + "Unsupported file type. Supported types: PDF, Markdown, and plain text files."); + } + + String filename = file.getOriginalFilename(); + if (filename == null || filename.trim().isEmpty()) { + throw new BusinessException(ErrorCode.INVALID_REQUEST, "Filename cannot be empty"); + } + + log.debug("File validation passed: filename={}", filename); + } + + /** + * Calculates SHA-256 checksum of the file content. + */ + private String calculateFileChecksum(MultipartFile file) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] fileBytes = file.getBytes(); + byte[] hashBytes = digest.digest(fileBytes); + + StringBuilder sb = new StringBuilder(); + for (byte b : hashBytes) { + sb.append(String.format("%02x", b)); + } + + return sb.toString(); + + } catch (NoSuchAlgorithmException | IOException e) { + log.error("Failed to calculate file checksum", e); + throw new BusinessException(ErrorCode.FILE_UPLOAD_FAILED, + "Failed to calculate file checksum: " + e.getMessage()); + } + } + + /** + * Determines file type from content type. + */ + private String determineFileType(String contentType) { + return switch (contentType) { + case "application/pdf" -> "PDF"; + case "text/markdown" -> "MARKDOWN"; + case "text/plain" -> "TEXT"; + default -> "UNKNOWN"; + }; + } + + /** + * Converts SourceDocument entity to DTO. + */ + private SourceDocumentDto convertToDto(SourceDocument document) { + return SourceDocumentDto.builder() + .id(document.getId().toString()) + .originalFilename(document.getOriginalFilename()) + .fileType(document.getFileType()) + .fileSize(document.getFileSize()) + .ingestionStatus(document.getIngestionStatus().name()) + .errorMessage(document.getErrorMessage()) + .lastIngestedAt(document.getLastIngestedAt()) + .createdAt(document.getCreatedAt()) + .updatedAt(document.getUpdatedAt()) + .build(); + } +}