-
Notifications
You must be signed in to change notification settings - Fork 1
fix(docker): Docker 배포 설정 오류 수정 #49
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f6ba43d
9000ea2
08279e2
527c0a9
5688122
a23df40
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -242,8 +242,12 @@ public ResponseEntity<CommonResponse<String>> deleteSourceDocument(@PathVariable | |
| } | ||
| } | ||
|
|
||
| @Value("${app.rag.service.url:http://open-context-rag:8001}") | ||
| private String ragServiceUrl; | ||
|
|
||
| /** | ||
| * Executes the document ingestion pipeline asynchronously. | ||
| * 문서 수집 파이프라인을 비동기적으로 실행합니다. | ||
| * RAG 서비스 (FastAPI)를 호출하여 처리합니다. | ||
| */ | ||
| @Async | ||
| @Transactional | ||
|
|
@@ -253,41 +257,58 @@ public void processIngestionPipeline(UUID documentId) { | |
| try { | ||
| // 1. Update status to PARSING | ||
| fileStorageService.updateDocumentStatus(documentId, IngestionStatus.PARSING); | ||
|
|
||
| // 2. Parse document using Unstructured API | ||
| var parsedElements = documentParsingService.parseDocument(documentId); | ||
| log.info("Document parsing completed: id={}, elements={}", documentId, parsedElements.size()); | ||
|
|
||
| // 3. Update status to CHUNKING | ||
| fileStorageService.updateDocumentStatus(documentId, IngestionStatus.CHUNKING); | ||
|
|
||
| // 4. Split into chunks | ||
| var structuredChunks = chunkingService.createChunks(documentId, parsedElements); | ||
| log.info("Document chunking completed: id={}, chunks={}", documentId, structuredChunks.size()); | ||
| // 2. Get document info | ||
| SourceDocument document = sourceDocumentRepository.findById(documentId) | ||
| .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, | ||
| "Document not found: " + documentId)); | ||
|
|
||
| // 3. Generate presigned MinIO URL (valid for 1 hour) | ||
| String presignedUrl = fileStorageService.generatePresignedUrl(document.getFileStoragePath()); | ||
| log.debug("Generated presigned URL for document: id={}, filename={}", documentId, document.getOriginalFilename()); | ||
|
|
||
| // 4. Call RAG service | ||
| com.opencontext.dto.ProcessDocumentRequest request = com.opencontext.dto.ProcessDocumentRequest.builder() | ||
| .documentId(documentId.toString()) | ||
| .fileUrl(presignedUrl) | ||
| .filename(document.getOriginalFilename()) | ||
| .fileType(document.getFileType()) | ||
| .build(); | ||
|
|
||
| // 5. Update status to EMBEDDING | ||
| fileStorageService.updateDocumentStatus(documentId, IngestionStatus.EMBEDDING); | ||
|
|
||
| // 6. Generate embeddings | ||
| var embeddedChunks = embeddingService.generateEmbeddings(documentId, structuredChunks); | ||
| log.info("Embedding generation completed: id={}, embedded_chunks={}", documentId, embeddedChunks.size()); | ||
| String ragProcessUrl = ragServiceUrl + "/api/v1/process"; | ||
| log.info("Calling RAG service: url={}, documentId={}", ragProcessUrl, documentId); | ||
|
|
||
| // 7. Update status to INDEXING | ||
| fileStorageService.updateDocumentStatus(documentId, IngestionStatus.INDEXING); | ||
|
|
||
| // 8. Store in Elasticsearch and PostgreSQL | ||
| indexingService.indexChunks(documentId, embeddedChunks); | ||
| log.info("Document indexing completed: id={}", documentId); | ||
| HttpHeaders headers = new HttpHeaders(); | ||
| headers.setContentType(MediaType.APPLICATION_JSON); | ||
| HttpEntity<com.opencontext.dto.ProcessDocumentRequest> requestEntity = new HttpEntity<>(request, headers); | ||
|
|
||
| ResponseEntity<com.opencontext.dto.ProcessDocumentResponse> response = restTemplate.postForEntity( | ||
| ragProcessUrl, | ||
| requestEntity, | ||
| com.opencontext.dto.ProcessDocumentResponse.class | ||
| ); | ||
|
Comment on lines
+285
to
+289
|
||
|
|
||
| // 5. Check response | ||
| if (response.getBody() != null && response.getBody().isSuccess()) { | ||
| com.opencontext.dto.ProcessDocumentResponse ragResponse = response.getBody(); | ||
|
|
||
| // Update status to COMPLETED | ||
| fileStorageService.updateDocumentStatusToCompleted(documentId); | ||
|
|
||
| log.info("RAG processing completed: documentId={}, chunks={}, status={}", | ||
| documentId, ragResponse.getChunksProcessed(), ragResponse.getStatus()); | ||
| } else { | ||
| String errorMessage = response.getBody() != null ? response.getBody().getErrorMessage() : "Unknown error"; | ||
| throw new BusinessException(ErrorCode.INGESTION_PIPELINE_FAILED, | ||
| "RAG service processing failed: " + errorMessage); | ||
| } | ||
|
|
||
| // 9. Update status to COMPLETED | ||
| fileStorageService.updateDocumentStatusToCompleted(documentId); | ||
|
|
||
| log.info("Ingestion pipeline completed successfully: documentId={}", documentId); | ||
|
|
||
| } catch (Exception e) { | ||
| log.error("Ingestion pipeline failed: documentId={}", documentId, e); | ||
| fileStorageService.updateDocumentStatusToError(documentId, e.getMessage()); | ||
| throw new BusinessException(ErrorCode.INGESTION_PIPELINE_FAILED, | ||
| throw new BusinessException(ErrorCode.INGESTION_PIPELINE_FAILED, | ||
| "Ingestion pipeline failed: " + e.getMessage()); | ||
| } | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package com.opencontext.dto; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Data; | ||
| import lombok.NoArgsConstructor; | ||
|
|
||
| /** | ||
| * Request DTO for RAG service document processing endpoint. | ||
| * Sent from core/ to rag/ FastAPI service. | ||
| */ | ||
| @Data | ||
| @Builder | ||
| @NoArgsConstructor | ||
| @AllArgsConstructor | ||
| public class ProcessDocumentRequest { | ||
|
|
||
| /** | ||
| * Document UUID | ||
| */ | ||
| private String documentId; | ||
|
|
||
| /** | ||
| * Presigned MinIO URL for downloading the file | ||
| */ | ||
| private String fileUrl; | ||
|
|
||
| /** | ||
| * Original filename | ||
| */ | ||
| private String filename; | ||
|
|
||
| /** | ||
| * File type (PDF, MARKDOWN, TEXT) | ||
| */ | ||
| private String fileType; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package com.opencontext.dto; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Data; | ||
| import lombok.NoArgsConstructor; | ||
|
|
||
| /** | ||
| * Response DTO from RAG service document processing endpoint. | ||
| * Received by core/ from rag/ FastAPI service. | ||
| */ | ||
| @Data | ||
| @Builder | ||
| @NoArgsConstructor | ||
| @AllArgsConstructor | ||
| public class ProcessDocumentResponse { | ||
|
|
||
| /** | ||
| * Whether processing completed successfully | ||
| */ | ||
| private boolean success; | ||
|
|
||
| /** | ||
| * Document UUID | ||
| */ | ||
| private String documentId; | ||
|
|
||
| /** | ||
| * Number of chunks created and indexed | ||
| */ | ||
| private int chunksProcessed; | ||
|
|
||
| /** | ||
| * Processing status (COMPLETED or ERROR) | ||
| */ | ||
| private String status; | ||
|
|
||
| /** | ||
| * Error message if processing failed | ||
| */ | ||
| private String errorMessage; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -678,4 +678,34 @@ private SourceDocumentDto convertToDto(SourceDocument document) { | |
| .updatedAt(document.getUpdatedAt()) | ||
| .build(); | ||
| } | ||
|
|
||
| /** | ||
| * Generates a presigned GET URL for downloading a file from MinIO. | ||
| * The URL is valid for 1 hour. | ||
| * | ||
| * @param objectKey the MinIO object key | ||
| * @return presigned GET URL | ||
| */ | ||
| public String generatePresignedUrl(String objectKey) { | ||
| try { | ||
| GetPresignedObjectUrlArgs args = GetPresignedObjectUrlArgs.builder() | ||
| .method(io.minio.http.Method.GET) | ||
| .bucket(minioConfig.getBucketName()) | ||
| .object(objectKey) | ||
| .expiry(3600) // 1 hour in seconds | ||
|
Comment on lines
+689
to
+695
|
||
| .build(); | ||
|
|
||
| String presignedUrl = minioClient.getPresignedObjectUrl(args); | ||
|
|
||
| log.debug("Generated presigned URL: objectKey={}, expirySeconds=3600", objectKey); | ||
|
|
||
| return presignedUrl; | ||
|
|
||
| } catch (Exception e) { | ||
| log.error("Failed to generate presigned URL: objectKey={}, error={}", | ||
| objectKey, e.getMessage(), e); | ||
| throw new BusinessException(ErrorCode.STORAGE_ERROR, | ||
| "Failed to generate presigned URL: " + e.getMessage()); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
API Key authentication has been disabled in the SecurityConfig for testing purposes. The comment indicates this is temporary, but deploying with authentication disabled is a critical security risk. This should either be removed before merging or controlled via a feature flag/environment variable.