diff --git a/core/src/main/java/com/opencontext/controller/DocsSourceController.java b/core/src/main/java/com/opencontext/controller/DocsSourceController.java index 86abefe..d924d21 100644 --- a/core/src/main/java/com/opencontext/controller/DocsSourceController.java +++ b/core/src/main/java/com/opencontext/controller/DocsSourceController.java @@ -88,7 +88,7 @@ public interface DocsSourceController { "ingestionStatus": "PENDING", "message": "File uploaded successfully and is now pending for ingestion." }, - "message": "요청이 성공적으로 처리되었습니다.", + "message": "Request processed successfully.", "errorCode": null, "timestamp": "2025-08-07T11:50:00Z" } @@ -210,7 +210,7 @@ ResponseEntity> uploadFile( "hasNext": true, "hasPrevious": false }, - "message": "요청이 성공적으로 처리되었습니다.", + "message": "Request processed successfully.", "timestamp": "2025-08-07T12:05:00Z" } """ @@ -256,7 +256,7 @@ ResponseEntity>> getAllSourceDocu { "success": true, "data": "Document re-ingestion has been queued successfully.", - "message": "요청이 성공적으로 처리되었습니다.", + "message": "Request processed successfully.", "timestamp": "2025-08-07T12:05:00Z" } """ @@ -304,7 +304,7 @@ ResponseEntity> resyncSourceDocument( { "success": true, "data": "Document deletion has been queued successfully.", - "message": "요청이 성공적으로 처리되었습니다.", + "message": "Request processed successfully.", "timestamp": "2025-08-07T12:05:00Z" } """ diff --git a/core/src/main/java/com/opencontext/controller/SearchController.java b/core/src/main/java/com/opencontext/controller/SearchController.java index f6c0fb6..d4898b1 100644 --- a/core/src/main/java/com/opencontext/controller/SearchController.java +++ b/core/src/main/java/com/opencontext/controller/SearchController.java @@ -16,8 +16,8 @@ import java.util.List; /** - * MCP 검색 API 컨트롤러 - * find_knowledge와 get_content MCP 도구를 위한 엔드포인트 제공 + * MCP Search API Controller + * Provides endpoints for find_knowledge and get_content MCP tools */ @Slf4j @RestController @@ -29,7 +29,7 @@ public class SearchController implements DocsSearchController { private final ContentRetrievalService contentRetrievalService; /** - * 하이브리드 검색 수행 - find_knowledge MCP 도구 + * Performs hybrid search - find_knowledge MCP tool */ @Override @GetMapping("/search") @@ -37,7 +37,7 @@ public ResponseEntity> search( @RequestParam String query, @RequestParam(defaultValue = "50") Integer topK) { - log.info("검색 요청: query='{}', topK={}", query, topK); + log.info("Search request: query='{}', topK={}", query, topK); if (query == null || query.trim().isEmpty()) { return ResponseEntity.badRequest() @@ -52,7 +52,7 @@ public ResponseEntity> search( } /** - * 청크 콘텐츠 조회 - get_content MCP 도구 + * Retrieves chunk content - get_content MCP tool */ @Override @PostMapping(value = "/get-content", consumes = MediaType.APPLICATION_JSON_VALUE) @@ -67,7 +67,7 @@ public ResponseEntity> getContent(@jakarta.va return ResponseEntity.badRequest() .body(CommonResponse.error("maxTokens must be positive", "VALIDATION_FAILED")); } - log.info("콘텐츠 조회 요청: chunkId={}, maxTokens={}", chunkId, maxTokens); + log.info("Content retrieval request: chunkId={}, maxTokens={}", chunkId, maxTokens); GetContentResponse response = contentRetrievalService.getContent(chunkId, maxTokens); return ResponseEntity.ok(CommonResponse.success(response, "Content retrieved successfully")); } diff --git a/core/src/main/java/com/opencontext/controller/SourceController.java b/core/src/main/java/com/opencontext/controller/SourceController.java index cbc6337..b5d8230 100644 --- a/core/src/main/java/com/opencontext/controller/SourceController.java +++ b/core/src/main/java/com/opencontext/controller/SourceController.java @@ -35,7 +35,7 @@ import java.util.UUID; /** - * 소스 문서 관리를 위한 REST API 컨트롤러. + * REST API controller for managing source documents. * * REST API controller for managing source documents. * @@ -63,10 +63,10 @@ public class SourceController implements DocsSourceController{ private String indexName; /** - * 파일 업로드 및 수집 파이프라인 시작 + * File upload and ingestion pipeline start * - * @param file 업로드할 파일 (multipart/form-data) - * @return 업로드 결과 및 문서 정보 + * @param file File to upload (multipart/form-data) + * @return Upload result and document information */ @Override @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @@ -75,15 +75,15 @@ public ResponseEntity> uploadFile( log.info("File upload requested: filename={}, size={}", file.getOriginalFilename(), file.getSize()); try { - // 파일 저장 및 문서 생성 (FileStorageService에서 검증 수행) + // File storage and document creation (validation performed in FileStorageService) SourceDocument sourceDocument = fileStorageService.uploadFileWithMetadata(file); log.info("File saved successfully: documentId={}, filename={}", sourceDocument.getId(), sourceDocument.getOriginalFilename()); - // 비동기 수집 파이프라인 시작 + // Start asynchronous ingestion pipeline processIngestionPipeline(sourceDocument.getId()); - // 응답 생성 + // Create response SourceUploadResponse response = SourceUploadResponse.success( sourceDocument.getId().toString(), sourceDocument.getOriginalFilename() @@ -99,17 +99,17 @@ public ResponseEntity> uploadFile( } catch (Exception e) { log.error("Unexpected error during file upload", e); return ResponseEntity.internalServerError() - .body(CommonResponse.error("파일 업로드 중 오류가 발생했습니다: " + e.getMessage())); + .body(CommonResponse.error("An error occurred during file upload: " + e.getMessage())); } } /** - * 업로드된 모든 문서의 최신 상태 목록 조회 + * Retrieve latest status list of all uploaded documents * - * @param page 페이지 번호 (0부터 시작) - * @param size 페이지 크기 - * @param sort 정렬 조건 - * @return 페이지네이션된 문서 목록 + * @param page Page number (starting from 0) + * @param size Page size + * @param sort Sorting criteria + * @return Paginated document list */ @Override @GetMapping @@ -120,14 +120,14 @@ public ResponseEntity>> getAllSou log.debug("Getting source documents: page={}, size={}, sort={}", page, size, sort); try { - // 정렬 조건 파싱 + // Parse sorting criteria Sort sortObj = parseSort(sort); Pageable pageable = PageRequest.of(page, size, sortObj); - // 문서 목록 조회 + // Retrieve document list Page documentPage = sourceDocumentRepository.findAll(pageable); - // DTO 변환 + // Convert to DTO List documentDtos = documentPage.getContent().stream() .map(this::convertToDto) .toList(); @@ -149,15 +149,15 @@ public ResponseEntity>> getAllSou } catch (Exception e) { log.error("Failed to get source documents", e); return ResponseEntity.internalServerError() - .body(CommonResponse.error("문서 목록 조회 중 오류가 발생했습니다: " + e.getMessage())); + .body(CommonResponse.error("An error occurred while retrieving document list: " + e.getMessage())); } } /** - * 특정 문서를 강제로 재수집 + * Force re-ingestion of a specific document * - * @param sourceId 재수집할 문서 ID - * @return 재수집 시작 응답 + * @param sourceId Document ID to re-ingest + * @return Re-ingestion start response */ @Override @PostMapping("/{sourceId}/resync") @@ -165,27 +165,27 @@ public ResponseEntity> resyncSourceDocument(@PathVariable log.info("Document resync requested: sourceId={}", sourceId); try { - // 문서 존재 확인 + // Check if document exists SourceDocument sourceDocument = findSourceDocumentById(sourceId); - // 처리 중인 문서인지 확인 + // Check if document is being processed if (sourceDocument.isProcessing()) { throw new BusinessException(ErrorCode.RESOURCE_IS_BEING_PROCESSED, - "문서가 이미 처리 중입니다."); + "Document is already being processed."); } - // 상태를 PENDING으로 초기화 + // Reset status to PENDING sourceDocument.updateIngestionStatus(IngestionStatus.PENDING); sourceDocument.clearErrorMessage(); sourceDocumentRepository.save(sourceDocument); - // 비동기 수집 파이프라인 시작 + // Start asynchronous ingestion pipeline processIngestionPipeline(sourceId); log.info("Document resync started successfully: sourceId={}", sourceId); return ResponseEntity.status(HttpStatus.ACCEPTED) - .body(CommonResponse.success("문서 재수집이 시작되었습니다.")); + .body(CommonResponse.success("Document re-ingestion has started.")); } catch (BusinessException e) { log.error("Document resync failed: sourceId={}, error={}", sourceId, e.getMessage()); @@ -194,15 +194,15 @@ public ResponseEntity> resyncSourceDocument(@PathVariable } catch (Exception e) { log.error("Unexpected error during document resync: sourceId={}", sourceId, e); return ResponseEntity.internalServerError() - .body(CommonResponse.error("문서 재수집 중 오류가 발생했습니다: " + e.getMessage())); + .body(CommonResponse.error("An error occurred during document re-ingestion: " + e.getMessage())); } } /** - * 특정 문서를 시스템에서 영구적으로 삭제 + * Permanently delete a specific document from the system * - * @param sourceId 삭제할 문서 ID - * @return 삭제 시작 응답 + * @param sourceId Document ID to delete + * @return Deletion start response */ @Override @DeleteMapping("/{sourceId}") @@ -210,26 +210,26 @@ public ResponseEntity> deleteSourceDocument(@PathVariable log.info("Document deletion requested: sourceId={}", sourceId); try { - // 문서 존재 확인 + // Check if document exists SourceDocument sourceDocument = findSourceDocumentById(sourceId); - // 처리 중인 문서인지 확인 + // Check if document is being processed if (sourceDocument.isProcessing()) { throw new BusinessException(ErrorCode.RESOURCE_IS_BEING_PROCESSED, - "처리 중인 문서는 삭제할 수 없습니다."); + "Documents being processed cannot be deleted."); } - // 삭제 상태로 변경 + // Change status to DELETING sourceDocument.updateIngestionStatus(IngestionStatus.DELETING); sourceDocumentRepository.save(sourceDocument); - // 비동기 삭제 파이프라인 시작 + // Start asynchronous deletion pipeline processDeletionPipeline(sourceId); log.info("Document deletion started successfully: sourceId={}", sourceId); return ResponseEntity.status(HttpStatus.ACCEPTED) - .body(CommonResponse.success("문서 삭제가 시작되었습니다.")); + .body(CommonResponse.success("Document deletion has started.")); } catch (BusinessException e) { log.error("Document deletion failed: sourceId={}, error={}", sourceId, e.getMessage()); @@ -238,12 +238,12 @@ public ResponseEntity> deleteSourceDocument(@PathVariable } catch (Exception e) { log.error("Unexpected error during document deletion: sourceId={}", sourceId, e); return ResponseEntity.internalServerError() - .body(CommonResponse.error("문서 삭제 중 오류가 발생했습니다: " + e.getMessage())); + .body(CommonResponse.error("An error occurred during document deletion: " + e.getMessage())); } } /** - * 문서 수집 파이프라인을 비동기적으로 실행합니다. + * Executes the document ingestion pipeline asynchronously. */ @Async @Transactional @@ -293,7 +293,7 @@ public void processIngestionPipeline(UUID documentId) { } /** - * 문서 삭제 파이프라인을 비동기적으로 실행합니다. + * Executes the document deletion pipeline asynchronously. */ @Async public void processDeletionPipeline(UUID documentId) { @@ -326,12 +326,12 @@ public void processDeletionPipeline(UUID documentId) { /** - * Elasticsearch에서 문서 관련 청크들을 삭제합니다. + * Deletes document-related chunks from Elasticsearch. */ @Transactional private void deleteFromElasticsearch(UUID documentId) { try { - // 문서 ID로 쿼리하여 관련 청크들을 삭제 + // Query by document ID to delete related chunks String query = String.format(""" { "query": { @@ -363,7 +363,7 @@ private void deleteFromElasticsearch(UUID documentId) { } /** - * PostgreSQL에서 문서 청크들을 삭제합니다. + * Deletes document chunks from PostgreSQL. */ @Transactional private void deleteChunksFromPostgreSQL(UUID documentId) { @@ -378,10 +378,10 @@ private void deleteChunksFromPostgreSQL(UUID documentId) { } } - // ===== 헬퍼 메소드들 ===== + // ===== Helper Methods ===== /** - * 정렬 조건 파싱 + * Parse sorting criteria */ private Sort parseSort(String sortParam) { try { @@ -401,7 +401,7 @@ private Sort parseSort(String sortParam) { } /** - * SourceDocument를 DTO로 변환 + * Convert SourceDocument to DTO */ private SourceDocumentDto convertToDto(SourceDocument document) { return SourceDocumentDto.builder() @@ -418,16 +418,16 @@ private SourceDocumentDto convertToDto(SourceDocument document) { } /** - * ID로 SourceDocument 조회 (없으면 예외 발생) + * Find SourceDocument by ID (throws exception if not found) */ private SourceDocument findSourceDocumentById(UUID sourceId) { return sourceDocumentRepository.findById(sourceId) .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, - "해당 ID의 문서를 찾을 수 없습니다: " + sourceId)); + "Document with the specified ID not found: " + sourceId)); } /** - * ErrorCode에 따른 HTTP 상태 코드 반환 + * Return HTTP status code based on ErrorCode */ private HttpStatus getHttpStatusFromErrorCode(ErrorCode errorCode) { return switch (errorCode) { diff --git a/core/src/main/java/com/opencontext/service/ChunkingService.java b/core/src/main/java/com/opencontext/service/ChunkingService.java index a3d10e3..8044ffa 100644 --- a/core/src/main/java/com/opencontext/service/ChunkingService.java +++ b/core/src/main/java/com/opencontext/service/ChunkingService.java @@ -31,7 +31,7 @@ public List createChunks(UUID documentId, List> currentChunkElements = new ArrayList<>(); @@ -47,29 +47,29 @@ public List createChunks(UUID documentId, List 0 && currentTitle != null) { String chunkContent = currentChunkContent.toString().trim(); - // 청크 분할하지 않고 하나의 청크로 생성 + // Create as one chunk without splitting StructuredChunk chunk = createTitleBasedChunk(documentId, chunkIndex++, currentTitle, chunkContent, currentTitleLevel, currentChunkElements); chunks.add(chunk); - // 생성된 청크 정보 로깅 + // Log created chunk information int tokenCount = estimateTokenCount(chunkContent); log.info("📦 [CHUNK CREATED] Title: '{}', Length: {}, Tokens: ~{}, Level: {}, ChunkId: {}", chunk.getTitle(), chunk.getContent().length(), tokenCount, chunk.getHierarchyLevel(), chunk.getChunkId()); @@ -77,28 +77,28 @@ public List createChunks(UUID documentId, List 200 ? chunkContent.substring(0, 200) + "..." : chunkContent); } - // 새로운 청크 시작 + // Start new chunk currentTitle = isH1Header ? extractMarkdownHeaderText(text) : text; currentChunkContent = new StringBuilder(); currentChunkElements = new ArrayList<>(); - // Title 레벨 결정 (H1이므로 레벨 1) + // Determine Title level (H1 is level 1) currentTitleLevel = 1; - log.info("🏷️ [CHUNKING] Starting new H1 chunk with title: '{}' (level: {})", + log.info("[CHUNKING] Starting new H1 chunk with title: '{}' (level: {})", currentTitle, currentTitleLevel); } else { - // Title이 아닌 요소들은 현재 청크에 추가 + // Elements without Title are considered as first elements and added to currentChunkContent if (currentTitle != null) { - // 요소 타입에 따라 적절한 구분자 추가 + // Add appropriate separator based on element type if (currentChunkContent.length() > 0) { currentChunkContent.append("\n\n"); } - // 요소 타입별로 적절한 포맷팅 + // Format based on element type switch (elementType) { case "Header" -> { - // 헤더의 레벨에 따라 마크다운 형식 적용 + // Apply markdown formatting based on header level int headerLevel = determineHeaderLevel(element); currentChunkContent.append("#".repeat(Math.max(1, headerLevel))).append(" ").append(text); } @@ -124,7 +124,7 @@ public List createChunks(UUID documentId, List 0) { currentChunkContent.append("\n\n"); @@ -132,31 +132,31 @@ public List createChunks(UUID documentId, List 0) { String chunkContent = currentChunkContent.toString().trim(); - // 제목이 없으면 기본 제목 설정 + // Set default title if none exists if (currentTitle == null) { - currentTitle = "문서 내용"; + currentTitle = "Document Content"; currentTitleLevel = 1; } - // 청크 분할하지 않고 하나의 청크로 생성 + // Create as one chunk without splitting StructuredChunk finalChunk = createTitleBasedChunk(documentId, chunkIndex++, currentTitle, chunkContent, currentTitleLevel, currentChunkElements); chunks.add(finalChunk); - // 마지막 청크 정보 로깅 + // Log final chunk information int finalTokenCount = estimateTokenCount(chunkContent); log.info("📦 [FINAL CHUNK] Title: '{}', Length: {}, Tokens: ~{}, Level: {}, ChunkId: {}", finalChunk.getTitle(), finalChunk.getContent().length(), finalTokenCount, finalChunk.getHierarchyLevel(), finalChunk.getChunkId()); @@ -170,7 +170,7 @@ public List createChunks(UUID documentId, List 0) { double avgChunkLength = chunks.stream() .mapToInt(c -> c.getContent().length()) @@ -185,7 +185,7 @@ public List createChunks(UUID documentId, List createChunks(UUID documentId, List 500 ? chunk.getContent().substring(0, 500) + "..." : chunk.getContent(); log.info(" Content Preview: {}", contentPreview); @@ -209,7 +209,7 @@ public List createChunks(UUID documentId, List> elements) { @@ -219,14 +219,14 @@ private StructuredChunk createTitleBasedChunk(UUID documentId, int chunkIndex, S .content(content) .title(title) .hierarchyLevel(titleLevel) - .parentChunkId(null) // Title 기반 청크는 독립적 + .parentChunkId(null) // Title-based chunks are independent .elementType("TitleBasedChunk") .metadata(createTitleBasedMetadata(title, titleLevel, elements)) .build(); } /** - * Title 기반 청크의 메타데이터를 생성합니다. + * Creates metadata for Title-based chunks. */ private Map createTitleBasedMetadata(String title, int titleLevel, List> elements) { Map metadata = new HashMap<>(); @@ -234,7 +234,7 @@ private Map createTitleBasedMetadata(String title, int titleLeve metadata.put("title_level", titleLevel); metadata.put("element_count", elements.size()); - // 요소 타입별 개수 집계 + // Count elements by type Map elementTypeCounts = elements.stream() .collect(java.util.stream.Collectors.groupingBy( e -> (String) e.get("type"), @@ -242,7 +242,7 @@ private Map createTitleBasedMetadata(String title, int titleLeve )); metadata.put("element_type_counts", elementTypeCounts); - // 첫 번째와 마지막 요소의 메타데이터 정보 보존 + // Preserve metadata information for the first and last elements if (!elements.isEmpty()) { Map firstElement = elements.get(0); Map lastElement = elements.get(elements.size() - 1); @@ -259,8 +259,8 @@ private Map createTitleBasedMetadata(String title, int titleLeve } /** - * H1 마크다운 헤더인지 확인합니다 (# 하나만). - * title_depth=1 설정으로 인해 #만 Title로 올 것이므로 간단한 확인만 필요 + * Checks if it's an H1 markdown header (single #). + * Since title_depth=1 is set, only # will come as Title, so simple check is sufficient */ private boolean isH1MarkdownHeader(String text) { if (text == null || text.trim().isEmpty()) { @@ -268,17 +268,17 @@ private boolean isH1MarkdownHeader(String text) { } String trimmed = text.trim(); - // # 하나로 시작하고 공백이 오는 패턴만 확인 + // Check for pattern starting with single # followed by space if (trimmed.matches("^#\\s+.+")) { String headerText = trimmed.replaceFirst("^#\\s*", "").trim(); - // JavaScript 키워드나 console.log 등으로 시작하는 경우 제외 + // Exclude JavaScript keywords or console.log etc. if (headerText.matches("^(console\\.|function\\s|var\\s|let\\s|const\\s|if\\s*\\(|for\\s*\\(|while\\s*\\(|switch\\s*\\(|class\\s|return\\s|break\\s*;|continue\\s*;).*")) { - log.debug("❌ [CHUNKING] Excluding JavaScript code from H1 header: '{}'", headerText); + log.debug("[CHUNKING] Excluding JavaScript code from H1 header: '{}'", headerText); return false; } - log.debug("✅ [CHUNKING] Valid H1 header detected: '{}'", headerText); + log.debug("[CHUNKING] Valid H1 header detected: '{}'", headerText); return true; } @@ -286,75 +286,75 @@ private boolean isH1MarkdownHeader(String text) { } /** - * 마크다운 헤더에서 텍스트를 추출합니다. + * Extracts text from markdown header. */ private String extractMarkdownHeaderText(String text) { if (text == null || text.trim().isEmpty()) { return text; } String trimmed = text.trim(); - // # 기호들과 공백을 제거하여 실제 제목 텍스트만 추출 + // Remove # symbols and spaces to extract only the actual title text return trimmed.replaceFirst("^#+\\s*", "").trim(); } /** - * 텍스트의 토큰 수를 추정합니다. - * GPT 계열 토크나이저를 기준으로 한 근사치 계산 - * (정확한 토큰 계산을 위해서는 실제 토크나이저 라이브러리를 사용해야 함) + * Estimates the token count of text. + * Approximate calculation based on GPT series tokenizer + * (For accurate token calculation, actual tokenizer library should be used) */ private int estimateTokenCount(String text) { if (text == null || text.trim().isEmpty()) { return 0; } - // 간단한 토큰 추정 알고리즘: - // 1. 공백으로 단어 분리 - // 2. 평균적으로 영어 단어 1개 = 1.3 토큰, 한글 1글자 = 1.5 토큰으로 계산 - // 3. 구두점, 특수문자 등도 고려 + // Simple token estimation algorithm: + // 1. Split by spaces into words + // 2. On average: English word = 1.3 tokens, Korean character = 1.5 tokens + // 3. Consider punctuation and special characters String cleanText = text.trim(); - // 공백 기준 단어 수 + // Word count based on spaces String[] words = cleanText.split("\\s+"); int wordCount = words.length; - // 한글 문자 수 + // Korean character count long koreanCharCount = cleanText.chars() .filter(c -> Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HANGUL_SYLLABLES || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HANGUL_JAMO || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HANGUL_COMPATIBILITY_JAMO) .count(); - // 영문자 수 + // English character count long englishCharCount = cleanText.chars() .filter(c -> (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) .count(); - // 숫자 및 특수문자 수 + // Number and special character count long otherCharCount = cleanText.length() - koreanCharCount - englishCharCount; - // 토큰 수 추정 - // - 영어 단어: 평균 1.3 토큰 - // - 한글 문자: 1.5 토큰 - // - 기타 문자: 0.8 토큰 + // Token count estimation + // - English words: average 1.3 tokens + // - Korean characters: 1.5 tokens + // - Other characters: 0.8 tokens double estimatedTokens = (wordCount * 1.3) + (koreanCharCount * 1.5) + (otherCharCount * 0.8); - // 최소 1 토큰은 보장 + // Ensure minimum 1 token return Math.max(1, (int) Math.round(estimatedTokens)); } /** - * 청크 ID를 생성합니다. + * Generates chunk ID. */ private String generateChunkId(UUID documentId, int chunkIndex) { return documentId.toString() + "-chunk-" + chunkIndex; } /** - * 헤더의 레벨을 결정합니다. + * Determines header level. */ private int determineHeaderLevel(Map element) { - // metadata에서 레벨 정보 추출 시도 + // Attempt to extract level information from metadata @SuppressWarnings("unchecked") Map metadata = (Map) element.get("metadata"); if (metadata != null && metadata.containsKey("category_depth")) { @@ -365,7 +365,7 @@ private int determineHeaderLevel(Map element) { } } - // 기본값 반환 + // Return default value return 2; } } \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/service/ContentRetrievalService.java b/core/src/main/java/com/opencontext/service/ContentRetrievalService.java index c48d2e7..df1762d 100644 --- a/core/src/main/java/com/opencontext/service/ContentRetrievalService.java +++ b/core/src/main/java/com/opencontext/service/ContentRetrievalService.java @@ -15,8 +15,8 @@ import java.util.Map; /** - * 청크 콘텐츠 조회 및 토큰 제한 처리 서비스 - * PRD 명세에 따라 tiktoken-cl100k_base 토크나이저 기준으로 토큰 제한 적용 + * Service for retrieving chunk content and applying token limits + * Applies token limits based on tiktoken-cl100k_base tokenizer according to PRD specifications */ @Slf4j @Service @@ -38,37 +38,37 @@ public class ContentRetrievalService { private String tokenizerName; /** - * 단일 청크의 전체 콘텐츠를 조회하고 토큰 제한을 적용 + * Retrieves the complete content of a single chunk and applies token limits * - * @param chunkId 조회할 청크 ID - * @param maxTokens 최대 토큰 수 (null인 경우 기본값 사용) - * @return 토큰 제한이 적용된 콘텐츠와 토큰 정보 + * @param chunkId Chunk ID to retrieve + * @param maxTokens Maximum token count (uses default if null) + * @return Content with token limits applied and token information */ public GetContentResponse getContent(String chunkId, Integer maxTokens) { long startTime = System.currentTimeMillis(); int effectiveMaxTokens = maxTokens != null ? maxTokens : defaultMaxTokens; - log.info("청크 콘텐츠 조회 시작: chunkId={}, maxTokens={}", chunkId, effectiveMaxTokens); + log.info("Starting chunk content retrieval: chunkId={}, maxTokens={}", chunkId, effectiveMaxTokens); try { - // 1단계: Elasticsearch에서 청크 내용 조회 + // Step 1: Retrieve chunk content from Elasticsearch String content = fetchChunkContent(chunkId); - // 2단계: 토큰 제한 적용 + // Step 2: Apply token limits GetContentResponse response = applyTokenLimit(content, effectiveMaxTokens); long duration = System.currentTimeMillis() - startTime; - log.info("청크 콘텐츠 조회 완료: chunkId={}, 원본길이={}, 토큰수={}, 소요시간={}ms", + log.info("Chunk content retrieval completed: chunkId={}, originalLength={}, tokenCount={}, duration={}ms", chunkId, content.length(), response.getTokenInfo().getActualTokens(), duration); return response; } catch (BusinessException e) { - throw e; // 비즈니스 예외는 그대로 전파 + throw e; // Propagate business exceptions as-is } catch (Exception e) { long duration = System.currentTimeMillis() - startTime; - log.error("청크 콘텐츠 조회 실패: chunkId={}, 소요시간={}ms, 오류={}", + log.error("Chunk content retrieval failed: chunkId={}, duration={}ms, error={}", chunkId, duration, e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Content retrieval failed: " + e.getMessage()); @@ -76,15 +76,15 @@ public GetContentResponse getContent(String chunkId, Integer maxTokens) { } /** - * Elasticsearch에서 특정 청크의 콘텐츠 조회 + * Retrieves content of a specific chunk from Elasticsearch */ private String fetchChunkContent(String chunkId) { - log.debug("Elasticsearch에서 청크 조회: chunkId={}", chunkId); + log.debug("Retrieving chunk from Elasticsearch: chunkId={}", chunkId); try { String getUrl = elasticsearchUrl + "/" + indexName + "/_doc/" + chunkId; - // _source 필터를 사용하여 content 필드만 조회 (성능 최적화) + // Use _source filter to retrieve only content field (performance optimization) String getUrlWithSource = getUrl + "?_source=content"; ResponseEntity response = restTemplate.getForEntity(getUrlWithSource, Map.class); @@ -105,7 +105,7 @@ private String fetchChunkContent(String chunkId) { "Chunk not found: " + chunkId); } - // _source에서 content 추출 + // Extract content from _source Map source = (Map) responseBody.get("_source"); if (source == null) { throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, @@ -118,73 +118,73 @@ private String fetchChunkContent(String chunkId) { "Content field is null for chunk: " + chunkId); } - log.debug("청크 콘텐츠 조회 성공: chunkId={}, 길이={}", chunkId, content.length()); + log.debug("Chunk content retrieval successful: chunkId={}, length={}", chunkId, content.length()); return content; } catch (BusinessException e) { throw e; } catch (Exception e) { - log.error("Elasticsearch 청크 조회 실패: chunkId={}, 오류={}", chunkId, e.getMessage(), e); + log.error("Elasticsearch chunk retrieval failed: chunkId={}, error={}", chunkId, e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Failed to fetch chunk from Elasticsearch: " + e.getMessage()); } } /** - * 콘텐츠에 토큰 제한을 적용하고 응답 DTO 생성 - * PRD 정책: maxTokens 초과 시 텍스트 끝부분을 잘라냄 (앞부분 우선 보존) + * Applies token limits to content and creates response DTO + * PRD policy: Truncate text from the end when maxTokens is exceeded (preserve beginning priority) */ private GetContentResponse applyTokenLimit(String content, int maxTokens) { - log.debug("토큰 제한 적용: 원본길이={}, maxTokens={}", content.length(), maxTokens); + log.debug("Applying token limit: originalLength={}, maxTokens={}", content.length(), maxTokens); try { - // 현재 콘텐츠의 토큰 수 계산 + // Calculate current token count of content int currentTokens = calculateTokenCount(content); String finalContent = content; int actualTokens = currentTokens; - // 토큰 수가 제한을 초과하는 경우 텍스트 끝부분을 잘라냄 + // Truncate text from the end if token count exceeds limit if (currentTokens > maxTokens) { - log.debug("토큰 제한 초과, 텍스트 자르기: 현재토큰={}, 제한토큰={}", currentTokens, maxTokens); + log.debug("Token limit exceeded, truncating text: currentTokens={}, limitTokens={}", currentTokens, maxTokens); finalContent = truncateContentByTokens(content, maxTokens); actualTokens = calculateTokenCount(finalContent); - log.debug("텍스트 자르기 완료: 최종길이={}, 최종토큰={}", finalContent.length(), actualTokens); + log.debug("Text truncation completed: finalLength={}, finalTokens={}", finalContent.length(), actualTokens); } - // 토큰 정보 생성 + // Create token information TokenInfo tokenInfo = TokenInfo.builder() .tokenizer(tokenizerName) .actualTokens(actualTokens) .build(); - // 응답 DTO 생성 + // Create response DTO return GetContentResponse.builder() .content(finalContent) .tokenInfo(tokenInfo) .build(); } catch (Exception e) { - log.error("토큰 제한 적용 실패: 오류={}", e.getMessage(), e); + log.error("Token limit application failed: error={}", e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Token limit processing failed: " + e.getMessage()); } } /** - * tiktoken-cl100k_base 토크나이저 기준으로 토큰 수 계산 - * 간단한 근사치 계산 (정확한 구현을 위해서는 실제 tiktoken 라이브러리 필요) + * Calculate token count based on tiktoken-cl100k_base tokenizer + * Simple approximation (actual tiktoken library needed for accurate implementation) */ private int calculateTokenCount(String text) { if (text == null || text.isEmpty()) { return 0; } - // 간단한 토큰 수 근사 계산 - // 실제로는 tiktoken Java 바인딩이나 외부 API를 사용해야 함 - // 현재는 영어 기준 평균 4글자 = 1토큰, 한글 기준 1.5글자 = 1토큰으로 근사 + // Simple token count approximation + // Actually need tiktoken Java binding or external API + // Currently approximates: English average 4 chars = 1 token, Korean 1.5 chars = 1 token int englishChars = 0; int koreanChars = 0; @@ -193,7 +193,7 @@ private int calculateTokenCount(String text) { for (char c : text.toCharArray()) { if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == ' ') { englishChars++; - } else if (c >= 0xAC00 && c <= 0xD7AF) { // 한글 범위 + } else if (c >= 0xAC00 && c <= 0xD7AF) { // Korean range koreanChars++; } else { otherChars++; @@ -204,19 +204,19 @@ private int calculateTokenCount(String text) { (int) Math.ceil(koreanChars / 1.5) + (int) Math.ceil(otherChars / 2.0); - return Math.max(estimatedTokens, 1); // 최소 1토큰 + return Math.max(estimatedTokens, 1); // Minimum 1 token } /** - * 토큰 수 기준으로 텍스트를 잘라냄 (앞부분 우선 보존) - * PRD 정책: 텍스트 끝부분부터 제거하여 앞부분의 중요한 내용 보존 + * Truncate text based on token count (preserve beginning priority) + * PRD policy: Remove from text end to preserve important content at beginning */ private String truncateContentByTokens(String content, int maxTokens) { if (content == null || content.isEmpty()) { return content; } - // 이진 탐색을 사용하여 적절한 자르기 지점 찾기 + // Use binary search to find the appropriate truncation point int left = 0; int right = content.length(); String result = content; diff --git a/core/src/main/java/com/opencontext/service/DocumentParsingService.java b/core/src/main/java/com/opencontext/service/DocumentParsingService.java index 7f53ceb..48af1db 100644 --- a/core/src/main/java/com/opencontext/service/DocumentParsingService.java +++ b/core/src/main/java/com/opencontext/service/DocumentParsingService.java @@ -13,12 +13,13 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.util.*; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.UUID; /** - * Unstructured API를 사용하여 문서를 파싱하는 서비스. + * Service for parsing documents using the Unstructured API. * * Service for parsing documents using the Unstructured API. * @@ -32,18 +33,18 @@ public class DocumentParsingService { private final FileStorageService fileStorageService; private final RestTemplate restTemplate; - @Value("${app.unstructured.api.url:http://localhost:8000}") + @Value("${app.unstructured.api.url:http://unstructured-api:8000}") private String unstructuredApiUrl; /** - * 문서를 파싱하여 구조화된 요소 목록을 반환합니다. + * Parses a document and returns a list of structured elements. * - * @param documentId 파싱할 문서 ID - * @return 파싱된 요소 목록 (Unstructured API 응답) + * @param documentId Document ID to parse + * @return List of parsed elements (Unstructured API response) */ public List> parseDocument(UUID documentId) { long startTime = System.currentTimeMillis(); - log.info("📝 [PARSING] Starting document parsing: documentId={}", documentId); + log.info("[PARSING] Starting document parsing: documentId={}", documentId); log.debug("📖 [PARSING] Step 1/3: Retrieving document metadata: documentId={}", documentId); SourceDocument document = fileStorageService.getDocument(documentId); @@ -51,27 +52,23 @@ public List> parseDocument(UUID documentId) { String fileType = document.getFileType(); long fileSize = document.getFileSize(); - log.info("✅ [PARSING] Document metadata retrieved: filename={}, fileType={}, size={} bytes", + log.info("[PARSING] Document metadata retrieved: filename={}, fileType={}, size={} bytes", filename, fileType, fileSize); try { - // Step 2: MinIO에서 파일 다운로드 - log.debug("☁️ [PARSING] Step 2/3: Downloading file from MinIO: path={}", document.getFileStoragePath()); + // Step 2: Download file from MinIO + log.debug("[PARSING] Step 2/3: Downloading file from MinIO: path={}", document.getFileStoragePath()); InputStream fileStream = fileStorageService.downloadFile(document.getFileStoragePath()); - log.info("✅ [PARSING] File downloaded from storage: filename={}, path={}", + log.info("[PARSING] File downloaded from storage: filename={}, path={}", filename, document.getFileStoragePath()); - List> parsedElements; - - // 마크다운 파일인 경우 특별 처리 - if (isMarkdownFile(filename, fileType)) { - log.info("📝 [PARSING] Detected markdown file, using hybrid parsing approach: {}", filename); - parsedElements = parseMarkdownDocument(fileStream, filename, fileType); - } else { - // Step 3: Unstructured API 호출 - log.debug("🤖 [PARSING] Step 3/3: Calling Unstructured API: filename={}, fileType={}", filename, fileType); - parsedElements = callUnstructuredApi(fileStream, filename, fileType); - } + // Step 3: Call Unstructured API + log.debug("🤖 [PARSING] Step 3/3: Calling Unstructured API: filename={}, fileType={}", filename, fileType); + List> parsedElements = callUnstructuredApi( + fileStream, + filename, + fileType + ); long duration = System.currentTimeMillis() - startTime; log.info("🎉 [PARSING] Document parsing completed successfully: documentId={}, filename={}, elements={}, duration={}ms", @@ -81,7 +78,7 @@ public List> parseDocument(UUID documentId) { } catch (Exception e) { long duration = System.currentTimeMillis() - startTime; - log.error("❌ [PARSING] Document parsing failed: documentId={}, filename={}, duration={}ms, error={}", + log.error("[PARSING] Document parsing failed: documentId={}, filename={}, duration={}ms, error={}", documentId, filename, duration, e.getMessage(), e); throw new BusinessException(ErrorCode.DOCUMENT_PARSING_FAILED, "Document parsing failed: " + e.getMessage()); @@ -89,7 +86,7 @@ public List> parseDocument(UUID documentId) { } /** - * Unstructured API를 호출하여 문서를 파싱합니다. + * Calls the Unstructured API to parse the document. */ @SuppressWarnings("unchecked") private List> callUnstructuredApi(InputStream fileStream, @@ -100,44 +97,27 @@ private List> callUnstructuredApi(InputStream fileStream, filename, fileType, unstructuredApiUrl); try { - // HTTP 헤더 설정 + // Set HTTP headers HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); - // 멀티파트 요청 생성 + // Create multipart request log.debug("📦 [UNSTRUCTURED-API] Preparing multipart request: filename={}", filename); MultiValueMap body = new LinkedMultiValueMap<>(); body.add("files", new InputStreamResource(fileStream, filename)); - // 파일 타입별 파싱 옵션 설정 - String strategy = determineOptimalStrategy(fileType, filename); - body.add("strategy", strategy); + // Set parsing options + body.add("strategy", "auto"); body.add("coordinates", "true"); body.add("extract_images_in_pdf", "false"); body.add("infer_table_structure", "true"); - // 마크다운 파일의 경우 추가 파라미터 설정 - if (isMarkdownFile(filename, fileType)) { - // 마크다운 파싱을 위한 특별한 설정 - body.add("include_page_breaks", "false"); - body.add("split_pdf_page", "false"); - body.add("split_pdf_allow_failed", "true"); - // 마크다운 헤더 인식 개선을 위한 설정 - body.add("skip_infer_table_types", "[]"); - body.add("languages", "ko,en"); // 한국어, 영어 지원 - body.add("detect_language_per_element", "true"); - // # (h1)만 title로 인식하도록 설정 - body.add("title_depth", "1"); // 1로 설정하면 #만 title로 인식 - body.add("heading_detection_strategy", "markdown"); // 마크다운 헤더 감지 전략 - log.debug("📝 [UNSTRUCTURED-API] Markdown-specific parameters applied with title_depth=1 (only # headers as titles)"); - } - - log.debug("⚙️ [UNSTRUCTURED-API] Request parameters: strategy={}, coordinates=true, extract_images_in_pdf=false, infer_table_structure=true", strategy); + log.debug("[UNSTRUCTURED-API] Request parameters: strategy=auto, coordinates=true, extract_images_in_pdf=false, infer_table_structure=true"); HttpEntity> requestEntity = new HttpEntity<>(body, headers); - // API 호출 - log.debug("🚀 [UNSTRUCTURED-API] Sending HTTP request: {}/general/v0/general", unstructuredApiUrl); + // Call API + log.debug("[UNSTRUCTURED-API] Sending HTTP request: {}/general/v0/general", unstructuredApiUrl); ResponseEntity>> response = restTemplate.exchange( unstructuredApiUrl + "/general/v0/general", HttpMethod.POST, @@ -148,7 +128,7 @@ private List> callUnstructuredApi(InputStream fileStream, long apiDuration = System.currentTimeMillis() - apiStartTime; if (response.getStatusCode() != HttpStatus.OK || response.getBody() == null) { - log.error("❌ [UNSTRUCTURED-API] API returned unexpected response: filename={}, status={}, duration={}ms", + log.error("[UNSTRUCTURED-API] API returned unexpected response: filename={}, status={}, duration={}ms", filename, response.getStatusCode(), apiDuration); throw new BusinessException(ErrorCode.EXTERNAL_API_ERROR, "Unstructured API returned unexpected response"); @@ -157,7 +137,7 @@ private List> callUnstructuredApi(InputStream fileStream, List> elements = response.getBody(); int elementCount = elements.size(); - log.info("✅ [UNSTRUCTURED-API] API call successful: filename={}, elements={}, duration={}ms, status={}", + log.info("[UNSTRUCTURED-API] API call successful: filename={}, elements={}, duration={}ms, status={}", filename, elementCount, apiDuration, response.getStatusCode()); if (elementCount > 0) { @@ -166,42 +146,13 @@ private List> callUnstructuredApi(InputStream fileStream, .map(e -> e.get("type")) .distinct() .collect(java.util.stream.Collectors.toList())); - - // 첫 번째 몇 개 요소의 상세 정보 로깅 - log.info("📋 [UNSTRUCTURED-API] First 10 elements details:"); - for (int i = 0; i < Math.min(10, elementCount); i++) { - Map element = elements.get(i); - String type = (String) element.get("type"); - String text = (String) element.get("text"); - - // 텍스트가 너무 길면 잘라서 표시 - String displayText = text != null && text.length() > 100 ? - text.substring(0, 100) + "..." : text; - - log.info(" {}. Type: '{}' | Text: '{}'", i + 1, type, displayText); - } - - // 마크다운 헤더를 찾아서 로깅 - log.info("🔍 [UNSTRUCTURED-API] Searching for markdown headers in parsed elements:"); - int headerCount = 0; - for (int i = 0; i < elementCount; i++) { - Map element = elements.get(i); - String text = (String) element.get("text"); - if (text != null && text.trim().matches("^#+\\s+.+")) { - headerCount++; - log.info(" Found potential header #{}: '{}'", headerCount, text.trim()); - } - } - if (headerCount == 0) { - log.warn("⚠️ [UNSTRUCTURED-API] No markdown headers found in parsed elements"); - } } return elements; } catch (Exception e) { long apiDuration = System.currentTimeMillis() - apiStartTime; - log.error("❌ [UNSTRUCTURED-API] API call failed: filename={}, duration={}ms, error={}", + log.error("[UNSTRUCTURED-API] API call failed: filename={}, duration={}ms, error={}", filename, apiDuration, e.getMessage(), e); throw new BusinessException(ErrorCode.EXTERNAL_API_ERROR, "Failed to call Unstructured API: " + e.getMessage()); @@ -209,50 +160,7 @@ private List> callUnstructuredApi(InputStream fileStream, } /** - * 파일 타입에 따라 최적의 파싱 전략을 결정합니다. - */ - private String determineOptimalStrategy(String fileType, String filename) { - if (isMarkdownFile(filename, fileType)) { - return "fast"; // 마크다운 파일은 fast 전략이 헤더 인식에 더 적합 - } else if (isPdfFile(filename, fileType)) { - return "hi_res"; // PDF는 레이아웃 분석이 중요 - } else { - return "auto"; // 기타 파일은 자동 탐지 - } - } - - /** - * 마크다운 파일인지 확인합니다. - */ - private boolean isMarkdownFile(String filename, String fileType) { - if (filename != null) { - String lowerFilename = filename.toLowerCase(); - if (lowerFilename.endsWith(".md") || lowerFilename.endsWith(".markdown")) { - return true; - } - } - if (fileType != null) { - String lowerFileType = fileType.toLowerCase(); - return lowerFileType.contains("markdown") || lowerFileType.equals("text/markdown"); - } - return false; - } - - /** - * PDF 파일인지 확인합니다. - */ - private boolean isPdfFile(String filename, String fileType) { - if (filename != null && filename.toLowerCase().endsWith(".pdf")) { - return true; - } - if (fileType != null) { - return fileType.toLowerCase().contains("pdf"); - } - return false; - } - - /** - * InputStream을 Spring의 Resource로 래핑하는 헬퍼 클래스 + * Helper class that wraps InputStream as Spring's Resource */ private static class InputStreamResource extends org.springframework.core.io.InputStreamResource { private final String filename; @@ -269,263 +177,7 @@ public String getFilename() { @Override public long contentLength() { - return -1; // 알 수 없음을 나타냄 + return -1; // Indicates unknown } } - - /** - * 마크다운 파일을 위한 하이브리드 파싱 메서드 - * 직접 파싱과 Unstructured API를 비교하여 더 나은 결과를 선택합니다. - */ - private List> parseMarkdownDocument(InputStream fileStream, String filename, String fileType) { - // 스트림을 두 번 사용하기 위해 바이트 배열로 변환 - try { - byte[] fileContent = fileStream.readAllBytes(); - - // 먼저 직접 파싱 시도 - try (InputStream directStream = new ByteArrayInputStream(fileContent)) { - List> directResult = processMarkdownDirectly(directStream, filename); - - // 헤더가 제대로 인식되었는지 확인 - long headerCount = directResult.stream() - .filter(e -> "Title".equals(e.get("type"))) - .map(e -> (String) e.get("text")) - .filter(text -> text != null && text.trim().matches("^#+\\s+.+")) - .count(); - - if (headerCount > 0) { - log.info("✅ [MARKDOWN-DECISION] Using direct processing: {} valid headers found", headerCount); - return directResult; - } - } - - // 직접 파싱이 실패했다면 Unstructured API 사용 - log.info("🔄 [MARKDOWN-DECISION] Direct parsing failed, falling back to Unstructured API"); - try (InputStream apiStream = new ByteArrayInputStream(fileContent)) { - List> apiResult = callUnstructuredApi(apiStream, filename, fileType); - - // API 결과에서 마크다운 후처리 적용 - return postProcessMarkdownElements(apiResult); - } - - } catch (IOException e) { - log.error("❌ [MARKDOWN-DECISION] Failed to process markdown: {}", filename, e); - throw new BusinessException(ErrorCode.DOCUMENT_PARSING_FAILED, - "Failed to process markdown file: " + e.getMessage()); - } - } - - /** - * 마크다운 파일을 위한 전용 직접 처리 메서드 - */ - private List> processMarkdownDirectly(InputStream fileStream, String filename) { - log.debug("📝 [MARKDOWN-PROCESSOR] Processing markdown file directly: {}", filename); - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(fileStream, StandardCharsets.UTF_8))) { - List> elements = new ArrayList<>(); - StringBuilder currentContent = new StringBuilder(); - String currentType = null; - int elementIndex = 0; - boolean inCodeBlock = false; - String codeLanguage = null; - - String line; - while ((line = reader.readLine()) != null) { - String trimmedLine = line.trim(); - - // 코드 블록 처리 - if (trimmedLine.startsWith("```")) { - if (!inCodeBlock) { - // 코드 블록 시작 - if (currentContent.length() > 0) { - addElement(elements, currentType, currentContent.toString(), filename, elementIndex++); - currentContent.setLength(0); - } - inCodeBlock = true; - codeLanguage = trimmedLine.substring(3).trim(); - currentType = "CodeBlock"; - } else { - // 코드 블록 종료 - addElement(elements, "CodeBlock", currentContent.toString(), filename, elementIndex++, codeLanguage); - currentContent.setLength(0); - inCodeBlock = false; - codeLanguage = null; - currentType = null; - } - continue; - } - - if (inCodeBlock) { - if (currentContent.length() > 0) { - currentContent.append("\n"); - } - currentContent.append(line); - continue; - } - - // 헤더 처리 (# ~ ######) - if (trimmedLine.matches("^#{1,6}\\s+.+")) { - if (currentContent.length() > 0) { - addElement(elements, currentType, currentContent.toString(), filename, elementIndex++); - currentContent.setLength(0); - } - - addElement(elements, "Title", trimmedLine, filename, elementIndex++); - currentType = null; - continue; - } - - // 빈 줄 처리 - if (trimmedLine.isEmpty()) { - if (currentContent.length() > 0 && currentType != null) { - addElement(elements, currentType, currentContent.toString(), filename, elementIndex++); - currentContent.setLength(0); - currentType = null; - } - continue; - } - - // 목록 처리 - if (trimmedLine.matches("^[-*+]\\s+.+") || trimmedLine.matches("^\\d+\\.\\s+.+")) { - if (currentType != null && !currentType.equals("ListItem")) { - if (currentContent.length() > 0) { - addElement(elements, currentType, currentContent.toString(), filename, elementIndex++); - currentContent.setLength(0); - } - } - - if (currentType == null || !currentType.equals("ListItem")) { - currentType = "ListItem"; - } - - if (currentContent.length() > 0) { - currentContent.append("\n"); - } - currentContent.append(line); - continue; - } - - // 인용문 처리 - if (trimmedLine.startsWith(">")) { - if (currentType != null && !currentType.equals("Quote")) { - if (currentContent.length() > 0) { - addElement(elements, currentType, currentContent.toString(), filename, elementIndex++); - currentContent.setLength(0); - } - } - currentType = "Quote"; - if (currentContent.length() > 0) { - currentContent.append("\n"); - } - currentContent.append(line); - continue; - } - - // 일반 텍스트 처리 - if (currentType == null) { - currentType = "NarrativeText"; - } - - if (currentContent.length() > 0) { - currentContent.append("\n"); - } - currentContent.append(line); - } - - // 마지막 요소 처리 - if (currentContent.length() > 0) { - addElement(elements, currentType != null ? currentType : "NarrativeText", - currentContent.toString(), filename, elementIndex++, codeLanguage); - } - - log.info("✅ [MARKDOWN-PROCESSOR] Processed {} elements from markdown file: {}", - elements.size(), filename); - - return elements; - - } catch (IOException e) { - log.error("❌ [MARKDOWN-PROCESSOR] Failed to process markdown file: {}", filename, e); - throw new BusinessException(ErrorCode.DOCUMENT_PARSING_FAILED, - "Failed to process markdown file: " + e.getMessage()); - } - } - - /** - * 요소를 리스트에 추가하는 헬퍼 메서드 - */ - private void addElement(List> elements, String type, String content, - String filename, int index, String... additionalInfo) { - Map element = new HashMap<>(); - element.put("type", type); - element.put("text", content.trim()); - element.put("element_id", UUID.randomUUID().toString().replace("-", "")); - - Map metadata = new HashMap<>(); - metadata.put("filename", filename); - metadata.put("filetype", "text/markdown"); - metadata.put("languages", List.of("eng")); - metadata.put("element_index", index); - - if (additionalInfo.length > 0 && additionalInfo[0] != null) { - metadata.put("code_language", additionalInfo[0]); - } - - element.put("metadata", metadata); - elements.add(element); - } - - /** - * 마크다운 파싱 결과를 후처리하여 구조를 개선합니다. - */ - private List> postProcessMarkdownElements(List> elements) { - List> processedElements = new ArrayList<>(); - - for (Map element : elements) { - String type = (String) element.get("type"); - String text = (String) element.get("text"); - - if (text != null && text.trim().length() > 0) { - String trimmedText = text.trim(); - - // 마크다운 헤더 패턴 감지 및 수정 - if (trimmedText.matches("^#+\\s+.+")) { - // 헤더로 인식되어야 하는데 다른 타입으로 분류된 경우 - Map correctedElement = new HashMap<>(element); - correctedElement.put("type", "Title"); - processedElements.add(correctedElement); - log.debug("🔧 [POST-PROCESS] Corrected header: '{}' -> Title", trimmedText); - } else if (trimmedText.startsWith("```") || trimmedText.endsWith("```")) { - // 코드 블록 시작/끝 마커는 제거 - continue; - } else if ("Title".equals(type) && !trimmedText.matches("^#+\\s+.+") && - !trimmedText.startsWith("**") && trimmedText.length() > 100) { - // Title로 잘못 분류된 긴 텍스트를 NarrativeText로 수정 - Map correctedElement = new HashMap<>(element); - correctedElement.put("type", "NarrativeText"); - processedElements.add(correctedElement); - log.debug("🔧 [POST-PROCESS] Corrected long title to narrative: '{}'...", - trimmedText.substring(0, Math.min(50, trimmedText.length()))); - } else if ("Title".equals(type) && (trimmedText.startsWith("console.log") || - trimmedText.startsWith("//") || trimmedText.contains("function ") || - trimmedText.contains("const ") || trimmedText.contains("let ") || - trimmedText.contains("var "))) { - // JavaScript 코드가 Title로 잘못 분류된 경우 - Map correctedElement = new HashMap<>(element); - correctedElement.put("type", "CodeBlock"); - processedElements.add(correctedElement); - log.debug("🔧 [POST-PROCESS] Corrected JS code title to code block: '{}'...", - trimmedText.substring(0, Math.min(30, trimmedText.length()))); - } else { - processedElements.add(element); - } - } else { - processedElements.add(element); - } - } - - log.info("🔧 [POST-PROCESS] Post-processing completed: {} -> {} elements", - elements.size(), processedElements.size()); - - return processedElements; - } } diff --git a/core/src/main/java/com/opencontext/service/EmbeddingService.java b/core/src/main/java/com/opencontext/service/EmbeddingService.java index 62c2f4a..082e88c 100644 --- a/core/src/main/java/com/opencontext/service/EmbeddingService.java +++ b/core/src/main/java/com/opencontext/service/EmbeddingService.java @@ -15,7 +15,7 @@ import java.util.*; /** - * LangChain4j와 Ollama를 사용하여 텍스트 청크의 임베딩 벡터를 생성하는 서비스. + * Service for generating embedding vectors for text chunks using LangChain4j and Ollama. * * Service for generating embedding vectors for text chunks using LangChain4j and Ollama. * @@ -44,11 +44,11 @@ public EmbeddingService( } /** - * 구조화된 청크들의 임베딩 벡터를 생성합니다. + * Generates embedding vectors for structured chunks. * - * @param documentId 문서 ID - * @param structuredChunks 임베딩을 생성할 청크 목록 - * @return 임베딩이 포함된 청크 목록 + * @param documentId Document ID + * @param structuredChunks List of chunks to generate embeddings for + * @return List of chunks with embeddings included */ public List generateEmbeddings(UUID documentId, List structuredChunks) { long startTime = System.currentTimeMillis(); @@ -63,7 +63,7 @@ public List generateEmbeddings(UUID documentId, List batch = structuredChunks.subList(i, endIndex); @@ -79,7 +79,7 @@ public List generateEmbeddings(UUID documentId, List generateEmbeddings(UUID documentId, List 0) { long avgEmbeddingTime = duration / finalEmbeddedCount; log.debug("📊 [EMBEDDING] Statistics: avgTimePerChunk={}ms, batchSize={}, totalBatches={}", @@ -100,7 +100,7 @@ public List generateEmbeddings(UUID documentId, List processBatchWithLangChain4j(List batch) { long batchStartTime = System.currentTimeMillis(); @@ -109,7 +109,7 @@ private List processBatchWithLangChain4j(List List result = new ArrayList<>(); try { - // 배치 처리를 위한 TextSegment 목록 생성 + // Create TextSegment list for batch processing List textSegments = new ArrayList<>(); for (StructuredChunk chunk : batch) { String textForEmbedding = prepareTextForEmbedding(chunk); @@ -117,30 +117,30 @@ private List processBatchWithLangChain4j(List textSegments.add(segment); } - // LangChain4j를 사용한 배치 임베딩 생성 - log.debug("🚀 [LANGCHAIN4J] Calling Ollama embedding model: segments={}", textSegments.size()); + // Generate batch embeddings using LangChain4j + log.debug("[LANGCHAIN4J] Calling Ollama embedding model: segments={}", textSegments.size()); long embeddingStartTime = System.currentTimeMillis(); Response> response = embeddingModel.embedAll(textSegments); List embeddings = response.content(); long embeddingDuration = System.currentTimeMillis() - embeddingStartTime; - log.info("✅ [LANGCHAIN4J] Ollama embedding completed: segments={}, duration={}ms, vectorDimension={}", + log.info("[LANGCHAIN4J] Ollama embedding completed: segments={}, duration={}ms, vectorDimension={}", textSegments.size(), embeddingDuration, embeddings.size() > 0 ? embeddings.get(0).dimension() : 0); - // 결과 처리 + // Process results for (int i = 0; i < batch.size(); i++) { StructuredChunk chunk = batch.get(i); Embedding embedding = embeddings.get(i); - // float[] 벡터를 List로 변환 + // Convert float[] vector to List List embeddingVector = new ArrayList<>(); float[] vector = embedding.vector(); for (float value : vector) { embeddingVector.add((double) value); } - // 임베딩이 포함된 새로운 청크 생성 + // Create new chunk with embedding included StructuredChunk embeddedChunk = StructuredChunk.builder() .chunkId(chunk.getChunkId()) .documentId(chunk.getDocumentId()) @@ -155,17 +155,17 @@ private List processBatchWithLangChain4j(List result.add(embeddedChunk); - log.debug("✅ [LANGCHAIN4J] Embedding processed for chunk: id={}, vectorSize={}, textLength={}", + log.debug("[LANGCHAIN4J] Embedding processed for chunk: id={}, vectorSize={}, textLength={}", chunk.getChunkId(), embedding.dimension(), chunk.getContent().length()); } long batchDuration = System.currentTimeMillis() - batchStartTime; - log.info("✅ [LANGCHAIN4J] Batch processing completed: processed={}, duration={}ms, avgPerChunk={}ms", + log.info("[LANGCHAIN4J] Batch processing completed: processed={}, duration={}ms, avgPerChunk={}ms", result.size(), batchDuration, batchDuration / batch.size()); } catch (Exception e) { long batchDuration = System.currentTimeMillis() - batchStartTime; - log.error("❌ [LANGCHAIN4J] Batch processing failed: chunks={}, duration={}ms, error={}", + log.error("[LANGCHAIN4J] Batch processing failed: chunks={}, duration={}ms, error={}", batch.size(), batchDuration, e.getMessage(), e); throw new BusinessException(ErrorCode.EMBEDDING_GENERATION_FAILED, "Failed to generate embeddings: " + e.getMessage()); @@ -175,25 +175,25 @@ private List processBatchWithLangChain4j(List } /** - * 임베딩 생성을 위한 텍스트를 준비합니다. - * 제목과 내용을 결합하여 더 풍부한 컨텍스트를 제공합니다. + * Prepares text for embedding generation. + * Combines title and content to provide richer context. */ private String prepareTextForEmbedding(StructuredChunk chunk) { StringBuilder text = new StringBuilder(); - log.debug("📝 [EMBEDDING] Preparing text for embedding: chunkId={}", chunk.getChunkId()); + log.debug("[EMBEDDING] Preparing text for embedding: chunkId={}", chunk.getChunkId()); - // 제목이 있으면 추가 + // Add title if exists if (chunk.getTitle() != null && !chunk.getTitle().trim().isEmpty()) { text.append("Title: ").append(chunk.getTitle()).append("\n"); } - // 내용 추가 + // Add content text.append(chunk.getContent()); String finalText = text.toString().trim(); - log.debug("✅ [EMBEDDING] Text prepared: chunkId={}, finalLength={}, hasTitle={}", + log.debug("[EMBEDDING] Text prepared: chunkId={}, finalLength={}, hasTitle={}", chunk.getChunkId(), finalText.length(), chunk.getTitle() != null); return finalText; diff --git a/core/src/main/java/com/opencontext/service/FileStorageService.java b/core/src/main/java/com/opencontext/service/FileStorageService.java index 23760c2..0ddda3e 100644 --- a/core/src/main/java/com/opencontext/service/FileStorageService.java +++ b/core/src/main/java/com/opencontext/service/FileStorageService.java @@ -71,38 +71,38 @@ public SourceDocument uploadFileWithMetadata(MultipartFile file) { long fileSize = file.getSize(); String contentType = resolveContentType(file); - log.info("📤 [UPLOAD] Starting file upload with metadata: filename={}, size={} bytes, contentType={}", + log.info("[UPLOAD] Starting file upload with metadata: filename={}, size={} bytes, contentType={}", filename, fileSize, contentType); long startTime = System.currentTimeMillis(); // Validate file - log.debug("📋 [UPLOAD] Step 1/5: Validating file: {}", filename); + log.debug(" [UPLOAD] Step 1/5: Validating file: {}", filename); validateFile(file); - log.info("✅ [UPLOAD] File validation completed successfully: {}", filename); + log.info("[UPLOAD] File validation completed successfully: {}", filename); // Calculate file checksum to prevent duplicates - log.debug("🔍 [UPLOAD] Step 2/5: Calculating file checksum: {}", filename); + log.debug("[UPLOAD] Step 2/5: Calculating file checksum: {}", filename); String fileChecksum = calculateFileChecksum(file); - log.info("✅ [UPLOAD] File checksum calculated: {} -> {}", filename, fileChecksum); + log.info("[UPLOAD] File checksum calculated: {} -> {}", filename, fileChecksum); // Check for duplicate files - log.debug("🔍 [UPLOAD] Step 3/5: Checking for duplicate files: {}", filename); + log.debug("[UPLOAD] Step 3/5: Checking for duplicate files: {}", filename); if (sourceDocumentRepository.existsByFileChecksum(fileChecksum)) { - log.warn("❌ [UPLOAD] Duplicate file upload attempt: filename={}, checksum={}", filename, fileChecksum); + log.warn("[UPLOAD] Duplicate file upload attempt: filename={}, checksum={}", filename, fileChecksum); throw new BusinessException(ErrorCode.DUPLICATE_FILE_UPLOADED, "A file with identical content already exists."); } - log.info("✅ [UPLOAD] No duplicate files found: {}", filename); + log.info("[UPLOAD] No duplicate files found: {}", filename); try { // Upload file to MinIO - log.debug("☁️ [UPLOAD] Step 4/5: Uploading file to MinIO: {}", filename); + log.debug("[UPLOAD] Step 4/5: Uploading file to MinIO: {}", filename); String objectKey = uploadFile(file, contentType); - log.info("✅ [UPLOAD] File uploaded to MinIO successfully: {} -> {}", filename, objectKey); + log.info("[UPLOAD] File uploaded to MinIO successfully: {} -> {}", filename, objectKey); // Create SourceDocument entity - log.debug("💾 [UPLOAD] Step 5/5: Creating database record: {}", filename); + log.debug("[UPLOAD] Step 5/5: Creating database record: {}", filename); SourceDocument sourceDocument = SourceDocument.builder() .originalFilename(file.getOriginalFilename()) .fileStoragePath(objectKey) @@ -116,14 +116,14 @@ public SourceDocument uploadFileWithMetadata(MultipartFile file) { SourceDocument savedDocument = sourceDocumentRepository.save(sourceDocument); long duration = System.currentTimeMillis() - startTime; - log.info("🎉 [UPLOAD] File upload completed successfully: id={}, filename={}, duration={}ms", + log.info(" [UPLOAD] File upload completed successfully: id={}, filename={}, duration={}ms", savedDocument.getId(), savedDocument.getOriginalFilename(), duration); return savedDocument; } catch (Exception e) { long duration = System.currentTimeMillis() - startTime; - log.error("❌ [UPLOAD] File upload failed: filename={}, duration={}ms, error={}", + log.error("[UPLOAD] File upload failed: filename={}, duration={}ms, error={}", filename, duration, e.getMessage(), e); if (e instanceof BusinessException) { throw e; @@ -157,13 +157,13 @@ public String uploadFile(MultipartFile file, String resolvedContentType) { minioClient.putObject(putObjectArgs); - log.debug("☁️ [MINIO] File uploaded to MinIO: {} -> bucket:{}, key:{}", + log.debug("[MINIO] File uploaded to MinIO: {} -> bucket:{}, key:{}", file.getOriginalFilename(), minioConfig.getBucketName(), objectKey); return objectKey; } catch (Exception e) { - log.error("❌ [MINIO] MinIO upload failed: filename={}, error={}", + log.error("[MINIO] MinIO upload failed: filename={}, error={}", file.getOriginalFilename(), e.getMessage(), e); throw new BusinessException(ErrorCode.FILE_UPLOAD_FAILED, "Failed to upload file to storage: " + e.getMessage()); @@ -184,11 +184,11 @@ public InputStream downloadFile(String objectKey) { .build(); InputStream stream = minioClient.getObject(getObjectArgs); - log.debug("☁️ [MINIO] File downloaded successfully: key={}", objectKey); + log.debug("[MINIO] File downloaded successfully: key={}", objectKey); return stream; } catch (Exception e) { - log.error("❌ [MINIO] File download failed: key={}, error={}", objectKey, e.getMessage(), e); + log.error("[MINIO] File download failed: key={}, error={}", objectKey, e.getMessage(), e); throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "Failed to download file: " + e.getMessage()); } @@ -207,10 +207,10 @@ public void deleteFile(String objectKey) { .build(); minioClient.removeObject(removeObjectArgs); - log.debug("🗑️ [MINIO] File deleted successfully: key={}", objectKey); + log.debug("[MINIO] File deleted successfully: key={}", objectKey); } catch (Exception e) { - log.error("❌ [MINIO] File deletion failed: key={}, error={}", objectKey, e.getMessage(), e); + log.error("[MINIO] File deletion failed: key={}, error={}", objectKey, e.getMessage(), e); throw new BusinessException(ErrorCode.FILE_DELETE_FAILED, "Failed to delete file: " + e.getMessage()); } @@ -254,11 +254,11 @@ private void ensureBucketExists() { .build(); minioClient.makeBucket(makeBucketArgs); - log.info("☁️ [MINIO] Created MinIO bucket: {}", minioConfig.getBucketName()); + log.info(" [MINIO] Created MinIO bucket: {}", minioConfig.getBucketName()); } } catch (Exception e) { - log.error("❌ [MINIO] Failed to ensure bucket exists: bucket={}, error={}", + log.error("[MINIO] Failed to ensure bucket exists: bucket={}, error={}", minioConfig.getBucketName(), e.getMessage(), e); throw new BusinessException(ErrorCode.STORAGE_ERROR, "Failed to ensure bucket exists: " + e.getMessage()); @@ -292,7 +292,7 @@ private String generateObjectKey(String originalFilename) { */ @Transactional(readOnly = true) public SourceDocument getDocument(UUID documentId) { - log.debug("📖 [QUERY] Retrieving document: id={}", documentId); + log.debug(" [QUERY] Retrieving document: id={}", documentId); return sourceDocumentRepository.findById(documentId) .orElseThrow(() -> new BusinessException(ErrorCode.SOURCE_DOCUMENT_NOT_FOUND, @@ -307,7 +307,7 @@ public SourceDocument getDocument(UUID documentId) { */ @Transactional(readOnly = true) public Page getAllDocuments(Pageable pageable) { - log.debug("📖 [QUERY] Retrieving documents with pagination: page={}, size={}", + log.debug(" [QUERY] Retrieving documents with pagination: page={}, size={}", pageable.getPageNumber(), pageable.getPageSize()); return sourceDocumentRepository.findAllByOrderByCreatedAtDesc(pageable) @@ -325,7 +325,7 @@ public void updateDocumentStatus(UUID documentId, IngestionStatus status) { .ifPresent(document -> { document.updateIngestionStatus(status); sourceDocumentRepository.save(document); - log.info("📝 [STATUS] Document status updated: id={}, status={}", documentId, status); + log.info(" [STATUS] Document status updated: id={}, status={}", documentId, status); }); } @@ -339,7 +339,7 @@ public void updateDocumentStatusToCompleted(UUID documentId) { .ifPresent(document -> { document.updateIngestionStatus(IngestionStatus.COMPLETED); sourceDocumentRepository.save(document); - log.info("🎉 [STATUS] Document status updated to COMPLETED: id={}", documentId); + log.info(" [STATUS] Document status updated to COMPLETED: id={}", documentId); }); } @@ -355,10 +355,10 @@ public void updateDocumentStatusToError(UUID documentId, String errorMessage) { .ifPresent(document -> { document.updateIngestionStatusToError(errorMessage); sourceDocumentRepository.save(document); - log.error("❌ [STATUS] Document status updated to ERROR: id={}, errorMessage={}", documentId, errorMessage); + log.error("[STATUS] Document status updated to ERROR: id={}, errorMessage={}", documentId, errorMessage); }); } catch (Exception e) { - log.error("❌ [STATUS] Failed to update document status to ERROR: id={}, error={}", + log.error("[STATUS] Failed to update document status to ERROR: id={}, error={}", documentId, e.getMessage(), e); } } @@ -372,18 +372,18 @@ public void updateDocumentStatusToError(UUID documentId, String errorMessage) { @Transactional public void deleteDocument(UUID documentId) { long startTime = System.currentTimeMillis(); - log.info("🗑️ [DELETE] Starting comprehensive document deletion: id={}", documentId); + log.info("[DELETE] Starting comprehensive document deletion: id={}", documentId); SourceDocument document = getDocument(documentId); String filename = document.getOriginalFilename(); IngestionStatus status = document.getIngestionStatus(); - log.info("📝 [DELETE] Document details: filename={}, status={}, size={} bytes", + log.info(" [DELETE] Document details: filename={}, status={}, size={} bytes", filename, status, document.getFileSize()); // Check if document is currently being processed (but allow DELETING status) if (document.isProcessing() && status != IngestionStatus.DELETING) { - log.warn("⚠️ [DELETE] Cannot delete document in processing state: id={}, status={}", + log.warn(" [DELETE] Cannot delete document in processing state: id={}, status={}", documentId, status); throw new BusinessException(ErrorCode.RESOURCE_IS_BEING_PROCESSED, "Document is currently being processed and cannot be deleted."); @@ -391,42 +391,42 @@ public void deleteDocument(UUID documentId) { // Update status to DELETING (only if not already DELETING) if (status != IngestionStatus.DELETING) { - log.debug("📝 [DELETE] Step 1/4: Updating status to DELETING: {}", filename); + log.debug(" [DELETE] Step 1/4: Updating status to DELETING: {}", filename); document.updateIngestionStatus(IngestionStatus.DELETING); sourceDocumentRepository.save(document); - log.info("✅ [DELETE] Status updated to DELETING: {}", filename); + log.info(" [DELETE] Status updated to DELETING: {}", filename); } else { - log.info("📝 [DELETE] Document already in DELETING status: {}", filename); + log.info(" [DELETE] Document already in DELETING status: {}", filename); } try { // Step 2: Delete from Elasticsearch (if exists) - log.debug("🔍 [DELETE] Step 2/4: Deleting from Elasticsearch: {}", filename); + log.debug(" [DELETE] Step 2/4: Deleting from Elasticsearch: {}", filename); deleteFromElasticsearch(documentId); - log.info("✅ [DELETE] Elasticsearch deletion completed: {}", filename); + log.info("[DELETE] Elasticsearch deletion completed: {}", filename); // Step 3: Delete chunks from PostgreSQL - log.debug("💾 [DELETE] Step 3/4: Deleting chunks from PostgreSQL: {}", filename); + log.debug("[DELETE] Step 3/4: Deleting chunks from PostgreSQL: {}", filename); int deletedChunks = deleteChunksFromPostgreSQL(documentId); - log.info("✅ [DELETE] PostgreSQL chunks deleted: {} chunks removed for {}", deletedChunks, filename); + log.info(" [DELETE] PostgreSQL chunks deleted: {} chunks removed for {}", deletedChunks, filename); // Step 4: Delete file from MinIO - log.debug("☁️ [DELETE] Step 4/4: Deleting file from MinIO: {}", filename); + log.debug("[DELETE] Step 4/4: Deleting file from MinIO: {}", filename); deleteFile(document.getFileStoragePath()); - log.info("✅ [DELETE] MinIO file deleted: {} -> {}", filename, document.getFileStoragePath()); + log.info("[DELETE] MinIO file deleted: {} -> {}", filename, document.getFileStoragePath()); // Final step: Delete SourceDocument record - log.debug("💾 [DELETE] Final step: Deleting source document record: {}", filename); + log.debug("[DELETE] Final step: Deleting source document record: {}", filename); sourceDocumentRepository.delete(document); - log.info("✅ [DELETE] Source document record deleted: {}", filename); + log.info("[DELETE] Source document record deleted: {}", filename); long duration = System.currentTimeMillis() - startTime; - log.info("🎉 [DELETE] Document deletion completed successfully: id={}, filename={}, duration={}ms", + log.info("[DELETE] Document deletion completed successfully: id={}, filename={}, duration={}ms", documentId, filename, duration); } catch (Exception e) { long duration = System.currentTimeMillis() - startTime; - log.error("❌ [DELETE] Document deletion failed: id={}, filename={}, duration={}ms, error={}", + log.error("[DELETE] Document deletion failed: id={}, filename={}, duration={}ms, error={}", documentId, filename, duration, e.getMessage(), e); // Try to revert status if possible @@ -435,10 +435,10 @@ public void deleteDocument(UUID documentId) { if (updatedDoc != null) { updatedDoc.updateIngestionStatusToError("Deletion failed: " + e.getMessage()); sourceDocumentRepository.save(updatedDoc); - log.info("🔄 [DELETE] Status reverted to ERROR after deletion failure: {}", filename); + log.info("[DELETE] Status reverted to ERROR after deletion failure: {}", filename); } } catch (Exception revertEx) { - log.error("❌ [DELETE] Failed to revert status after deletion failure: {}", filename, revertEx); + log.error("[DELETE] Failed to revert status after deletion failure: {}", filename, revertEx); } throw new BusinessException(ErrorCode.DELETION_PIPELINE_FAILED, @@ -482,7 +482,7 @@ public String getDocumentStoragePath(UUID documentId) { */ private void deleteFromElasticsearch(UUID documentId) { try { - log.debug("🔍 [ELASTICSEARCH] Starting deletion for document: {}", documentId); + log.debug("[ELASTICSEARCH] Starting deletion for document: {}", documentId); // Create delete-by-query request String deleteQuery = String.format( @@ -490,14 +490,14 @@ private void deleteFromElasticsearch(UUID documentId) { documentId.toString() ); - log.debug("🔍 [ELASTICSEARCH] Delete query: {}", deleteQuery); + log.debug(" [ELASTICSEARCH] Delete query: {}", deleteQuery); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); HttpEntity requestEntity = new HttpEntity<>(deleteQuery, headers); String deleteUrl = String.format("%s/%s/_delete_by_query", elasticsearchUrl, indexName); - log.debug("🔍 [ELASTICSEARCH] Delete URL: {}", deleteUrl); + log.debug(" [ELASTICSEARCH] Delete URL: {}", deleteUrl); ResponseEntity response = restTemplate.exchange( deleteUrl, @@ -508,10 +508,10 @@ private void deleteFromElasticsearch(UUID documentId) { if (response.getStatusCode().is2xxSuccessful()) { String responseBody = response.getBody(); - log.info("✅ [ELASTICSEARCH] Document chunks deleted successfully: documentId={}, response={}", + log.info(" [ELASTICSEARCH] Document chunks deleted successfully: documentId={}, response={}", documentId, responseBody); } else { - log.warn("⚠️ [ELASTICSEARCH] Delete operation returned non-2xx status: documentId={}, status={}, response={}", + log.warn(" [ELASTICSEARCH] Delete operation returned non-2xx status: documentId={}, status={}, response={}", documentId, response.getStatusCode(), response.getBody()); } @@ -519,9 +519,9 @@ private void deleteFromElasticsearch(UUID documentId) { // Log the error but don't fail the entire deletion process // This handles cases where the document was never indexed (ERROR status) if (e.getMessage() != null && e.getMessage().contains("index_not_found_exception")) { - log.info("📝 [ELASTICSEARCH] Index not found - document was likely never indexed: documentId={}", documentId); + log.info(" [ELASTICSEARCH] Index not found - document was likely never indexed: documentId={}", documentId); } else { - log.warn("⚠️ [ELASTICSEARCH] Failed to delete from Elasticsearch (continuing deletion): documentId={}, error={}", + log.warn(" [ELASTICSEARCH] Failed to delete from Elasticsearch (continuing deletion): documentId={}, error={}", documentId, e.getMessage(), e); } } @@ -535,17 +535,17 @@ private void deleteFromElasticsearch(UUID documentId) { */ private int deleteChunksFromPostgreSQL(UUID documentId) { try { - log.debug("💾 [POSTGRESQL] Starting chunk deletion for document: {}", documentId); + log.debug(" [POSTGRESQL] Starting chunk deletion for document: {}", documentId); int deletedChunks = documentChunkRepository.deleteBySourceDocumentId(documentId); - log.info("✅ [POSTGRESQL] Chunks deleted successfully: documentId={}, deletedCount={}", + log.info(" [POSTGRESQL] Chunks deleted successfully: documentId={}, deletedCount={}", documentId, deletedChunks); return deletedChunks; } catch (Exception e) { - log.error("❌ [POSTGRESQL] Failed to delete chunks: documentId={}, error={}", + log.error(" [POSTGRESQL] Failed to delete chunks: documentId={}, error={}", documentId, e.getMessage(), e); throw new BusinessException(ErrorCode.DATABASE_ERROR, "Failed to delete document chunks: " + e.getMessage()); @@ -577,13 +577,13 @@ private void validateFile(MultipartFile file) { // Check if the resolved content type is supported if (!ALLOWED_CANONICAL_CONTENT_TYPES.contains(resolvedContentType)) { - log.debug("❌ [UPLOAD] Unsupported content type: filename={}, original={}, resolved={}", + log.debug(" [UPLOAD] Unsupported content type: filename={}, original={}, resolved={}", filename, file.getContentType(), resolvedContentType); throw new BusinessException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported file type. Supported types: PDF, Markdown, and plain text files."); } - log.debug("✅ [UPLOAD] File validation passed: filename={}, resolved_type={}", filename, resolvedContentType); + log.debug(" [UPLOAD] File validation passed: filename={}, resolved_type={}", filename, resolvedContentType); } /** @@ -603,7 +603,7 @@ private String calculateFileChecksum(MultipartFile file) { return sb.toString(); } catch (NoSuchAlgorithmException | IOException e) { - log.error("❌ [UPLOAD] Failed to calculate file checksum: filename={}, error={}", + log.error(" [UPLOAD] Failed to calculate file checksum: filename={}, error={}", file.getOriginalFilename(), e.getMessage(), e); throw new BusinessException(ErrorCode.FILE_UPLOAD_FAILED, "Failed to calculate file checksum: " + e.getMessage()); diff --git a/core/src/main/java/com/opencontext/service/IndexingService.java b/core/src/main/java/com/opencontext/service/IndexingService.java index 75edbd3..06cbbdf 100644 --- a/core/src/main/java/com/opencontext/service/IndexingService.java +++ b/core/src/main/java/com/opencontext/service/IndexingService.java @@ -21,9 +21,6 @@ import java.util.Arrays; /** - * 임베딩된 청크를 Elasticsearch와 PostgreSQL에 저장하는 서비스. - * - * Elasticsearch에는 검색을 위한 벡터와 메타데이터를, * Service for storing embedded chunks in Elasticsearch and PostgreSQL. * * Stores vectors and metadata in Elasticsearch for search, @@ -47,10 +44,10 @@ public class IndexingService { private String indexName; /** - * 임베딩된 청크들을 Elasticsearch와 PostgreSQL에 저장합니다. + * Stores embedded chunks in Elasticsearch and PostgreSQL. * - * @param documentId 문서 ID - * @param embeddedChunks 저장할 임베딩된 청크 목록 + * @param documentId Document ID + * @param embeddedChunks List of embedded chunks to store */ public void indexChunks(UUID documentId, List embeddedChunks) { long startTime = System.currentTimeMillis(); @@ -59,12 +56,12 @@ public void indexChunks(UUID documentId, List embeddedChunks) { log.info("📎 [INDEXING] Starting chunk indexing: documentId={}, chunks={}", documentId, totalChunks); try { - // Step 1: Elasticsearch에 벡터 데이터 저장 + // Step 1: Store vector data in Elasticsearch log.debug("🔍 [INDEXING] Step 1/2: Indexing to Elasticsearch: chunks={}", totalChunks); bulkIndexToElasticsearch(embeddedChunks); log.info("✅ [INDEXING] Elasticsearch indexing completed: documentId={}, chunks={}", documentId, totalChunks); - // Step 2: PostgreSQL에 계층 구조 정보 저장 + // Step 2: Store hierarchical structure information in PostgreSQL log.debug("💾 [INDEXING] Step 2/2: Saving hierarchy to PostgreSQL: chunks={}", totalChunks); int savedChunks = saveChunkHierarchyToPostgreSQL(documentId, embeddedChunks); log.info("✅ [INDEXING] PostgreSQL hierarchy saved: documentId={}, savedChunks={}", documentId, savedChunks); @@ -83,7 +80,7 @@ public void indexChunks(UUID documentId, List embeddedChunks) { } /** - * Elasticsearch에 청크들을 벌크 인덱싱합니다. + * Bulk index chunks to Elasticsearch. */ private void bulkIndexToElasticsearch(List chunks) { long esStartTime = System.currentTimeMillis(); @@ -92,37 +89,37 @@ private void bulkIndexToElasticsearch(List chunks) { log.debug("🔍 [ELASTICSEARCH] Starting bulk indexing: chunks={}, index={}", totalChunks, indexName); try { - // 벌크 요청 바디 생성 + // Create bulk request body log.debug("📦 [ELASTICSEARCH] Building bulk request body: chunks={}", totalChunks); StringBuilder bulkBody = new StringBuilder(); for (StructuredChunk chunk : chunks) { - // 인덱스 메타데이터 + // Index metadata Map indexMeta = Map.of( "index", Map.of("_id", chunk.getChunkId()) ); try { bulkBody.append(objectMapper.writeValueAsString(indexMeta)).append("\n"); - // 문서 데이터 + // Document data Map doc = createElasticsearchDocumentPRD(chunk); bulkBody.append(objectMapper.writeValueAsString(doc)).append("\n"); } catch (Exception jsonException) { - log.error("JSON 직렬화 실패, 청크 건너뛰기: chunkId={}, error={}", + log.error("JSON serialization failed, skipping chunk: chunkId={}, error={}", chunk.getChunkId(), jsonException.getMessage()); - continue; // 해당 청크는 건너뛰고 계속 진행 + continue; // Skip this chunk and continue processing } } - // HTTP 헤더 설정 (UTF-8 명시) + // Set HTTP headers (specify UTF-8) HttpHeaders headers = new HttpHeaders(); headers.setContentType(new MediaType("application", "x-ndjson", StandardCharsets.UTF_8)); - // 본문을 UTF-8 바이트로 전송하여 한글이 '?'로 치환되는 문제 방지 + // Send body as UTF-8 bytes to prevent Korean characters from being replaced with '?' byte[] requestBytes = bulkBody.toString().getBytes(StandardCharsets.UTF_8); HttpEntity requestEntity = new HttpEntity<>(requestBytes, headers); - // 벌크 API 호출 + // Call bulk API ResponseEntity> response = restTemplate.exchange( elasticsearchUrl + "/" + indexName + "/_bulk", HttpMethod.POST, @@ -135,10 +132,10 @@ private void bulkIndexToElasticsearch(List chunks) { "Elasticsearch bulk indexing failed"); } - // 벌크 응답에서 오류 확인 + // Check for errors in bulk response Map responseBody = response.getBody(); if (responseBody != null && Boolean.TRUE.equals(responseBody.get("errors"))) { - // 첫 번째 에러의 상세 정보 추출 + // Extract detailed information from the first error List> items = (List>) responseBody.get("items"); if (items != null && !items.isEmpty()) { Map firstItem = items.get(0); @@ -169,7 +166,7 @@ private void bulkIndexToElasticsearch(List chunks) { } /** - * PostgreSQL에 청크 계층 구조 정보를 저장합니다. + * Save chunk hierarchy information to PostgreSQL. */ private int saveChunkHierarchyToPostgreSQL(UUID documentId, List chunks) { long pgStartTime = System.currentTimeMillis(); @@ -179,7 +176,7 @@ private int saveChunkHierarchyToPostgreSQL(UUID documentId, List new BusinessException(ErrorCode.DATABASE_ERROR, "SourceDocument not found: " + documentId)); @@ -206,13 +203,13 @@ private int saveChunkHierarchyToPostgreSQL(UUID documentId, List savedChunks = documentChunkRepository.saveAll(documentChunks); long saveDuration = System.currentTimeMillis() - saveStartTime; @@ -233,26 +230,26 @@ private int saveChunkHierarchyToPostgreSQL(UUID documentId, List createElasticsearchDocumentPRD(StructuredChunk chunk) { Map doc = new HashMap<>(); - // 루트 필드 (camelCase) + // Root fields (camelCase) doc.put("chunkId", chunk.getChunkId()); doc.put("sourceDocumentId", chunk.getDocumentId()); doc.put("content", sanitizeContent(chunk.getContent())); doc.put("embedding", chunk.getEmbedding()); - doc.put("indexedAt", java.time.Instant.now().toString()); // ISO 문자열 + doc.put("indexedAt", java.time.Instant.now().toString()); // ISO string - // metadata 구조 + // metadata structure Map metadata = new HashMap<>(); metadata.put("title", chunk.getTitle()); metadata.put("hierarchyLevel", chunk.getHierarchyLevel()); - metadata.put("sequenceInDocument", 0); // 기본값 - metadata.put("language", "ko"); // 한국어 기본값 + metadata.put("sequenceInDocument", 0); // default value + metadata.put("language", "ko"); // Korean default value - // 실제 파일 타입을 SourceDocument에서 조회하여 반영 + // Reflect actual file type by querying from SourceDocument String resolvedFileType = "UNKNOWN"; try { UUID srcId = UUID.fromString(chunk.getDocumentId()); @@ -264,8 +261,8 @@ private Map createElasticsearchDocumentPRD(StructuredChunk chunk } metadata.put("fileType", resolvedFileType); - // breadcrumbs 처리 (기본값: 빈 배열) - metadata.put("breadcrumbs", Arrays.asList()); // 빈 배열 기본값 + // Handle breadcrumbs (default: empty array) + metadata.put("breadcrumbs", Arrays.asList()); // empty array default value doc.put("metadata", metadata); @@ -273,7 +270,7 @@ private Map createElasticsearchDocumentPRD(StructuredChunk chunk } /** - * content 필드의 Java 코드나 특수문자를 정리하여 JSON 파싱 에러를 방지합니다. + * Cleans up Java code or special characters in content field to prevent JSON parsing errors. */ private String sanitizeContent(String content) { if (content == null || content.trim().isEmpty()) { @@ -281,20 +278,20 @@ private String sanitizeContent(String content) { } return content - // Java 코드 관련 정리 - .replaceAll("\\{[^}]*\\}", "") // 중괄호 내용 제거 - .replaceAll("\\[.*?\\]", "") // 대괄호 내용 제거 - .replaceAll("\\b(int|String|void|public|private|static|final|class|interface|extends|implements)\\b", "") // Java 키워드 제거 - .replaceAll("\\b(if|else|for|while|switch|case|break|continue|return|throw|try|catch|finally)\\b", "") // Java 제어문 제거 - .replaceAll("\\b(new|this|super|null|true|false)\\b", "") // Java 리터럴 제거 + // Java code related cleanup + .replaceAll("\\{[^}]*\\}", "") // Remove curly brace content + .replaceAll("\\[.*?\\]", "") // Remove square bracket content + .replaceAll("\\b(int|String|void|public|private|static|final|class|interface|extends|implements)\\b", "") // Remove Java keywords + .replaceAll("\\b(if|else|for|while|switch|case|break|continue|return|throw|try|catch|finally)\\b", "") // Remove Java control statements + .replaceAll("\\b(new|this|super|null|true|false)\\b", "") // Remove Java literals - // 특수문자 정리 - .replaceAll("\\s+", " ") // 연속 공백을 단일 공백으로 - .replaceAll("[\\r\\n\\t]+", " ") // 줄바꿈, 탭을 공백으로 - .replaceAll("\\s*[;=+\\-*/%<>!&|^~]\\s*", " ") // 연산자 주변 공백 정리 + // Special character cleanup + .replaceAll("\\s+", " ") // Replace consecutive spaces with single space + .replaceAll("[\\r\\n\\t]+", " ") // Replace newlines and tabs with space + .replaceAll("\\s*[;=+\\-*/%<>!&|^~]\\s*", " ") // Clean up spaces around operators - // 기타 정리 - .replaceAll("\\s+", " ") // 다시 연속 공백 정리 + // Other cleanup + .replaceAll("\\s+", " ") // Clean up consecutive spaces again .trim(); } diff --git a/core/src/main/java/com/opencontext/service/SearchService.java b/core/src/main/java/com/opencontext/service/SearchService.java index d6d8f11..af5a794 100644 --- a/core/src/main/java/com/opencontext/service/SearchService.java +++ b/core/src/main/java/com/opencontext/service/SearchService.java @@ -17,8 +17,8 @@ import java.util.*; /** - * Elasticsearch 하이브리드 검색 서비스 - * BM25 키워드 검색과 벡터 유사도 검색을 결합하여 최적의 검색 결과 제공 + * Elasticsearch hybrid search service + * Combines BM25 keyword search and vector similarity search to provide optimal search results */ @Slf4j @Service @@ -44,36 +44,36 @@ public class SearchService { private double vectorWeight; /** - * 하이브리드 검색 실행 - 키워드 검색과 의미 검색을 결합 + * Execute hybrid search - combines keyword search and semantic search * - * @param query 검색어 - * @param topK 반환할 최대 결과 수 - * @return 관련도 순으로 정렬된 검색 결과 목록 + * @param query Search query + * @param topK Maximum number of results to return + * @return List of search results sorted by relevance */ public List search(String query, int topK) { long startTime = System.currentTimeMillis(); - log.info("하이브리드 검색 시작: query='{}', topK={}", query, topK); + log.info("Starting hybrid search: query='{}', topK={}", query, topK); try { - // 1단계: 검색어를 임베딩 벡터로 변환 (float 타입으로 ES 호환성 확보) + // Step 1: Convert search query to embedding vector (float type for ES compatibility) List queryEmbedding = generateQueryEmbedding(query); - // 2단계: Elasticsearch 하이브리드 쿼리 실행 + // Step 2: Execute Elasticsearch hybrid query Map searchResponse = executeElasticsearchQuery(query, queryEmbedding, topK); - // 3단계: 검색 결과를 DTO로 변환 + // Step 3: Convert search results to DTO List results = parseSearchResults(searchResponse); long duration = System.currentTimeMillis() - startTime; - log.info("하이브리드 검색 완료: query='{}', 결과수={}, 소요시간={}ms", + log.info("Hybrid search completed: query='{}', resultCount={}, duration={}ms", query, results.size(), duration); return results; } catch (Exception e) { long duration = System.currentTimeMillis() - startTime; - log.error("하이브리드 검색 실패: query='{}', 소요시간={}ms, 오류={}", + log.error("Hybrid search failed: query='{}', duration={}ms, error={}", query, duration, e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Search operation failed: " + e.getMessage()); @@ -81,38 +81,38 @@ public List search(String query, int topK) { } /** - * 검색어를 임베딩 벡터로 변환 - * ES cosineSimilarity 함수 호환을 위해 List 타입 사용 + * Convert search query to embedding vector + * Uses List type for ES cosineSimilarity function compatibility */ private List generateQueryEmbedding(String query) { - log.debug("쿼리 임베딩 생성: query='{}'", query); + log.debug("Generating query embedding: query='{}'", query); try { TextSegment textSegment = TextSegment.from(query); Embedding embedding = embeddingModel.embed(textSegment).content(); - // float 배열을 List로 변환 (ES 호환성) + // Convert float array to List (ES compatibility) List embeddingVector = new ArrayList<>(); float[] vector = embedding.vector(); for (float value : vector) { embeddingVector.add(value); } - log.debug("쿼리 임베딩 생성 완료: 차원수={}", embedding.dimension()); + log.debug("Query embedding generation completed: dimensions={}", embedding.dimension()); return embeddingVector; } catch (Exception e) { - log.error("쿼리 임베딩 생성 실패: query='{}', 오류={}", query, e.getMessage(), e); + log.error("Query embedding generation failed: query='{}', error={}", query, e.getMessage(), e); throw new BusinessException(ErrorCode.EMBEDDING_GENERATION_FAILED, "Failed to generate query embedding: " + e.getMessage()); } } /** - * Elasticsearch에 하이브리드 검색 쿼리 실행 + * Execute hybrid search query on Elasticsearch */ private Map executeElasticsearchQuery(String query, List queryEmbedding, int topK) { - log.debug("Elasticsearch 쿼리 실행: topK={}", topK); + log.debug("Executing Elasticsearch query: topK={}", topK); try { Map searchQuery = buildHybridSearchQuery(query, queryEmbedding, topK); @@ -132,23 +132,23 @@ private Map executeElasticsearchQuery(String query, List "Empty response from Elasticsearch"); } - log.debug("검색 쿼리 실행 성공"); + log.debug("Search query execution successful"); return responseBody; } catch (Exception e) { - log.error("검색 쿼리 실행 실패: 오류={}", e.getMessage(), e); + log.error("Search query execution failed: error={}", e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Elasticsearch query execution failed: " + e.getMessage()); } } /** - * BM25 키워드 검색과 벡터 유사도 검색을 결합한 하이브리드 쿼리 구성 - * 각 쿼리를 나란히 배치하여 점수 정상 반영 + * Build hybrid query combining BM25 keyword search and vector similarity search + * Place queries side by side to properly reflect scores */ private Map buildHybridSearchQuery(String query, List queryEmbedding, int topK) { - // BM25 키워드 검색 쿼리 + // BM25 keyword search query Map bm25Query = Map.of( "multi_match", Map.of( "query", query, @@ -159,7 +159,7 @@ private Map buildHybridSearchQuery(String query, List que ) ); - // 벡터 유사도 검색 쿼리 (가중치를 스크립트 내부에서 적용) + // Vector similarity search query (apply weight inside script) Map vectorQuery = Map.of( "script_score", Map.of( "query", Map.of("match_all", Map.of()), @@ -173,14 +173,14 @@ private Map buildHybridSearchQuery(String query, List que ) ); - // 하이브리드 쿼리 (bool.should에 두 쿼리를 나란히 배치) + // Hybrid query (place two queries side by side in bool.should) Map hybridQuery = Map.of( "bool", Map.of( "should", Arrays.asList(bm25Query, vectorQuery) ) ); - // 최종 검색 쿼리 + // Final search query return Map.of( "size", topK, "query", hybridQuery, @@ -195,26 +195,26 @@ private Map buildHybridSearchQuery(String query, List que } /** - * Elasticsearch 응답을 SearchResultItem 목록으로 변환 - * 응답 내 최대 점수 대비 상대적 정규화 적용 + * Convert Elasticsearch response to SearchResultItem list + * Apply relative normalization against maximum score in response */ private List parseSearchResults(Map response) { - log.debug("검색 결과 파싱 중"); + log.debug("Parsing search results"); try { Map hits = (Map) response.get("hits"); if (hits == null) { - log.warn("Elasticsearch 응답에 'hits' 필드가 없음"); + log.warn("Elasticsearch response missing 'hits' field"); return Collections.emptyList(); } List> hitList = (List>) hits.get("hits"); if (hitList == null || hitList.isEmpty()) { - log.info("검색 결과가 없음"); + log.info("No search results found"); return Collections.emptyList(); } - // 응답 내 최대 점수 계산 (상대적 정규화를 위함) + // Calculate maximum score in response (for relative normalization) double maxScore = hitList.stream() .mapToDouble(hit -> ((Number) hit.get("_score")).doubleValue()) .max() @@ -229,23 +229,23 @@ private List parseSearchResults(Map response) results.add(item); } } catch (Exception e) { - log.warn("검색 결과 파싱 실패, 건너뛰기: {}", e.getMessage()); - // 개별 hit 파싱 실패는 전체 검색을 중단하지 않음 + log.warn("Search result parsing failed, skipping: {}", e.getMessage()); + // Individual hit parsing failure does not stop the entire search } } - log.debug("검색 결과 파싱 완료: 결과수={}", results.size()); + log.debug("Search result parsing completed: resultCount={}", results.size()); return results; } catch (Exception e) { - log.error("검색 결과 파싱 실패: 오류={}", e.getMessage(), e); + log.error("Search result parsing failed: error={}", e.getMessage(), e); throw new BusinessException(ErrorCode.ELASTICSEARCH_ERROR, "Failed to parse search results: " + e.getMessage()); } } /** - * 개별 검색 결과를 SearchResultItem으로 변환 + * Convert individual search result to SearchResultItem */ private SearchResultItem parseSearchHit(Map hit, double maxScore) { Map source = (Map) hit.get("_source"); @@ -253,30 +253,30 @@ private SearchResultItem parseSearchHit(Map hit, double maxScore return null; } - // PRD 스키마에 따른 필드 추출 + // Extract fields according to PRD schema String chunkId = (String) source.get("chunkId"); String content = (String) source.get("content"); Double score = ((Number) hit.get("_score")).doubleValue(); - // PRD 스키마: title은 metadata.title에 위치 + // PRD schema: title is located in metadata.title String title = extractTitle(source); - // PRD 정책에 따른 스니펫 생성 + // Generate snippet according to PRD policy String snippet = generateSnippet(content); - // 응답 내 최대 점수 대비 상대적 정규화 + // Relative normalization against maximum score in response double relevanceScore = normalizeScore(score, maxScore); return SearchResultItem.builder() .chunkId(chunkId) - .title(title != null ? title : "제목 없음") + .title(title != null ? title : "No Title") .snippet(snippet) .relevanceScore(relevanceScore) .build(); } /** - * 스키마에서 제목 추출 (metadata.title) + * Extract title from schema (metadata.title) */ private String extractTitle(Map source) { Map metadata = (Map) source.get("metadata"); @@ -287,14 +287,14 @@ private String extractTitle(Map source) { } /** - * 스니펫 생성 - * - 기본 길이: 50자 - * - 50자 초과 시: 앞 50자 + "..." (항상 추가) - * - 50자 미만 시: 원본 그대로 (... 생략) + * Generate snippet + * - Default length: 50 characters + * - If over 50 characters: first 50 characters + "..." (always added) + * - If under 50 characters: original as-is (no ...) */ private String generateSnippet(String content) { if (content == null || content.trim().isEmpty()) { - return "내용이 없습니다"; + return "No content available"; } String cleanContent = content.trim(); @@ -307,7 +307,7 @@ private String generateSnippet(String content) { } /** - * 점수 정규화 - 응답 내 최대 점수 대비 상대적 비율로 계산 + * Score normalization - calculate relative ratio against maximum score in response */ private double normalizeScore(double score, double maxScore) { if (maxScore <= 0) {