Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,12 @@ public interface DocsSourceController {
**Authentication Required:** X-API-KEY header must be provided.

**Supported File Types:**
- PDF documents (application/pdf)
- Markdown files (text/markdown)
- Plain text files (text/plain)
- PDF documents (application/pdf, .pdf files)
- Markdown files (text/markdown, .md or .markdown files)
- Plain text files (text/plain, .txt files)

**Note:** The system automatically detects file types based on both the Content-Type header
and file extension. the system will properly recognize Markdown files regardless of how the browser sends the Content-Type.

Copilot AI Aug 18, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing capitalization: 'the system' should be 'The system' as it starts a new sentence.

Suggested change
and file extension. the system will properly recognize Markdown files regardless of how the browser sends the Content-Type.
and file extension. The system will properly recognize Markdown files regardless of how the browser sends the Content-Type.

Copilot uses AI. Check for mistakes.

**File Size Limit:** 100MB

Expand All @@ -60,9 +63,9 @@ public interface DocsSourceController {
required = true,
content = @Content(
mediaType = MediaType.MULTIPART_FORM_DATA_VALUE,
examples = @ExampleObject(
name = "Upload a Markdown file",
value = "(Use the file picker UI to attach a .md file)"
encoding = @io.swagger.v3.oas.annotations.media.Encoding(
name = "file",
contentType = "application/octet-stream"
)
)
)
Expand Down Expand Up @@ -151,8 +154,11 @@ public interface DocsSourceController {
@ApiResponse(responseCode = "415", description = "Unsupported Media Type - invalid file format")
})
ResponseEntity<CommonResponse<SourceUploadResponse>> uploadFile(
@Parameter(description = "File to upload (PDF, Markdown, or plain text)", required = true)
@RequestParam("file") @NotNull MultipartFile file
@Parameter(
description = "File to upload (PDF, Markdown, or plain text)",
required = true
)
@RequestParam("file") MultipartFile file
);

/**
Expand Down
73 changes: 59 additions & 14 deletions core/src/main/java/com/opencontext/service/FileStorageService.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ public class FileStorageService {
@Value("${app.elasticsearch.index:document-chunks}")
private String indexName;

// Supported file types for document processing
private static final List<String> SUPPORTED_CONTENT_TYPES = List.of(
// Supported file types for document processing (canonical)
private static final List<String> ALLOWED_CANONICAL_CONTENT_TYPES = List.of(
"application/pdf",
"text/markdown",
"text/plain"
Expand All @@ -69,7 +69,7 @@ public class FileStorageService {
public SourceDocument uploadFileWithMetadata(MultipartFile file) {
String filename = file.getOriginalFilename();
long fileSize = file.getSize();
String contentType = file.getContentType();
String contentType = resolveContentType(file);

log.info("📤 [UPLOAD] Starting file upload with metadata: filename={}, size={} bytes, contentType={}",
filename, fileSize, contentType);
Expand Down Expand Up @@ -98,15 +98,15 @@ public SourceDocument uploadFileWithMetadata(MultipartFile file) {
try {
// Upload file to MinIO
log.debug("☁️ [UPLOAD] Step 4/5: Uploading file to MinIO: {}", filename);
String objectKey = uploadFile(file);
String objectKey = uploadFile(file, contentType);
log.info("✅ [UPLOAD] File uploaded to MinIO successfully: {} -> {}", filename, objectKey);

// Create SourceDocument entity
log.debug("💾 [UPLOAD] Step 5/5: Creating database record: {}", filename);
SourceDocument sourceDocument = SourceDocument.builder()
.originalFilename(file.getOriginalFilename())
.fileStoragePath(objectKey)
.fileType(determineFileType(file.getContentType()))
.fileType(determineFileType(contentType))
.fileSize(file.getSize())
.fileChecksum(fileChecksum)
.ingestionStatus(IngestionStatus.PENDING)
Expand Down Expand Up @@ -139,7 +139,7 @@ public SourceDocument uploadFileWithMetadata(MultipartFile file) {
* @param file the multipart file to upload
* @return the object key where the file was stored
*/
public String uploadFile(MultipartFile file) {
public String uploadFile(MultipartFile file, String resolvedContentType) {
try {
// Ensure bucket exists
ensureBucketExists();
Expand All @@ -152,7 +152,7 @@ public String uploadFile(MultipartFile file) {
.bucket(minioConfig.getBucketName())
.object(objectKey)
.stream(file.getInputStream(), file.getSize(), -1)
.contentType(file.getContentType())
.contentType(resolvedContentType)
.build();

minioClient.putObject(putObjectArgs);
Expand Down Expand Up @@ -562,18 +562,23 @@ private void validateFile(MultipartFile file) {
"File size exceeds maximum limit of 100MB");
}

String contentType = file.getContentType();
if (contentType == null || !SUPPORTED_CONTENT_TYPES.contains(contentType)) {
throw new BusinessException(ErrorCode.UNSUPPORTED_MEDIA_TYPE,
"Unsupported file type. Supported types: PDF, Markdown, and plain text files.");
}

String filename = file.getOriginalFilename();
if (filename == null || filename.trim().isEmpty()) {
throw new BusinessException(ErrorCode.INVALID_REQUEST, "Filename cannot be empty");
}

log.debug("✅ [UPLOAD] File validation passed: filename={}", filename);
// Resolve canonical content type from both content-type header and filename extension
String resolvedContentType = resolveContentType(file);

// Check if the resolved content type is supported
if (!ALLOWED_CANONICAL_CONTENT_TYPES.contains(resolvedContentType)) {
log.debug("❌ [UPLOAD] Unsupported content type: filename={}, original={}, resolved={}",
filename, file.getContentType(), resolvedContentType);
throw new BusinessException(ErrorCode.UNSUPPORTED_MEDIA_TYPE,
"Unsupported file type. Supported types: PDF, Markdown, and plain text files.");
}

log.debug("✅ [UPLOAD] File validation passed: filename={}, resolved_type={}", filename, resolvedContentType);
}

/**
Expand Down Expand Up @@ -612,6 +617,46 @@ private String determineFileType(String contentType) {
};
}

/**
* Resolves the effective content type for the uploaded file.
* If the inbound content type is null or generic (e.g., application/octet-stream),
* infer from the filename extension and normalize to a canonical, allowed type.
*/
private String resolveContentType(MultipartFile file) {
String inbound = file.getContentType();
// If clearly an allowed canonical type, return as-is
if (ALLOWED_CANONICAL_CONTENT_TYPES.contains(inbound)) {
return inbound;
}

// Try to infer from file extension for generic or alternate types
String filename = file.getOriginalFilename();
String ext = (filename == null) ? null : getFileExtension(filename).toLowerCase();

if (ext != null) {
return switch (ext) {
case "pdf" -> "application/pdf";
case "md", "markdown" -> "text/markdown";
case "txt" -> "text/plain";
default -> (inbound != null ? inbound : "application/octet-stream");
};
}

// Fallback to inbound or generic
return inbound != null ? inbound : "application/octet-stream";

Copilot AI Aug 18, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The fallback logic returns the original inbound content type for unknown extensions, which could bypass validation. Consider returning a specific unsupported type or logging this case, as it may allow unsupported file types to pass through if they have an unrecognized extension but valid inbound content type.

Suggested change
return inbound != null ? inbound : "application/octet-stream";
default -> {
log.warn("Unknown file extension '{}', treating as unsupported type. Inbound content type: '{}'", ext, inbound);
yield "unsupported/unknown";
}
};
}
// Fallback to unsupported type if extension is missing
log.warn("Unable to determine file extension for filename '{}', treating as unsupported type. Inbound content type: '{}'", filename, inbound);
return "unsupported/unknown";

Copilot uses AI. Check for mistakes.
}

/**
* Returns the extension without the leading dot. Example: "README.md" -> "md".
*/
private String getFileExtension(String filename) {
int lastDot = filename.lastIndexOf('.');
if (lastDot == -1 || lastDot == filename.length() - 1) {

Copilot AI Aug 18, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The condition lastDot == filename.length() - 1 will incorrectly return an empty string for files ending with a dot (e.g., 'file.'). This should return the empty string only when there's no extension, but a file ending with a dot should be handled differently or treated as having no valid extension.

Suggested change
if (lastDot == -1 || lastDot == filename.length() - 1) {
// No dot, or dot is the first character (dotfile), or dot is the last character (ends with dot)
if (lastDot <= 0 || lastDot == filename.length() - 1) {

Copilot uses AI. Check for mistakes.
return "";
}
return filename.substring(lastDot + 1);
}

/**
* Converts SourceDocument entity to DTO.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public List<SearchResultItem> search(String query, int topK) {
Map<String, Object> searchResponse = executeElasticsearchQuery(query, queryEmbedding, topK);

// 3단계: 검색 결과를 DTO로 변환
List<SearchResultItem> results = parseSearchResults(searchResponse);
List<SearchResultItem> results = parseSearchResults(searchResponse);

long duration = System.currentTimeMillis() - startTime;
log.info("하이브리드 검색 완료: query='{}', 결과수={}, 소요시간={}ms",
Expand Down