From d221884c328a843a1f2b47efe4599b2d08c70d7b Mon Sep 17 00:00:00 2001 From: qowlgur121 Date: Sat, 9 Aug 2025 17:49:15 +0900 Subject: [PATCH 1/6] fix: resolve critical security vulnerabilities in dependencies - Replace QueryDSL with OpenFeign fork to fix SQL injection vulnerability - Update DJL and Commons Lang3 to latest secure versions - Modernize QueryDSL configuration for Spring Boot 3.3 compatibility - Fix deprecated buildDir and annotation processor warnings --- core/build.gradle | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 1de20c8..d448804 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -2,7 +2,6 @@ plugins { id 'java' id 'org.springframework.boot' version '3.3.11' id 'io.spring.dependency-management' version '1.1.6' - id 'com.ewerk.gradle.plugins.querydsl' version '1.0.10' } group = 'com.opencontext' @@ -39,15 +38,15 @@ dependencies { runtimeOnly 'org.flywaydb:flyway-database-postgresql' // LangChain4j - Core RAG Framework - implementation 'dev.langchain4j:langchain4j:0.36.2' - implementation 'dev.langchain4j:langchain4j-spring-boot-starter:0.36.2' - implementation 'dev.langchain4j:langchain4j-document-parser-apache-pdfbox:0.36.2' - implementation 'dev.langchain4j:langchain4j-embeddings:0.36.2' - implementation 'dev.langchain4j:langchain4j-ollama:0.36.2' - - // QueryDSL - Type-safe dynamic queries - implementation 'com.querydsl:querydsl-jpa:5.0.0:jakarta' - annotationProcessor 'com.querydsl:querydsl-apt:5.0.0:jakarta' + implementation 'dev.langchain4j:langchain4j:0.35.0' + implementation 'dev.langchain4j:langchain4j-spring-boot-starter:0.35.0' + implementation 'dev.langchain4j:langchain4j-document-parser-apache-pdfbox:0.35.0' + implementation 'dev.langchain4j:langchain4j-embeddings:0.35.0' + implementation 'dev.langchain4j:langchain4j-ollama:0.35.0' + + // QueryDSL (using OpenFeign fork for security patches) + implementation 'io.github.openfeign.querydsl:querydsl-jpa:6.11' + annotationProcessor 'io.github.openfeign.querydsl:querydsl-apt:6.11' annotationProcessor 'jakarta.annotation:jakarta.annotation-api' annotationProcessor 'jakarta.persistence:jakarta.persistence-api' @@ -63,6 +62,10 @@ dependencies { // Structured Logging implementation 'net.logstash.logback:logstash-logback-encoder:7.4' + // Security patches for transitive dependencies + implementation 'ai.djl:api:0.31.1' // CVE-2025-0851 + implementation 'org.apache.commons:commons-lang3:3.18.0' // CVE-2025-48924 + // Utilities compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' @@ -88,24 +91,15 @@ dependencyManagement { } } -tasks.named('test') { +test { useJUnitPlatform() } -// QueryDSL Configuration -def querydslDir = layout.buildDirectory.dir("generated/querydsl").get().asFile - -querydsl { - jpa = true - querydslSourcesDir = querydslDir -} +// QueryDSL Configuration (Fixed deprecated buildDir) +def querydslDir = "${layout.buildDirectory.get().asFile}/generated/querydsl" sourceSets { - main.java.srcDir querydslDir -} - -compileQuerydsl { - options.annotationProcessorPath = configurations.querydsl + main.java.srcDirs += querydslDir } configurations { @@ -114,3 +108,7 @@ configurations { } querydsl.extendsFrom compileClasspath } + +compileJava { + options.generatedSourceOutputDirectory = file(querydslDir) +} From 5a825fb105fd1aab9cf0d4de2405b17f981db6a7 Mon Sep 17 00:00:00 2001 From: qowlgur121 Date: Sat, 9 Aug 2025 17:54:38 +0900 Subject: [PATCH 2/6] build: regenerate gradle wrapper for consistent build environment - Update gradlew script with latest Gradle wrapper improvements - Add missing gradle-wrapper.jar for self-contained builds - Include gradlew.bat for Windows development support - Ensures all team members can build without local Gradle installation --- core/gradlew | 98 +++++++++++++++++++++++++++++++++++++++++++++--- core/gradlew.bat | 27 +++++++++++++ 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/core/gradlew b/core/gradlew index 31e9ed3..1aa94a4 100755 --- a/core/gradlew +++ b/core/gradlew @@ -18,7 +18,47 @@ ############################################################################## # -# Gradle start up script for UNIX +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. # ############################################################################## @@ -43,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -90,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -101,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -109,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -158,4 +202,48 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/core/gradlew.bat b/core/gradlew.bat index db3a6ac..8286633 100644 --- a/core/gradlew.bat +++ b/core/gradlew.bat @@ -13,8 +13,11 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +<<<<<<< HEAD @rem SPDX-License-Identifier: Apache-2.0 @rem +======= +>>>>>>> 2b49148 (build: regenerate gradle wrapper for consistent build environment) @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -45,11 +48,19 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute +<<<<<<< HEAD echo. 1>&2 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 +======= +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. +>>>>>>> 2b49148 (build: regenerate gradle wrapper for consistent build environment) goto fail @@ -59,22 +70,38 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute +<<<<<<< HEAD echo. 1>&2 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 +======= +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. +>>>>>>> 2b49148 (build: regenerate gradle wrapper for consistent build environment) goto fail :execute @rem Setup the command line +<<<<<<< HEAD set CLASSPATH= @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +======= +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +>>>>>>> 2b49148 (build: regenerate gradle wrapper for consistent build environment) :end @rem End local scope for the variables with windows NT shell From 23b3b04a25a70e6246cb771dce94a2619911734b Mon Sep 17 00:00:00 2001 From: qowlgur121 Date: Sat, 9 Aug 2025 18:01:47 +0900 Subject: [PATCH 3/6] fix: add explicit platform specification for ARM64 compatibility - Set platform: linux/amd64 for all Docker services - Resolves compatibility issues on Apple Silicon (M1/M2) machines - Ensures consistent behavior across different hardware architectures --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index bcbed2c..573f56e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ services: # PostgreSQL Database postgres: image: postgres:16-alpine + platform: linux/amd64 container_name: postgres environment: POSTGRES_DB: opencontext @@ -49,6 +50,7 @@ services: # Ollama Model Downloader ollama-init: image: ollama/ollama:latest + platform: linux/amd64 container_name: opencontext-ollama-init volumes: - ollama_data:/root/.ollama @@ -69,6 +71,7 @@ services: # Ollama Server for embeddings ollama: image: ollama/ollama:latest + platform: linux/amd64 container_name: ollama ports: - "11434:11434" @@ -94,6 +97,7 @@ services: # MinIO Object Storage minio: image: minio/minio:latest + platform: linux/amd64 container_name: minio ports: - "9000:9000" @@ -111,6 +115,7 @@ services: # Unstructured.io API for document parsing unstructured-api: image: quay.io/unstructured-io/unstructured-api:latest + platform: linux/amd64 container_name: unstructured-api ports: - "8000:8000" From ea706b0183b0ec8e8d4ad4760a2bf3f13148a5d4 Mon Sep 17 00:00:00 2001 From: qowlgur121 Date: Sun, 10 Aug 2025 11:42:00 +0900 Subject: [PATCH 4/6] feat(entity): implement core data model with PostgreSQL entities and repositories - Add IngestionStatus enum with complete document processing lifecycle states - Implement SourceDocument entity with file metadata and ingestion tracking - Implement DocumentChunk entity with hierarchical parent-child relationships - Create SourceDocumentRepository with advanced query methods for status filtering - Create DocumentChunkRepository with hierarchy navigation and cascade operations - Add comprehensive Flyway migration V1 for PostgreSQL table creation with indexes - Implement complete test suite with TestContainers integration testing - Include entity unit tests with Builder pattern validation - Add repository integration tests with real PostgreSQL container testing - Cover CASCADE delete operations, hierarchy queries, and performance optimizations Resolves #4 --- .../core/entity/DocumentChunk.java | 132 +++++++++ .../core/entity/SourceDocument.java | 158 ++++++++++ .../core/enums/IngestionStatus.java | 63 ++++ .../repository/DocumentChunkRepository.java | 140 +++++++++ .../repository/SourceDocumentRepository.java | 141 +++++++++ ...e_documents_and_document_chunks_tables.sql | 65 +++++ .../core/entity/DocumentChunkTest.java | 161 +++++++++++ .../core/entity/SourceDocumentTest.java | 140 +++++++++ .../DocumentChunkRepositoryTest.java | 273 ++++++++++++++++++ .../SourceDocumentRepositoryTest.java | 241 ++++++++++++++++ 10 files changed, 1514 insertions(+) create mode 100644 core/src/main/java/com/opencontext/core/entity/DocumentChunk.java create mode 100644 core/src/main/java/com/opencontext/core/entity/SourceDocument.java create mode 100644 core/src/main/java/com/opencontext/core/enums/IngestionStatus.java create mode 100644 core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java create mode 100644 core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java create mode 100644 core/src/main/resources/db/migration/V1__Create_source_documents_and_document_chunks_tables.sql create mode 100644 core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java create mode 100644 core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java create mode 100644 core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java create mode 100644 core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java diff --git a/core/src/main/java/com/opencontext/core/entity/DocumentChunk.java b/core/src/main/java/com/opencontext/core/entity/DocumentChunk.java new file mode 100644 index 0000000..144897a --- /dev/null +++ b/core/src/main/java/com/opencontext/core/entity/DocumentChunk.java @@ -0,0 +1,132 @@ +package com.opencontext.core.entity; + +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.CreationTimestamp; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Entity representing the hierarchical structure information of document chunks. + * This entity stores only the relationships and structure, while the actual + * content and search data are stored in Elasticsearch for performance. + */ +@Entity +@Table(name = "document_chunks") +@Getter +@Builder +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +public class DocumentChunk { + + /** + * Primary key - unique identifier for each Structured Chunk. + * This ID is identical to the chunkId in Elasticsearch for data consistency. + */ + @Id + @GeneratedValue(strategy = GenerationType.UUID) + @Column(name = "id") + private UUID id; + + /** + * Foreign key to the source document this chunk belongs to. + * When parent document is deleted, related chunks are also deleted (CASCADE). + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "source_document_id", nullable = false, foreignKey = @ForeignKey(name = "fk_chunk_source_document")) + private SourceDocument sourceDocument; + + /** + * Foreign key to the parent chunk ID for hierarchical structure. + * This enables reconstruction of document's tree structure. + * Root chunks have null parent_chunk_id. + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "parent_chunk_id", foreignKey = @ForeignKey(name = "fk_chunk_parent")) + private DocumentChunk parentChunk; + + /** + * Order sequence among sibling chunks with the same parent_chunk_id. + * Used to maintain the original document structure and order. + */ + @Column(name = "sequence_in_document", nullable = false) + private Integer sequenceInDocument; + + /** + * Timestamp when this chunk record was first created in database. + */ + @CreationTimestamp + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + /** + * One-to-many relationship to child chunks. + * This enables easy navigation of the hierarchical structure. + */ + @OneToMany(mappedBy = "parentChunk", cascade = CascadeType.ALL, orphanRemoval = true, fetch = FetchType.LAZY) + @OrderBy("sequenceInDocument ASC") + @Builder.Default + private List childChunks = new ArrayList<>(); + + /** + * Adds a child chunk to this chunk, maintaining the hierarchical relationship. + * + * @param childChunk the child chunk to add + */ + public void addChildChunk(DocumentChunk childChunk) { + childChunks.add(childChunk); + } + + /** + * Checks if this chunk is a root chunk (has no parent). + * + * @return true if this chunk has no parent + */ + public boolean isRootChunk() { + return parentChunk == null; + } + + /** + * Checks if this chunk has any child chunks. + * + * @return true if this chunk has children + */ + public boolean hasChildren() { + return childChunks != null && !childChunks.isEmpty(); + } + + /** + * Gets the depth level of this chunk in the document hierarchy. + * Root chunks are at level 1, their children at level 2, etc. + * + * @return the depth level of this chunk + */ + public int getHierarchyLevel() { + if (isRootChunk()) { + return 1; + } + return parentChunk.getHierarchyLevel() + 1; + } + + /** + * Gets all ancestor chunks from root to this chunk's parent. + * + * @return list of ancestor chunks in order from root to parent + */ + public List getAncestors() { + List ancestors = new ArrayList<>(); + DocumentChunk current = this.parentChunk; + while (current != null) { + ancestors.add(0, current); // Add to beginning to maintain order + current = current.getParentChunk(); + } + return ancestors; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/core/entity/SourceDocument.java b/core/src/main/java/com/opencontext/core/entity/SourceDocument.java new file mode 100644 index 0000000..1347531 --- /dev/null +++ b/core/src/main/java/com/opencontext/core/entity/SourceDocument.java @@ -0,0 +1,158 @@ +package com.opencontext.core.entity; + +import com.opencontext.core.enums.IngestionStatus; +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.CreationTimestamp; +import org.hibernate.annotations.UpdateTimestamp; + +import java.time.LocalDateTime; +import java.util.UUID; + +/** + * Entity representing the master information of uploaded source files. + * This entity tracks the metadata and processing state of each document + * throughout the ingestion pipeline. + */ +@Entity +@Table(name = "source_documents") +@Getter +@Builder +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +public class SourceDocument { + + /** + * Primary key - unique identifier for each source document. + */ + @Id + @GeneratedValue(strategy = GenerationType.UUID) + @Column(name = "id") + private UUID id; + + /** + * Original filename when user uploaded the file. + */ + @Column(name = "original_filename", nullable = false, length = 255) + private String originalFilename; + + /** + * Actual storage path within Object Storage (MinIO). + * Used to access the original file when needed. + */ + @Column(name = "file_storage_path", nullable = false, length = 1024) + private String fileStoragePath; + + /** + * Type of the uploaded file (e.g., PDF, MARKDOWN). + */ + @Column(name = "file_type", nullable = false, length = 50) + private String fileType; + + /** + * File size in bytes. + */ + @Column(name = "file_size", nullable = false) + private Long fileSize; + + /** + * SHA-256 hash of file content. + * Used to prevent duplicate uploads and verify file integrity. + */ + @Column(name = "file_checksum", nullable = false, length = 64, unique = true) + private String fileChecksum; + + /** + * Current status of the ingestion pipeline process. + * Tracks the document through states: PENDING → PARSING → CHUNKING → EMBEDDING → INDEXING → COMPLETED. + */ + @Enumerated(EnumType.STRING) + @Column(name = "ingestion_status", nullable = false, length = 20) + @Builder.Default + private IngestionStatus ingestionStatus = IngestionStatus.PENDING; + + /** + * Detailed error message when ingestion_status is ERROR. + * Used for developer debugging and user feedback. + */ + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + /** + * Timestamp when this document was last successfully ingested. + */ + @Column(name = "last_ingested_at") + private LocalDateTime lastIngestedAt; + + /** + * Timestamp when this record was first created in database. + */ + @CreationTimestamp + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + /** + * Timestamp when this record was last updated. + */ + @UpdateTimestamp + @Column(name = "updated_at", nullable = false) + private LocalDateTime updatedAt; + + /** + * Updates the ingestion status and clears error message if successful. + * + * @param newStatus the new status to set + */ + public void updateIngestionStatus(IngestionStatus newStatus) { + this.ingestionStatus = newStatus; + if (newStatus == IngestionStatus.COMPLETED) { + this.errorMessage = null; + this.lastIngestedAt = LocalDateTime.now(); + } + } + + /** + * Updates the ingestion status to ERROR with a detailed error message. + * + * @param errorMessage detailed error message for debugging + */ + public void updateIngestionStatusToError(String errorMessage) { + this.ingestionStatus = IngestionStatus.ERROR; + this.errorMessage = errorMessage; + } + + /** + * Checks if the document is currently being processed. + * + * @return true if document is in any processing state + */ + public boolean isProcessing() { + return ingestionStatus == IngestionStatus.PARSING + || ingestionStatus == IngestionStatus.CHUNKING + || ingestionStatus == IngestionStatus.EMBEDDING + || ingestionStatus == IngestionStatus.INDEXING + || ingestionStatus == IngestionStatus.DELETING; + } + + /** + * Checks if the document processing has completed successfully. + * + * @return true if ingestion status is COMPLETED + */ + public boolean isCompleted() { + return ingestionStatus == IngestionStatus.COMPLETED; + } + + /** + * Checks if the document processing has failed with an error. + * + * @return true if ingestion status is ERROR + */ + public boolean hasError() { + return ingestionStatus == IngestionStatus.ERROR; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java b/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java new file mode 100644 index 0000000..aa70b06 --- /dev/null +++ b/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java @@ -0,0 +1,63 @@ +package com.opencontext.core.enums; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Represents the current status of the document ingestion pipeline. + * This enum defines all possible states that a source document can be in + * during its processing lifecycle from upload to completion. + */ +@Getter +@RequiredArgsConstructor +public enum IngestionStatus { + /** + * Initial state after user upload, waiting for processing to begin. + * The system will soon start processing this document. + */ + PENDING("Processing pending"), + + /** + * Currently analyzing document structure and content via Unstructured API. + * This step may take time depending on document complexity. + */ + PARSING("Document parsing in progress"), + + /** + * Splitting parsed content into meaningful hierarchical chunks. + * This step creates the logical structure of the document. + */ + CHUNKING("Content chunking in progress"), + + /** + * Converting each chunk into vector embeddings using embedding model. + * This step creates semantic data for similarity search. + */ + EMBEDDING("Semantic vector generation in progress"), + + /** + * Storing vectors and metadata in Elasticsearch, hierarchy in PostgreSQL. + * Final data storage step to make content searchable. + */ + INDEXING("Search data storage in progress"), + + /** + * All ingestion processes completed successfully, content is searchable. + * Users can now search for content from this document. + */ + COMPLETED("Completed"), + + /** + * Unrecoverable error occurred during ingestion process. + * Details are recorded in the 'error_message' column. + */ + ERROR("Error occurred"), + + /** + * Document deletion in progress, removing all related data. + * No other operations can be performed during this state. + */ + DELETING("Deletion in progress"); + + private final String description; +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java b/core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java new file mode 100644 index 0000000..bb97dff --- /dev/null +++ b/core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java @@ -0,0 +1,140 @@ +package com.opencontext.core.repository; + +import com.opencontext.core.entity.DocumentChunk; +import com.opencontext.core.entity.SourceDocument; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.UUID; + +/** + * Repository interface for DocumentChunk entity operations. + * Provides data access methods for hierarchical chunk structure management + * and chunk relationship queries. + */ +@Repository +public interface DocumentChunkRepository extends JpaRepository { + + /** + * Finds all chunks belonging to a specific source document. + * + * @param sourceDocument the source document to find chunks for + * @return list of chunks belonging to the source document + */ + List findBySourceDocument(SourceDocument sourceDocument); + + /** + * Finds all chunks belonging to a source document, ordered by sequence. + * + * @param sourceDocument the source document to find chunks for + * @return list of chunks ordered by sequence in document + */ + List findBySourceDocumentOrderBySequenceInDocumentAsc(SourceDocument sourceDocument); + + /** + * Finds all child chunks of a specific parent chunk. + * + * @param parentChunk the parent chunk to find children for + * @return list of child chunks + */ + List findByParentChunk(DocumentChunk parentChunk); + + /** + * Finds all root chunks (chunks without parent) for a source document. + * + * @param sourceDocument the source document to find root chunks for + * @return list of root chunks + */ + @Query("SELECT dc FROM DocumentChunk dc WHERE dc.sourceDocument = :sourceDocument AND dc.parentChunk IS NULL ORDER BY dc.sequenceInDocument ASC") + List findRootChunksBySourceDocument(@Param("sourceDocument") SourceDocument sourceDocument); + + /** + * Finds all root chunks (chunks without parent). + * + * @return list of all root chunks + */ + List findByParentChunkIsNull(); + + /** + * Finds all child chunks ordered by sequence. + * + * @param parentChunk the parent chunk to find children for + * @return list of child chunks ordered by sequence + */ + List findByParentChunkOrderBySequenceInDocumentAsc(DocumentChunk parentChunk); + + /** + * Counts the total number of chunks for a source document. + * + * @param sourceDocument the source document to count chunks for + * @return count of chunks belonging to the source document + */ + long countBySourceDocument(SourceDocument sourceDocument); + + /** + * Counts the number of child chunks for a parent chunk. + * + * @param parentChunk the parent chunk to count children for + * @return count of child chunks + */ + long countByParentChunk(DocumentChunk parentChunk); + + /** + * Checks if any chunks exist for a source document. + * + * @param sourceDocument the source document to check + * @return true if chunks exist for the source document + */ + boolean existsBySourceDocument(SourceDocument sourceDocument); + + /** + * Finds chunks by depth/hierarchy level using recursive query. + * This uses a Common Table Expression (CTE) to traverse the hierarchy. + * + * @param sourceDocument the source document to search within + * @param hierarchyLevel the hierarchy level to filter by (1 = root, 2 = first level children, etc.) + * @return list of chunks at the specified hierarchy level + */ + @Query(value = """ + WITH RECURSIVE chunk_hierarchy AS ( + SELECT id, source_document_id, parent_chunk_id, sequence_in_document, 1 as level + FROM document_chunks + WHERE source_document_id = :#{#sourceDocument.id} AND parent_chunk_id IS NULL + + UNION ALL + + SELECT dc.id, dc.source_document_id, dc.parent_chunk_id, dc.sequence_in_document, ch.level + 1 + FROM document_chunks dc + INNER JOIN chunk_hierarchy ch ON dc.parent_chunk_id = ch.id + ) + SELECT dc.* FROM document_chunks dc + INNER JOIN chunk_hierarchy ch ON dc.id = ch.id + WHERE ch.level = :hierarchyLevel + ORDER BY dc.sequence_in_document ASC + """, nativeQuery = true) + List findChunksByHierarchyLevel(@Param("sourceDocument") SourceDocument sourceDocument, + @Param("hierarchyLevel") int hierarchyLevel); + + /** + * Finds the maximum sequence number for a given source document and parent chunk. + * Used to determine the next sequence number when adding new chunks. + * + * @param sourceDocument the source document + * @param parentChunk the parent chunk (can be null for root chunks) + * @return the maximum sequence number, or 0 if no chunks exist + */ + @Query("SELECT COALESCE(MAX(dc.sequenceInDocument), 0) FROM DocumentChunk dc WHERE dc.sourceDocument = :sourceDocument AND dc.parentChunk = :parentChunk") + Integer findMaxSequenceForParent(@Param("sourceDocument") SourceDocument sourceDocument, + @Param("parentChunk") DocumentChunk parentChunk); + + /** + * Deletes all chunks belonging to a specific source document. + * This is used when a source document is being reprocessed. + * + * @param sourceDocument the source document whose chunks should be deleted + */ + void deleteBySourceDocument(SourceDocument sourceDocument); +} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java b/core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java new file mode 100644 index 0000000..d67c60e --- /dev/null +++ b/core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java @@ -0,0 +1,141 @@ +package com.opencontext.core.repository; + +import com.opencontext.core.entity.SourceDocument; +import com.opencontext.core.enums.IngestionStatus; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +/** + * Repository interface for SourceDocument entity operations. + * Provides data access methods for document metadata management + * and ingestion pipeline status tracking. + */ +@Repository +public interface SourceDocumentRepository extends JpaRepository { + + /** + * Finds a document by its file checksum to prevent duplicate uploads. + * + * @param fileChecksum SHA-256 hash of the file content + * @return Optional containing the document if found + */ + Optional findByFileChecksum(String fileChecksum); + + /** + * Finds all documents with a specific ingestion status. + * + * @param status the ingestion status to filter by + * @return list of documents with the specified status + */ + List findByIngestionStatus(IngestionStatus status); + + /** + * Finds all documents with a specific ingestion status with pagination. + * + * @param status the ingestion status to filter by + * @param pageable pagination parameters + * @return page of documents with the specified status + */ + Page findByIngestionStatus(IngestionStatus status, Pageable pageable); + + /** + * Finds all documents that have been successfully ingested. + * + * @return list of completed documents + */ + @Query("SELECT sd FROM SourceDocument sd WHERE sd.ingestionStatus = 'COMPLETED'") + List findAllCompleted(); + + /** + * Finds all documents that failed during ingestion. + * + * @return list of documents with errors + */ + @Query("SELECT sd FROM SourceDocument sd WHERE sd.ingestionStatus = 'ERROR'") + List findAllWithErrors(); + + /** + * Finds all documents that are currently being processed. + * + * @return list of documents in processing states + */ + @Query("SELECT sd FROM SourceDocument sd WHERE sd.ingestionStatus IN ('PARSING', 'CHUNKING', 'EMBEDDING', 'INDEXING')") + List findAllProcessing(); + + /** + * Finds documents by filename pattern (case-insensitive). + * + * @param filename the filename pattern to search for + * @return list of documents matching the filename pattern + */ + @Query("SELECT sd FROM SourceDocument sd WHERE LOWER(sd.originalFilename) LIKE LOWER(CONCAT('%', :filename, '%'))") + List findByOriginalFilenameContainingIgnoreCase(@Param("filename") String filename); + + /** + * Finds documents created within a specific time range. + * + * @param startDate the start of the time range + * @param endDate the end of the time range + * @return list of documents created within the range + */ + List findByCreatedAtBetween(LocalDateTime startDate, LocalDateTime endDate); + + /** + * Finds documents that were last ingested within a specific time range. + * + * @param startDate the start of the time range + * @param endDate the end of the time range + * @return list of documents last ingested within the range + */ + List findByLastIngestedAtBetween(LocalDateTime startDate, LocalDateTime endDate); + + /** + * Counts the total number of documents by ingestion status. + * + * @param status the ingestion status to count + * @return count of documents with the specified status + */ + long countByIngestionStatus(IngestionStatus status); + + /** + * Finds documents by file type (case-insensitive). + * + * @param fileType the file type to filter by (e.g., "PDF", "MARKDOWN") + * @return list of documents with the specified file type + */ + List findByFileTypeIgnoreCase(String fileType); + + /** + * Checks if a document with the given checksum already exists. + * + * @param fileChecksum SHA-256 hash of the file content + * @return true if a document with this checksum exists + */ + boolean existsByFileChecksum(String fileChecksum); + + /** + * Finds documents ordered by creation date (most recent first). + * + * @param pageable pagination parameters + * @return page of documents ordered by creation date + */ + Page findAllByOrderByCreatedAtDesc(Pageable pageable); + + /** + * Finds documents that have been processing for too long (potential stuck documents). + * + * @param cutoffTime the time before which documents are considered stuck + * @return list of potentially stuck documents + */ + @Query("SELECT sd FROM SourceDocument sd WHERE sd.ingestionStatus IN ('PARSING', 'CHUNKING', 'EMBEDDING', 'INDEXING') AND sd.updatedAt < :cutoffTime") + List findStuckProcessingDocuments(@Param("cutoffTime") LocalDateTime cutoffTime); +} \ No newline at end of file diff --git a/core/src/main/resources/db/migration/V1__Create_source_documents_and_document_chunks_tables.sql b/core/src/main/resources/db/migration/V1__Create_source_documents_and_document_chunks_tables.sql new file mode 100644 index 0000000..1189f76 --- /dev/null +++ b/core/src/main/resources/db/migration/V1__Create_source_documents_and_document_chunks_tables.sql @@ -0,0 +1,65 @@ +-- OpenContext MVP Database Schema +-- This migration creates the core tables for document storage and hierarchical chunk management +-- Based on PRD Section 5.1 PostgreSQL Data Model + +-- Table: source_documents +-- Stores master information for all uploaded source files +CREATE TABLE source_documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + original_filename VARCHAR(255) NOT NULL, + file_storage_path VARCHAR(1024) NOT NULL, + file_type VARCHAR(50) NOT NULL, + file_size BIGINT NOT NULL, + file_checksum VARCHAR(64) NOT NULL, + ingestion_status VARCHAR(20) NOT NULL DEFAULT 'PENDING', + error_message TEXT, + last_ingested_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_file_checksum UNIQUE (file_checksum) +); + +-- Table: document_chunks +-- Stores hierarchical structure information for structured chunks +CREATE TABLE document_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_document_id UUID NOT NULL REFERENCES source_documents(id) ON DELETE CASCADE, + parent_chunk_id UUID REFERENCES document_chunks(id) ON DELETE CASCADE, + sequence_in_document INT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- Indexes for performance optimization +-- Index for querying chunks by source document +CREATE INDEX idx_chunk_source_document_id ON document_chunks(source_document_id); + +-- Index for hierarchical navigation (parent-child relationships) +CREATE INDEX idx_chunk_parent_chunk_id ON document_chunks(parent_chunk_id); + +-- Index for duplicate file prevention via checksum +CREATE INDEX idx_source_documents_file_checksum ON source_documents(file_checksum); + +-- Index for querying documents by ingestion status +CREATE INDEX idx_source_documents_ingestion_status ON source_documents(ingestion_status); + +-- Comments for table and column documentation +COMMENT ON TABLE source_documents IS 'Master table for storing metadata of all uploaded source files'; +COMMENT ON COLUMN source_documents.id IS 'Primary key - unique identifier for each uploaded source document'; +COMMENT ON COLUMN source_documents.original_filename IS 'Original filename provided by user during upload'; +COMMENT ON COLUMN source_documents.file_storage_path IS 'Storage path in Object Storage (MinIO) for accessing the original file'; +COMMENT ON COLUMN source_documents.file_type IS 'Type of uploaded file (e.g., PDF, MARKDOWN)'; +COMMENT ON COLUMN source_documents.file_size IS 'Size of uploaded file in bytes'; +COMMENT ON COLUMN source_documents.file_checksum IS 'SHA-256 hash of file content for duplicate prevention and integrity verification'; +COMMENT ON COLUMN source_documents.ingestion_status IS 'Current status of document ingestion process (PENDING, PARSING, CHUNKING, EMBEDDING, INDEXING, COMPLETED, ERROR, DELETING)'; +COMMENT ON COLUMN source_documents.error_message IS 'Detailed error message when ingestion_status is ERROR (for developer debugging)'; +COMMENT ON COLUMN source_documents.last_ingested_at IS 'Timestamp when this document was last successfully ingested'; +COMMENT ON COLUMN source_documents.created_at IS 'Timestamp when this record was first created in database'; +COMMENT ON COLUMN source_documents.updated_at IS 'Timestamp when this record was last updated'; +COMMENT ON CONSTRAINT uq_file_checksum ON source_documents IS 'Ensures file_checksum values are unique across the table, essential for MVP implementation'; + +COMMENT ON TABLE document_chunks IS 'Table storing hierarchical structure information for document chunks'; +COMMENT ON COLUMN document_chunks.id IS 'Primary key - unique identifier for each structured chunk. Same value as chunkId in Elasticsearch'; +COMMENT ON COLUMN document_chunks.source_document_id IS 'Foreign key to source_documents table. Parent document is cascade deleted when chunk is removed'; +COMMENT ON COLUMN document_chunks.parent_chunk_id IS 'Foreign key to this table for hierarchical parent chunk. NULL for top-level chunks'; +COMMENT ON COLUMN document_chunks.sequence_in_document IS 'Sequential order within document, among sibling chunks with same parent_chunk_id'; +COMMENT ON COLUMN document_chunks.created_at IS 'Timestamp when this chunk record was first created'; \ No newline at end of file diff --git a/core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java b/core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java new file mode 100644 index 0000000..79072b9 --- /dev/null +++ b/core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java @@ -0,0 +1,161 @@ +package com.opencontext.core.entity; + +import com.opencontext.core.enums.IngestionStatus; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@DisplayName("DocumentChunk Entity Unit Tests") +class DocumentChunkTest { + + @Test + @DisplayName("Can create DocumentChunk using builder pattern") + void canCreateDocumentChunkWithBuilder() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + int sequence = 1; + + // When + DocumentChunk chunk = DocumentChunk.builder() + .sourceDocument(sourceDocument) + .sequenceInDocument(sequence) + .build(); + + // Then + assertThat(chunk.getSourceDocument()).isEqualTo(sourceDocument); + assertThat(chunk.getSequenceInDocument()).isEqualTo(sequence); + assertThat(chunk.getParentChunk()).isNull(); + } + + @Test + @DisplayName("Root chunk correctly identifies itself as root") + void rootChunkIdentifiesAsRoot() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + + // When + DocumentChunk rootChunk = DocumentChunk.builder() + .sourceDocument(sourceDocument) + .parentChunk(null) + .sequenceInDocument(0) + .build(); + + // Then + assertThat(rootChunk.isRootChunk()).isTrue(); + assertThat(rootChunk.getParentChunk()).isNull(); + } + + @Test + @DisplayName("Child chunk correctly identifies parent relationship") + void childChunkIdentifiesParentRelationship() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + DocumentChunk parentChunk = createTestChunk(sourceDocument, null, 0); + + // When + DocumentChunk childChunk = DocumentChunk.builder() + .sourceDocument(sourceDocument) + .parentChunk(parentChunk) + .sequenceInDocument(1) + .build(); + + // Then + assertThat(childChunk.isRootChunk()).isFalse(); + assertThat(childChunk.getParentChunk()).isEqualTo(parentChunk); + } + + @Test + @DisplayName("Can add child chunks and check hasChildren") + void canAddChildChunksAndCheckHasChildren() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + DocumentChunk parent = createTestChunk(sourceDocument, null, 0); + DocumentChunk child1 = createTestChunk(sourceDocument, parent, 1); + DocumentChunk child2 = createTestChunk(sourceDocument, parent, 2); + + // When + parent.addChildChunk(child1); + parent.addChildChunk(child2); + + // Then + assertThat(parent.hasChildren()).isTrue(); + assertThat(parent.getChildChunks()).hasSize(2); + assertThat(parent.getChildChunks()).containsExactly(child1, child2); + assertThat(child1.hasChildren()).isFalse(); + assertThat(child2.hasChildren()).isFalse(); + } + + @Test + @DisplayName("Hierarchy level calculation works correctly") + void hierarchyLevelCalculationWorks() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + DocumentChunk root = createTestChunk(sourceDocument, null, 0); + DocumentChunk level1 = createTestChunk(sourceDocument, root, 1); + DocumentChunk level2 = createTestChunk(sourceDocument, level1, 2); + + // When & Then + assertThat(root.getHierarchyLevel()).isEqualTo(1); + assertThat(level1.getHierarchyLevel()).isEqualTo(2); + assertThat(level2.getHierarchyLevel()).isEqualTo(3); + } + + @Test + @DisplayName("getAncestors returns correct ancestor hierarchy") + void getAncestorsReturnsCorrectHierarchy() { + // Given - Create a hierarchy: root -> level1 -> level2 -> level3 + SourceDocument sourceDocument = createTestSourceDocument(); + DocumentChunk root = createTestChunk(sourceDocument, null, 0); + DocumentChunk level1 = createTestChunk(sourceDocument, root, 1); + DocumentChunk level2 = createTestChunk(sourceDocument, level1, 2); + DocumentChunk level3 = createTestChunk(sourceDocument, level2, 3); + + // When + List ancestorsOfLevel3 = level3.getAncestors(); + List ancestorsOfLevel2 = level2.getAncestors(); + List ancestorsOfRoot = root.getAncestors(); + + // Then + assertThat(ancestorsOfLevel3).hasSize(3); + assertThat(ancestorsOfLevel3).containsExactly(root, level1, level2); + + assertThat(ancestorsOfLevel2).hasSize(2); + assertThat(ancestorsOfLevel2).containsExactly(root, level1); + + assertThat(ancestorsOfRoot).isEmpty(); + } + + @Test + @DisplayName("Empty child list returns hasChildren as false") + void emptyChildListReturnsHasChildrenAsFalse() { + // Given + SourceDocument sourceDocument = createTestSourceDocument(); + DocumentChunk leafChunk = createTestChunk(sourceDocument, null, 0); + + // When & Then + assertThat(leafChunk.hasChildren()).isFalse(); + assertThat(leafChunk.getChildChunks()).isEmpty(); + } + + private SourceDocument createTestSourceDocument() { + return SourceDocument.builder() + .originalFilename("test-document.pdf") + .fileStoragePath("/test-document.pdf") + .fileType("PDF") + .fileSize(1024L) + .fileChecksum("test-checksum") + .ingestionStatus(IngestionStatus.PENDING) + .build(); + } + + private DocumentChunk createTestChunk(SourceDocument sourceDocument, DocumentChunk parent, int sequence) { + return DocumentChunk.builder() + .sourceDocument(sourceDocument) + .parentChunk(parent) + .sequenceInDocument(sequence) + .build(); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java b/core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java new file mode 100644 index 0000000..f146348 --- /dev/null +++ b/core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java @@ -0,0 +1,140 @@ +package com.opencontext.core.entity; + +import com.opencontext.core.enums.IngestionStatus; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +@DisplayName("SourceDocument Entity Unit Tests") +class SourceDocumentTest { + + @Test + @DisplayName("Can create SourceDocument using builder pattern") + void canCreateSourceDocumentWithBuilder() { + // Given + String filename = "test-document.pdf"; + String storagePath = "/documents/test-document.pdf"; + String fileType = "PDF"; + Long fileSize = 1024L; + String checksum = "abc123def456"; + + // When + SourceDocument document = SourceDocument.builder() + .originalFilename(filename) + .fileStoragePath(storagePath) + .fileType(fileType) + .fileSize(fileSize) + .fileChecksum(checksum) + .ingestionStatus(IngestionStatus.PENDING) + .build(); + + // Then + assertThat(document.getOriginalFilename()).isEqualTo(filename); + assertThat(document.getFileStoragePath()).isEqualTo(storagePath); + assertThat(document.getFileType()).isEqualTo(fileType); + assertThat(document.getFileSize()).isEqualTo(fileSize); + assertThat(document.getFileChecksum()).isEqualTo(checksum); + assertThat(document.getIngestionStatus()).isEqualTo(IngestionStatus.PENDING); + } + + @Test + @DisplayName("Can update ingestion status to ERROR with error message") + void canUpdateIngestionStatusToError() { + // Given + SourceDocument document = createTestDocument(); + String errorMessage = "Parsing failed"; + + // When + document.updateIngestionStatusToError(errorMessage); + + // Then + assertThat(document.getIngestionStatus()).isEqualTo(IngestionStatus.ERROR); + assertThat(document.getErrorMessage()).isEqualTo(errorMessage); + } + + @Test + @DisplayName("Updating status to COMPLETED sets lastIngestedAt and clears error message") + void setsLastIngestedAtWhenCompletingIngestion() { + // Given + SourceDocument document = createTestDocument(); + LocalDateTime beforeUpdate = LocalDateTime.now().minusSeconds(1); + + // When + document.updateIngestionStatus(IngestionStatus.COMPLETED); + + // Then + LocalDateTime afterUpdate = LocalDateTime.now().plusSeconds(1); + assertThat(document.getIngestionStatus()).isEqualTo(IngestionStatus.COMPLETED); + assertThat(document.getErrorMessage()).isNull(); + assertThat(document.getLastIngestedAt()).isBetween(beforeUpdate, afterUpdate); + } + + @Test + @DisplayName("isProcessing method correctly identifies processing states") + void isProcessingReturnsTrueForProcessingStatuses() { + // Given + SourceDocument document = createTestDocument(); + + // When & Then - Processing statuses + document.updateIngestionStatus(IngestionStatus.PARSING); + assertThat(document.isProcessing()).isTrue(); + + document.updateIngestionStatus(IngestionStatus.CHUNKING); + assertThat(document.isProcessing()).isTrue(); + + document.updateIngestionStatus(IngestionStatus.EMBEDDING); + assertThat(document.isProcessing()).isTrue(); + + document.updateIngestionStatus(IngestionStatus.INDEXING); + assertThat(document.isProcessing()).isTrue(); + + document.updateIngestionStatus(IngestionStatus.DELETING); + assertThat(document.isProcessing()).isTrue(); + + // When & Then - Non-processing statuses + document.updateIngestionStatus(IngestionStatus.PENDING); + assertThat(document.isProcessing()).isFalse(); + + document.updateIngestionStatus(IngestionStatus.COMPLETED); + assertThat(document.isProcessing()).isFalse(); + + document.updateIngestionStatusToError("Some error"); + assertThat(document.isProcessing()).isFalse(); + } + + @Test + @DisplayName("Status check methods work correctly") + void statusCheckMethodsWorkCorrectly() { + // Given + SourceDocument document = createTestDocument(); + + // When & Then - COMPLETED status + document.updateIngestionStatus(IngestionStatus.COMPLETED); + assertThat(document.isCompleted()).isTrue(); + assertThat(document.hasError()).isFalse(); + + // When & Then - ERROR status + document.updateIngestionStatusToError("Test error"); + assertThat(document.isCompleted()).isFalse(); + assertThat(document.hasError()).isTrue(); + + // When & Then - PENDING status + document.updateIngestionStatus(IngestionStatus.PENDING); + assertThat(document.isCompleted()).isFalse(); + assertThat(document.hasError()).isFalse(); + } + + private SourceDocument createTestDocument() { + return SourceDocument.builder() + .originalFilename("test.pdf") + .fileStoragePath("/test.pdf") + .fileType("PDF") + .fileSize(1024L) + .fileChecksum("test-checksum") + .ingestionStatus(IngestionStatus.PENDING) + .build(); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java b/core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java new file mode 100644 index 0000000..d4b6efb --- /dev/null +++ b/core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java @@ -0,0 +1,273 @@ +package com.opencontext.core.repository; + +import com.opencontext.core.entity.DocumentChunk; +import com.opencontext.core.entity.SourceDocument; +import com.opencontext.core.enums.IngestionStatus; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@DirtiesContext +@DisplayName("DocumentChunkRepository Integration Tests") +class DocumentChunkRepositoryTest { + + static PostgreSQLContainer postgres; + + @BeforeAll + static void initContainer() { + postgres = new PostgreSQLContainer<>("postgres:16-alpine") + .withDatabaseName("test_db") + .withUsername("test_user") + .withPassword("test_password"); + postgres.start(); + } + + @AfterAll + static void cleanupContainer() { + if (postgres != null) { + postgres.close(); + } + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", postgres::getJdbcUrl); + registry.add("spring.datasource.username", postgres::getUsername); + registry.add("spring.datasource.password", postgres::getPassword); + } + + @Autowired + private DocumentChunkRepository documentChunkRepository; + + @Autowired + private SourceDocumentRepository sourceDocumentRepository; + + @Test + @DisplayName("Can save and retrieve DocumentChunk by ID") + @Transactional + void canSaveAndRetrieveDocumentChunkById() { + // Given + SourceDocument sourceDocument = createAndSaveSourceDocument("test.pdf", "checksum-1"); + DocumentChunk chunk = createTestChunk(sourceDocument, null, 1); + + // When + DocumentChunk savedChunk = documentChunkRepository.save(chunk); + Optional foundChunk = documentChunkRepository.findById(savedChunk.getId()); + + // Then + assertThat(foundChunk).isPresent(); + assertThat(foundChunk.get().getSourceDocument()).isEqualTo(sourceDocument); + assertThat(foundChunk.get().getSequenceInDocument()).isEqualTo(1); + assertThat(foundChunk.get().getParentChunk()).isNull(); + } + + @Test + @DisplayName("Can find chunks by source document ID") + @Transactional + void canFindChunksBySourceDocumentId() { + // Given + SourceDocument doc1 = createAndSaveSourceDocument("doc1.pdf", "checksum-1"); + SourceDocument doc2 = createAndSaveSourceDocument("doc2.pdf", "checksum-2"); + + DocumentChunk chunk1 = createTestChunk(doc1, null, 1); + DocumentChunk chunk2 = createTestChunk(doc1, null, 2); + DocumentChunk chunk3 = createTestChunk(doc2, null, 1); + + documentChunkRepository.saveAll(List.of(chunk1, chunk2, chunk3)); + + // When + List chunksForDoc1 = documentChunkRepository.findBySourceDocument(doc1); + List chunksForDoc2 = documentChunkRepository.findBySourceDocument(doc2); + + // Then + assertThat(chunksForDoc1).hasSize(2); + assertThat(chunksForDoc1).allMatch(chunk -> chunk.getSourceDocument().equals(doc1)); + + assertThat(chunksForDoc2).hasSize(1); + assertThat(chunksForDoc2.get(0).getSourceDocument()).isEqualTo(doc2); + } + + @Test + @DisplayName("Can find chunks by parent chunk (hierarchical relationship)") + @Transactional + void canFindChunksByParentChunk() { + // Given + SourceDocument sourceDocument = createAndSaveSourceDocument("test.pdf", "checksum-1"); + DocumentChunk parentChunk = createTestChunk(sourceDocument, null, 1); + documentChunkRepository.save(parentChunk); + + DocumentChunk child1 = createTestChunk(sourceDocument, parentChunk, 2); + DocumentChunk child2 = createTestChunk(sourceDocument, parentChunk, 3); + DocumentChunk orphanChunk = createTestChunk(sourceDocument, null, 4); + + documentChunkRepository.saveAll(List.of(child1, child2, orphanChunk)); + + // When + List childChunks = documentChunkRepository.findByParentChunk(parentChunk); + List rootChunks = documentChunkRepository.findByParentChunkIsNull(); + + // Then + assertThat(childChunks).hasSize(2); + assertThat(childChunks).allMatch(chunk -> chunk.getParentChunk().equals(parentChunk)); + + assertThat(rootChunks).hasSize(2); // parentChunk and orphanChunk + assertThat(rootChunks).allMatch(chunk -> chunk.getParentChunk() == null); + } + + @Test + @DisplayName("Can find root chunks (chunks without parent)") + @Transactional + void canFindRootChunks() { + // Given + SourceDocument sourceDocument = createAndSaveSourceDocument("test.pdf", "checksum-1"); + + DocumentChunk rootChunk1 = createTestChunk(sourceDocument, null, 1); + DocumentChunk rootChunk2 = createTestChunk(sourceDocument, null, 2); + + documentChunkRepository.save(rootChunk1); + documentChunkRepository.save(rootChunk2); + + DocumentChunk childChunk = createTestChunk(sourceDocument, rootChunk1, 3); + documentChunkRepository.save(childChunk); + + // When + List rootChunks = documentChunkRepository.findByParentChunkIsNull(); + + // Then + assertThat(rootChunks).hasSize(2); + assertThat(rootChunks).allMatch(DocumentChunk::isRootChunk); + } + + @Test + @DisplayName("Can find chunks ordered by sequence in document") + @Transactional + void canFindChunksOrderedBySequence() { + // Given + SourceDocument sourceDocument = createAndSaveSourceDocument("test.pdf", "checksum-1"); + + DocumentChunk chunk3 = createTestChunk(sourceDocument, null, 3); + DocumentChunk chunk1 = createTestChunk(sourceDocument, null, 1); + DocumentChunk chunk2 = createTestChunk(sourceDocument, null, 2); + + // Save in random order + documentChunkRepository.saveAll(List.of(chunk3, chunk1, chunk2)); + + // When + List orderedChunks = documentChunkRepository.findBySourceDocumentOrderBySequenceInDocumentAsc(sourceDocument); + + // Then + assertThat(orderedChunks).hasSize(3); + assertThat(orderedChunks.get(0).getSequenceInDocument()).isEqualTo(1); + assertThat(orderedChunks.get(1).getSequenceInDocument()).isEqualTo(2); + assertThat(orderedChunks.get(2).getSequenceInDocument()).isEqualTo(3); + } + + @Test + @DisplayName("Can count chunks by source document") + @Transactional + void canCountChunksBySourceDocument() { + // Given + SourceDocument doc1 = createAndSaveSourceDocument("doc1.pdf", "checksum-1"); + SourceDocument doc2 = createAndSaveSourceDocument("doc2.pdf", "checksum-2"); + + DocumentChunk chunk1 = createTestChunk(doc1, null, 1); + DocumentChunk chunk2 = createTestChunk(doc1, null, 2); + DocumentChunk chunk3 = createTestChunk(doc2, null, 1); + + documentChunkRepository.saveAll(List.of(chunk1, chunk2, chunk3)); + + // When + long countForDoc1 = documentChunkRepository.countBySourceDocument(doc1); + long countForDoc2 = documentChunkRepository.countBySourceDocument(doc2); + + // Then + assertThat(countForDoc1).isEqualTo(2L); + assertThat(countForDoc2).isEqualTo(1L); + } + + @Test + @DisplayName("Can check if chunks exist for source document") + @Transactional + void canCheckIfChunksExistForSourceDocument() { + // Given + SourceDocument docWithChunks = createAndSaveSourceDocument("with-chunks.pdf", "checksum-1"); + SourceDocument docWithoutChunks = createAndSaveSourceDocument("without-chunks.pdf", "checksum-2"); + + DocumentChunk chunk = createTestChunk(docWithChunks, null, 1); + documentChunkRepository.save(chunk); + + // When + boolean hasChunks = documentChunkRepository.existsBySourceDocument(docWithChunks); + boolean hasNoChunks = documentChunkRepository.existsBySourceDocument(docWithoutChunks); + + // Then + assertThat(hasChunks).isTrue(); + assertThat(hasNoChunks).isFalse(); + } + + @Test + @DisplayName("Cascading delete works when source document is deleted") + @Transactional + void cascadingDeleteWorksWhenSourceDocumentIsDeleted() { + // Given + SourceDocument sourceDocument = createAndSaveSourceDocument("test.pdf", "checksum-1"); + DocumentChunk chunk1 = createTestChunk(sourceDocument, null, 1); + DocumentChunk chunk2 = createTestChunk(sourceDocument, null, 2); + + documentChunkRepository.saveAll(List.of(chunk1, chunk2)); + + UUID sourceDocumentId = sourceDocument.getId(); + long initialChunkCount = documentChunkRepository.countBySourceDocument(sourceDocument); + assertThat(initialChunkCount).isEqualTo(2L); + + // When + sourceDocumentRepository.delete(sourceDocument); + sourceDocumentRepository.flush(); // Ensure deletion is executed + + // Then + boolean sourceExists = sourceDocumentRepository.existsById(sourceDocumentId); + long remainingChunkCount = documentChunkRepository.count(); + + assertThat(sourceExists).isFalse(); + assertThat(remainingChunkCount).isEqualTo(0L); // All chunks should be deleted due to CASCADE + } + + private SourceDocument createAndSaveSourceDocument(String filename, String checksum) { + SourceDocument document = SourceDocument.builder() + .originalFilename(filename) + .fileStoragePath("/" + filename) + .fileType("PDF") + .fileSize(1024L) + .fileChecksum(checksum) + .ingestionStatus(IngestionStatus.PENDING) + .build(); + return sourceDocumentRepository.save(document); + } + + private DocumentChunk createTestChunk(SourceDocument sourceDocument, DocumentChunk parentChunk, int sequence) { + return DocumentChunk.builder() + .sourceDocument(sourceDocument) + .parentChunk(parentChunk) + .sequenceInDocument(sequence) + .build(); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java b/core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java new file mode 100644 index 0000000..1967ac3 --- /dev/null +++ b/core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java @@ -0,0 +1,241 @@ +package com.opencontext.core.repository; + +import com.opencontext.core.entity.SourceDocument; +import com.opencontext.core.enums.IngestionStatus; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@DirtiesContext +@DisplayName("SourceDocumentRepository Integration Tests") +class SourceDocumentRepositoryTest { + + static PostgreSQLContainer postgres; + + @BeforeAll + static void initContainer() { + postgres = new PostgreSQLContainer<>("postgres:16-alpine") + .withDatabaseName("test_db") + .withUsername("test_user") + .withPassword("test_password"); + postgres.start(); + } + + @AfterAll + static void cleanupContainer() { + if (postgres != null) { + postgres.close(); + } + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", postgres::getJdbcUrl); + registry.add("spring.datasource.username", postgres::getUsername); + registry.add("spring.datasource.password", postgres::getPassword); + } + + @Autowired + private SourceDocumentRepository sourceDocumentRepository; + + @Test + @DisplayName("Can save and retrieve SourceDocument by ID") + @Transactional + void canSaveAndRetrieveSourceDocumentById() { + // Given + SourceDocument document = createTestDocument("test.pdf", "unique-checksum-1"); + + // When + SourceDocument savedDocument = sourceDocumentRepository.save(document); + Optional foundDocument = sourceDocumentRepository.findById(savedDocument.getId()); + + // Then + assertThat(foundDocument).isPresent(); + assertThat(foundDocument.get().getOriginalFilename()).isEqualTo("test.pdf"); + assertThat(foundDocument.get().getFileChecksum()).isEqualTo("unique-checksum-1"); + assertThat(foundDocument.get().getIngestionStatus()).isEqualTo(IngestionStatus.PENDING); + } + + @Test + @DisplayName("Can find document by file checksum") + @Transactional + void canFindDocumentByFileChecksum() { + // Given + String uniqueChecksum = "unique-checksum-for-find-test"; + SourceDocument document = createTestDocument("findme.pdf", uniqueChecksum); + sourceDocumentRepository.save(document); + + // When + Optional foundDocument = sourceDocumentRepository.findByFileChecksum(uniqueChecksum); + + // Then + assertThat(foundDocument).isPresent(); + assertThat(foundDocument.get().getOriginalFilename()).isEqualTo("findme.pdf"); + assertThat(foundDocument.get().getFileChecksum()).isEqualTo(uniqueChecksum); + } + + @Test + @DisplayName("Returns empty when searching for non-existent checksum") + @Transactional + void returnsEmptyForNonExistentChecksum() { + // Given + String nonExistentChecksum = "non-existent-checksum"; + + // When + Optional foundDocument = sourceDocumentRepository.findByFileChecksum(nonExistentChecksum); + + // Then + assertThat(foundDocument).isEmpty(); + } + + @Test + @DisplayName("Can find documents by ingestion status") + @Transactional + void canFindDocumentsByIngestionStatus() { + // Given + SourceDocument pendingDoc1 = createTestDocument("pending1.pdf", "checksum-1"); + SourceDocument pendingDoc2 = createTestDocument("pending2.pdf", "checksum-2"); + SourceDocument completedDoc = createTestDocument("completed.pdf", "checksum-3"); + completedDoc.updateIngestionStatus(IngestionStatus.COMPLETED); + + sourceDocumentRepository.saveAll(List.of(pendingDoc1, pendingDoc2, completedDoc)); + + // When + List pendingDocuments = sourceDocumentRepository.findByIngestionStatus(IngestionStatus.PENDING); + List completedDocuments = sourceDocumentRepository.findByIngestionStatus(IngestionStatus.COMPLETED); + + // Then + assertThat(pendingDocuments).hasSize(2); + assertThat(pendingDocuments).extracting(SourceDocument::getOriginalFilename) + .containsExactlyInAnyOrder("pending1.pdf", "pending2.pdf"); + + assertThat(completedDocuments).hasSize(1); + assertThat(completedDocuments.get(0).getOriginalFilename()).isEqualTo("completed.pdf"); + } + + @Test + @DisplayName("Can count documents by ingestion status") + @Transactional + void canCountDocumentsByIngestionStatus() { + // Given + SourceDocument pendingDoc = createTestDocument("pending.pdf", "checksum-pending"); + SourceDocument errorDoc = createTestDocument("error.pdf", "checksum-error"); + errorDoc.updateIngestionStatusToError("Test error"); + + sourceDocumentRepository.saveAll(List.of(pendingDoc, errorDoc)); + + // When + long pendingCount = sourceDocumentRepository.countByIngestionStatus(IngestionStatus.PENDING); + long errorCount = sourceDocumentRepository.countByIngestionStatus(IngestionStatus.ERROR); + long completedCount = sourceDocumentRepository.countByIngestionStatus(IngestionStatus.COMPLETED); + + // Then + assertThat(pendingCount).isEqualTo(1L); + assertThat(errorCount).isEqualTo(1L); + assertThat(completedCount).isEqualTo(0L); + } + + @Test + @DisplayName("Can find documents by file type") + @Transactional + void canFindDocumentsByFileType() { + // Given + SourceDocument pdfDoc = createTestDocument("doc1.pdf", "checksum-pdf"); + pdfDoc = SourceDocument.builder() + .originalFilename("doc1.pdf") + .fileStoragePath("/doc1.pdf") + .fileType("PDF") + .fileSize(1024L) + .fileChecksum("checksum-pdf") + .build(); + + SourceDocument markdownDoc = SourceDocument.builder() + .originalFilename("doc2.md") + .fileStoragePath("/doc2.md") + .fileType("MARKDOWN") + .fileSize(512L) + .fileChecksum("checksum-md") + .build(); + + sourceDocumentRepository.saveAll(List.of(pdfDoc, markdownDoc)); + + // When + List pdfDocuments = sourceDocumentRepository.findByFileTypeIgnoreCase("PDF"); + List markdownDocuments = sourceDocumentRepository.findByFileTypeIgnoreCase("MARKDOWN"); + + // Then + assertThat(pdfDocuments).hasSize(1); + assertThat(pdfDocuments.get(0).getOriginalFilename()).isEqualTo("doc1.pdf"); + + assertThat(markdownDocuments).hasSize(1); + assertThat(markdownDocuments.get(0).getOriginalFilename()).isEqualTo("doc2.md"); + } + + @Test + @DisplayName("Can check if document exists by checksum") + @Transactional + void canCheckIfDocumentExistsByChecksum() { + // Given + String existingChecksum = "existing-checksum"; + String nonExistingChecksum = "non-existing-checksum"; + SourceDocument document = createTestDocument("existing.pdf", existingChecksum); + sourceDocumentRepository.save(document); + + // When + boolean exists = sourceDocumentRepository.existsByFileChecksum(existingChecksum); + boolean doesNotExist = sourceDocumentRepository.existsByFileChecksum(nonExistingChecksum); + + // Then + assertThat(exists).isTrue(); + assertThat(doesNotExist).isFalse(); + } + + @Test + @DisplayName("Can find documents created within time range") + @Transactional + void canFindDocumentsCreatedWithinTimeRange() { + // Given + LocalDateTime startTime = LocalDateTime.now().minusHours(1); + LocalDateTime endTime = LocalDateTime.now().plusHours(1); + + SourceDocument recentDocument = createTestDocument("recent.pdf", "recent-checksum"); + sourceDocumentRepository.save(recentDocument); + + // When + List documentsInRange = sourceDocumentRepository.findByCreatedAtBetween(startTime, endTime); + + // Then + assertThat(documentsInRange).hasSize(1); + assertThat(documentsInRange.get(0).getOriginalFilename()).isEqualTo("recent.pdf"); + } + + private SourceDocument createTestDocument(String filename, String checksum) { + return SourceDocument.builder() + .originalFilename(filename) + .fileStoragePath("/" + filename) + .fileType("PDF") + .fileSize(1024L) + .fileChecksum(checksum) + .ingestionStatus(IngestionStatus.PENDING) + .build(); + } +} \ No newline at end of file From 2afcd9075e31d933a944a2911420e77861d0454d Mon Sep 17 00:00:00 2001 From: qowlgur121 Date: Sun, 10 Aug 2025 12:19:39 +0900 Subject: [PATCH 5/6] refactor: restructure package hierarchy to align with team standards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove redundant 'core' package nesting (com.opencontext.core.* → com.opencontext.*) - Eliminate duplicate IngestionStatus enum, use team member's implementation - Maintain full entity relationship functionality and test coverage - Update all imports to reflect new package structure This resolves package structure inconsistencies and enables seamless integration with common modules established in PR #5. --- .../core/enums/IngestionStatus.java | 63 ------------------- .../{core => }/entity/DocumentChunk.java | 2 +- .../{core => }/entity/SourceDocument.java | 4 +- .../repository/DocumentChunkRepository.java | 6 +- .../repository/SourceDocumentRepository.java | 6 +- .../{core => }/entity/DocumentChunkTest.java | 4 +- .../{core => }/entity/SourceDocumentTest.java | 4 +- .../DocumentChunkRepositoryTest.java | 8 +-- .../SourceDocumentRepositoryTest.java | 6 +- 9 files changed, 20 insertions(+), 83 deletions(-) delete mode 100644 core/src/main/java/com/opencontext/core/enums/IngestionStatus.java rename core/src/main/java/com/opencontext/{core => }/entity/DocumentChunk.java (99%) rename core/src/main/java/com/opencontext/{core => }/entity/SourceDocument.java (98%) rename core/src/main/java/com/opencontext/{core => }/repository/DocumentChunkRepository.java (97%) rename core/src/main/java/com/opencontext/{core => }/repository/SourceDocumentRepository.java (97%) rename core/src/test/java/com/opencontext/{core => }/entity/DocumentChunkTest.java (98%) rename core/src/test/java/com/opencontext/{core => }/entity/SourceDocumentTest.java (98%) rename core/src/test/java/com/opencontext/{core => }/repository/DocumentChunkRepositoryTest.java (98%) rename core/src/test/java/com/opencontext/{core => }/repository/SourceDocumentRepositoryTest.java (98%) diff --git a/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java b/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java deleted file mode 100644 index aa70b06..0000000 --- a/core/src/main/java/com/opencontext/core/enums/IngestionStatus.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.opencontext.core.enums; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -/** - * Represents the current status of the document ingestion pipeline. - * This enum defines all possible states that a source document can be in - * during its processing lifecycle from upload to completion. - */ -@Getter -@RequiredArgsConstructor -public enum IngestionStatus { - /** - * Initial state after user upload, waiting for processing to begin. - * The system will soon start processing this document. - */ - PENDING("Processing pending"), - - /** - * Currently analyzing document structure and content via Unstructured API. - * This step may take time depending on document complexity. - */ - PARSING("Document parsing in progress"), - - /** - * Splitting parsed content into meaningful hierarchical chunks. - * This step creates the logical structure of the document. - */ - CHUNKING("Content chunking in progress"), - - /** - * Converting each chunk into vector embeddings using embedding model. - * This step creates semantic data for similarity search. - */ - EMBEDDING("Semantic vector generation in progress"), - - /** - * Storing vectors and metadata in Elasticsearch, hierarchy in PostgreSQL. - * Final data storage step to make content searchable. - */ - INDEXING("Search data storage in progress"), - - /** - * All ingestion processes completed successfully, content is searchable. - * Users can now search for content from this document. - */ - COMPLETED("Completed"), - - /** - * Unrecoverable error occurred during ingestion process. - * Details are recorded in the 'error_message' column. - */ - ERROR("Error occurred"), - - /** - * Document deletion in progress, removing all related data. - * No other operations can be performed during this state. - */ - DELETING("Deletion in progress"); - - private final String description; -} \ No newline at end of file diff --git a/core/src/main/java/com/opencontext/core/entity/DocumentChunk.java b/core/src/main/java/com/opencontext/entity/DocumentChunk.java similarity index 99% rename from core/src/main/java/com/opencontext/core/entity/DocumentChunk.java rename to core/src/main/java/com/opencontext/entity/DocumentChunk.java index 144897a..bba3173 100644 --- a/core/src/main/java/com/opencontext/core/entity/DocumentChunk.java +++ b/core/src/main/java/com/opencontext/entity/DocumentChunk.java @@ -1,4 +1,4 @@ -package com.opencontext.core.entity; +package com.opencontext.entity; import jakarta.persistence.*; import lombok.AccessLevel; diff --git a/core/src/main/java/com/opencontext/core/entity/SourceDocument.java b/core/src/main/java/com/opencontext/entity/SourceDocument.java similarity index 98% rename from core/src/main/java/com/opencontext/core/entity/SourceDocument.java rename to core/src/main/java/com/opencontext/entity/SourceDocument.java index 1347531..bfacbd7 100644 --- a/core/src/main/java/com/opencontext/core/entity/SourceDocument.java +++ b/core/src/main/java/com/opencontext/entity/SourceDocument.java @@ -1,6 +1,6 @@ -package com.opencontext.core.entity; +package com.opencontext.entity; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.enums.IngestionStatus; import jakarta.persistence.*; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java b/core/src/main/java/com/opencontext/repository/DocumentChunkRepository.java similarity index 97% rename from core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java rename to core/src/main/java/com/opencontext/repository/DocumentChunkRepository.java index bb97dff..72e2ccd 100644 --- a/core/src/main/java/com/opencontext/core/repository/DocumentChunkRepository.java +++ b/core/src/main/java/com/opencontext/repository/DocumentChunkRepository.java @@ -1,7 +1,7 @@ -package com.opencontext.core.repository; +package com.opencontext.repository; -import com.opencontext.core.entity.DocumentChunk; -import com.opencontext.core.entity.SourceDocument; +import com.opencontext.entity.DocumentChunk; +import com.opencontext.entity.SourceDocument; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java b/core/src/main/java/com/opencontext/repository/SourceDocumentRepository.java similarity index 97% rename from core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java rename to core/src/main/java/com/opencontext/repository/SourceDocumentRepository.java index d67c60e..f21779a 100644 --- a/core/src/main/java/com/opencontext/core/repository/SourceDocumentRepository.java +++ b/core/src/main/java/com/opencontext/repository/SourceDocumentRepository.java @@ -1,7 +1,7 @@ -package com.opencontext.core.repository; +package com.opencontext.repository; -import com.opencontext.core.entity.SourceDocument; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.entity.SourceDocument; +import com.opencontext.enums.IngestionStatus; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java b/core/src/test/java/com/opencontext/entity/DocumentChunkTest.java similarity index 98% rename from core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java rename to core/src/test/java/com/opencontext/entity/DocumentChunkTest.java index 79072b9..279c390 100644 --- a/core/src/test/java/com/opencontext/core/entity/DocumentChunkTest.java +++ b/core/src/test/java/com/opencontext/entity/DocumentChunkTest.java @@ -1,6 +1,6 @@ -package com.opencontext.core.entity; +package com.opencontext.entity; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.enums.IngestionStatus; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java b/core/src/test/java/com/opencontext/entity/SourceDocumentTest.java similarity index 98% rename from core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java rename to core/src/test/java/com/opencontext/entity/SourceDocumentTest.java index f146348..1ea5de4 100644 --- a/core/src/test/java/com/opencontext/core/entity/SourceDocumentTest.java +++ b/core/src/test/java/com/opencontext/entity/SourceDocumentTest.java @@ -1,6 +1,6 @@ -package com.opencontext.core.entity; +package com.opencontext.entity; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.enums.IngestionStatus; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java b/core/src/test/java/com/opencontext/repository/DocumentChunkRepositoryTest.java similarity index 98% rename from core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java rename to core/src/test/java/com/opencontext/repository/DocumentChunkRepositoryTest.java index d4b6efb..a76d11c 100644 --- a/core/src/test/java/com/opencontext/core/repository/DocumentChunkRepositoryTest.java +++ b/core/src/test/java/com/opencontext/repository/DocumentChunkRepositoryTest.java @@ -1,8 +1,8 @@ -package com.opencontext.core.repository; +package com.opencontext.repository; -import com.opencontext.core.entity.DocumentChunk; -import com.opencontext.core.entity.SourceDocument; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.entity.DocumentChunk; +import com.opencontext.entity.SourceDocument; +import com.opencontext.enums.IngestionStatus; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; diff --git a/core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java b/core/src/test/java/com/opencontext/repository/SourceDocumentRepositoryTest.java similarity index 98% rename from core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java rename to core/src/test/java/com/opencontext/repository/SourceDocumentRepositoryTest.java index 1967ac3..08eef76 100644 --- a/core/src/test/java/com/opencontext/core/repository/SourceDocumentRepositoryTest.java +++ b/core/src/test/java/com/opencontext/repository/SourceDocumentRepositoryTest.java @@ -1,7 +1,7 @@ -package com.opencontext.core.repository; +package com.opencontext.repository; -import com.opencontext.core.entity.SourceDocument; -import com.opencontext.core.enums.IngestionStatus; +import com.opencontext.entity.SourceDocument; +import com.opencontext.enums.IngestionStatus; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; From 0c48280f185790b526b4c605d7b8a25a13e3f087 Mon Sep 17 00:00:00 2001 From: Jihyeok Date: Sun, 10 Aug 2025 12:31:57 +0900 Subject: [PATCH 6/6] Update core/gradlew.bat Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- core/gradlew.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/gradlew.bat b/core/gradlew.bat index 8286633..52bfe2c 100644 --- a/core/gradlew.bat +++ b/core/gradlew.bat @@ -17,7 +17,8 @@ @rem SPDX-License-Identifier: Apache-2.0 @rem ======= ->>>>>>> 2b49148 (build: regenerate gradle wrapper for consistent build environment) +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ##########################################################################