feat: implement structured chunk data structures and development security config

core/src/main/java/com/opencontext/config/SecurityConfig.java

@@ -0,0 +1,78 @@
+package com.opencontext.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
+import org.springframework.security.config.http.SessionCreationPolicy;
+import org.springframework.security.web.SecurityFilterChain;
+import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Spring Security configuration for OpenContext API.
+ * Implements API Key authentication for Admin APIs and allows unrestricted access
+ * to MCP APIs and development endpoints.
+ */
+@Slf4j
+@Configuration
+@EnableWebSecurity
+@RequiredArgsConstructor
+public class SecurityConfig {
+
+    private final Environment environment;
+
+    @Bean
+    public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
+        log.info("Configuring Spring Security with profile-specific rules");
+
+        http
+            // Disable CSRF for REST API
+            .csrf(AbstractHttpConfigurer::disable)
+
+            // Disable form login and basic auth
+            .formLogin(AbstractHttpConfigurer::disable)
+            .httpBasic(AbstractHttpConfigurer::disable)
+
+            // Stateless session management for REST API
+            .sessionManagement(session -> 
+                session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
+
+            // Configure authorization rules
+            .authorizeHttpRequests(authz -> authz
+                // Allow health check endpoints
+                .requestMatchers("/actuator/health/**", "/actuator/info").permitAll()
+
+                // Allow Swagger UI and API documentation (development)
+                .requestMatchers(
+                    "/swagger-ui/**", 
+                    "/v3/api-docs/**", 
+                    "/swagger-ui.html",
+                    "/swagger-resources/**",
+                    "/webjars/**"
+                ).permitAll()
+
+                // Allow MCP API endpoints (no authentication required per PRD)
+                .requestMatchers("/api/v1/search/**", "/api/v1/get-content/**").permitAll()
+
+                // Allow development mock data endpoints (development only)
+                .requestMatchers("/api/v1/dev/**").permitAll()
+
+                // Admin APIs will require API Key authentication (TODO: implement in next step)
+                .requestMatchers("/api/v1/sources/**").permitAll() // Temporary - will add API key auth
+
+                // All other requests require authentication
+                .anyRequest().authenticated()
+            );
+
+        // TODO: Add API Key authentication filter for Admin APIs
+        // http.addFilterBefore(apiKeyAuthenticationFilter, UsernamePasswordAuthenticationFilter.class);
+
+        log.info("Security configuration completed successfully");
+        return http.build();
+    }
+}

core/src/main/java/com/opencontext/dto/ChunkMetadata.java

@@ -0,0 +1,68 @@
+package com.opencontext.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.PositiveOrZero;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+/**
+ * Hierarchical and contextual metadata for document chunks.
+ * Supports document structure navigation and search result organization.
+ */
+@Schema(description = "Hierarchical and contextual metadata for document chunks")
+@Getter
+@Builder
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor
+public class ChunkMetadata {
+
+    /**
+     * Title or heading of this chunk.
+     * Used for displaying chunk context to users in search results.
+     */
+    @Schema(description = "Title or heading of the chunk", example = "5.8.2. JWT Authentication Converter")
+    private String title;
+
+    /**
+     * Hierarchical breadcrumbs from document root to this chunk.
+     * Stored as array for flexibility in UI display and filtering.
+     * ES keyword type naturally supports arrays.
+     */
+    @Schema(description = "Breadcrumb path from root to current chunk", 
+            example = "[\"Chapter 5\", \"Security\", \"JWT\", \"Configuration\"]")
+    private List<String> breadcrumbs;
+
+    /**
+     * Depth level in document hierarchy (0 for root chunks).
+     * Used for hierarchy-aware search and display organization.
+     */
+    @Schema(description = "Hierarchical depth level (0 for root)", example = "2")
+    @PositiveOrZero
+    private Integer hierarchyLevel;
+
+    /**
+     * Sequential order within document structure.
+     * Maintains original document flow for coherent retrieval.
+     */
+    @Schema(description = "Sequential position in document", example = "15")
+    @PositiveOrZero
+    private Integer sequenceInDocument;
+
+    /**
+     * Document language for proper text processing.
+     * Currently supports Korean (ko) and English (en).
+     */
+    @Schema(description = "Document language code", example = "ko")
+    private String language;
+
+    /**
+     * Source file type for context and processing hints.
+     */
+    @Schema(description = "Source file type", example = "PDF")
+    private String fileType;
+}

core/src/main/java/com/opencontext/dto/GetContentResponse.java

@@ -0,0 +1,38 @@
+package com.opencontext.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * Response DTO for the get_content API endpoint.
+ * Provides complete chunk content with token information for LLM context management.
+ */
+@Schema(description = "Response for focused content retrieval (get_content)")
+@Getter
+@Builder
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor
+public class GetContentResponse {
+
+    /**
+     * Complete text content of the requested chunk.
+     * May be truncated if exceeds maxTokens limit, but still returns 200 OK.
+     */
+    @Schema(description = "Complete chunk content (may be token-limited)", 
+            requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotBlank
+    private String content;
+
+    /**
+     * Token processing information for LLM context management.
+     */
+    @Schema(description = "Token processing information", requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotNull
+    private TokenInfo tokenInfo;
+}

core/src/main/java/com/opencontext/dto/SearchResultItem.java

@@ -0,0 +1,67 @@
+package com.opencontext.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.DecimalMax;
+import jakarta.validation.constraints.DecimalMin;
+import jakarta.validation.constraints.NotNull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Individual search result item for exploratory knowledge discovery.
+ * Contains essential information for users to identify and select relevant chunks.
+ */
+@Schema(description = "Individual search result item for exploratory search (find_knowledge)")
+@Getter
+@Builder
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor
+public class SearchResultItem {
+
+    /**
+     * Unique identifier of the chunk for subsequent get_content requests.
+     */
+    @Schema(description = "Chunk identifier for content retrieval", requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotNull
+    private UUID chunkId;
+
+    /**
+     * Title or heading of the chunk for user context.
+     */
+    @Schema(description = "Chunk title or heading", example = "5.8.2. JWT Authentication Converter")
+    private String title;
+
+    /**
+     * Content preview generated by extracting first 50 characters + "...".
+     * Following PRD specification for simple extraction over AI summarization.
+     */
+    @Schema(description = "Content preview (first 50 chars + '...')", 
+            example = "To customize the conversion from a JWT to an Auth...")
+    private String snippet;
+
+    /**
+     * Relevance score from hybrid search (BM25 + Vector similarity).
+     * Normalized to 0.0-1.0 range for consistent interpretation.
+     * Always present in search results.
+     */
+    @Schema(description = "Search relevance score", example = "0.92", minimum = "0.0", maximum = "1.0")
+    @DecimalMin(value = "0.0")
+    @DecimalMax(value = "1.0")
+    @NotNull
+    private Double relevanceScore;
+
+    /**
+     * Hierarchical breadcrumbs for user navigation context.
+     * Helps users understand the chunk's position within document structure.
+     * Optional field that enhances UX when available.
+     */
+    @Schema(description = "Breadcrumb path for navigation context", 
+            example = "[\"Chapter 5\", \"Security\", \"JWT\", \"Configuration\"]")
+    private List<String> breadcrumbs;
+}

core/src/main/java/com/opencontext/dto/StructuredChunk.java

@@ -0,0 +1,71 @@
+package com.opencontext.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.util.UUID;
+
+/**
+ * Structured document chunk for Elasticsearch indexing and storage.
+ * Contains content, embeddings, and hierarchical metadata for semantic search.
+ */
+@Schema(description = "Elasticsearch document structure for structured chunks")
+@Getter
+@Builder
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor
+public class StructuredChunk {
+
+    /**
+     * Unique identifier for the chunk, identical to PostgreSQL document_chunks.id.
+     * This maintains data consistency between relational and search stores.
+     */
+    @Schema(description = "Unique chunk identifier", requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotNull
+    private UUID chunkId;
+
+    /**
+     * UUID of the source document this chunk belongs to.
+     * Used for filtering and document-level operations.
+     */
+    @Schema(description = "Source document identifier", requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotNull
+    private UUID sourceDocumentId;
+
+    /**
+     * Original filename of the source document.
+     * Provides context for users to identify document source.
+     */
+    @Schema(description = "Original filename of source document", example = "spring-security-guide.pdf")
+    private String originalFilename;
+
+    /**
+     * The actual text content of this chunk.
+     * This is the primary searchable content processed by Korean Nori analyzer.
+     */
+    @Schema(description = "Text content of the chunk", requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotBlank
+    private String content;
+
+    /**
+     * 1024-dimensional embedding vector for semantic search.
+     * Generated by Qwen3-Embedding-0.6B model via Ollama.
+     * Uses float[] for optimal performance with Elasticsearch dense_vector type.
+     */
+    @Schema(description = "Embedding vector for semantic search (1024 dimensions)")
+    private float[] embedding;
+
+    /**
+     * Metadata containing hierarchical and contextual information.
+     * Separated into its own class following PRD coding standards.
+     */
+    @Schema(description = "Chunk metadata including hierarchy and context")
+    @NotNull
+    private ChunkMetadata metadata;
+}

core/src/main/java/com/opencontext/dto/TokenInfo.java

@@ -0,0 +1,40 @@
+package com.opencontext.dto;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.PositiveOrZero;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * Token processing information for LLM context management.
+ */
+@Schema(description = "Token processing information for LLM context management")
+@Getter
+@Builder
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor
+public class TokenInfo {
+
+    /**
+     * Name of the tokenizer used for token counting.
+     * Currently fixed to tiktoken-cl100k_base but extensible for future LLM support.
+     */
+    @Schema(description = "Tokenizer used for counting", example = "tiktoken-cl100k_base", 
+            requiredMode = Schema.RequiredMode.REQUIRED)
+    @NotBlank
+    private String tokenizer;
+
+    /**
+     * Actual number of tokens in the returned content.
+     * May be less than original if content was truncated due to maxTokens limit.
+     * Can be 0 in extreme cases (empty content after processing).
+     */
+    @Schema(description = "Actual token count of returned content", example = "7999", 
+            requiredMode = Schema.RequiredMode.REQUIRED)
+    @PositiveOrZero
+    private Integer actualTokens;
+}

core/src/main/resources/application.yml

@@ -18,13 +18,17 @@ spring:
   # JPA Configuration
   jpa:
     hibernate:
-      ddl-auto: create-drop
+      ddl-auto: update
     show-sql: true
     properties:
       hibernate:
         dialect: org.hibernate.dialect.PostgreSQLDialect
         format_sql: true
 
+  # Flyway Configuration
+  flyway:
+    baseline-on-migrate: true
+
   # Servlet Configuration
   servlet:
     multipart: