diff --git a/.jules/bolt.md b/.jules/bolt.md index 19b4c613..f61962d1 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -40,6 +40,3 @@ ## 2025-01-24 - 단일 readAttributes 호출로 파일 속성 조회 최적화 **학습:** `isDirectory`, `!it.isDirectory()`, `isSymbolicLink` 3개의 개별적인 파일 시스템 I/O 호출을 수행하면 성능 저하가 큽니다. 이를 단일 `Files.readAttributes` 호출로 변경하여 메타데이터를 한 번에 조회함으로써 I/O 오버헤드를 대폭 줄일 수 있음을 확인했습니다. **조치:** 디렉토리 순회 시 파일의 여러 속성을 확인할 때는 개별적인 stat 호출보다 `Files.readAttributes`를 사용하여 필요한 모든 속성을 한 번에 가져오는 방식을 우선적으로 고려해야 합니다. -## 2025-01-24 - 단일 readAttributes 호출로 파일 속성 조회 최적화 -**학습:** `isDirectory`, `!it.isDirectory()`, `isSymbolicLink` 3개의 개별적인 파일 시스템 I/O 호출을 수행하면 성능 저하가 큽니다. 이를 단일 `Files.readAttributes` 호출로 변경하여 메타데이터를 한 번에 조회함으로써 I/O 오버헤드를 대폭 줄일 수 있음을 확인했습니다. -**조치:** 디렉토리 순회 시 파일의 여러 속성을 확인할 때는 개별적인 stat 호출보다 `Files.readAttributes`를 사용하여 필요한 모든 속성을 한 번에 가져오는 방식을 우선적으로 고려해야 합니다. diff --git a/src/main/kotlin/html4tree/main.kt b/src/main/kotlin/html4tree/main.kt index f52a1468..0c26dfdc 100644 --- a/src/main/kotlin/html4tree/main.kt +++ b/src/main/kotlin/html4tree/main.kt @@ -1,6 +1,7 @@ package html4tree import java.io.File +import java.io.IOException import java.security.MessageDigest import java.nio.file.Files import java.nio.file.LinkOption @@ -119,6 +120,21 @@ internal fun read_file_identity(file: File): FileIdentity { } } +internal fun read_basic_file_attributes( + file: File, + reader: (File) -> BasicFileAttributes = { + Files.readAttributes(it.toPath(), BasicFileAttributes::class.java, LinkOption.NOFOLLOW_LINKS) + } +): BasicFileAttributes? { + return try { + reader(file) + } catch (e: IOException) { + null + } catch (e: SecurityException) { + null + } +} + fun go(topDir: String, maxLevel: Int) { require(topDir.isNotBlank()) require(!topDir.contains("..")) { "Path traversal sequences are not allowed." } @@ -144,14 +160,14 @@ internal fun crawl_directories( processDirectory: (File, Set, Array?) -> Unit = { file, exclude, files -> process_dir(file, exclude, files) }, processIgnoreFile: (File, Array?) -> Set = { file, names -> process_ignore_file(file, names) }, listFiles: (File) -> Array? = { it.listFiles() }, - isDirectory: (File) -> Boolean = { Files.isDirectory(it.toPath(), LinkOption.NOFOLLOW_LINKS) }, - isSymbolicLink: (File) -> Boolean = { Files.isSymbolicLink(it.toPath()) }, + readAttributes: (File) -> BasicFileAttributes? = { read_basic_file_attributes(it) }, readIdentity: (File) -> FileIdentity = ::read_file_identity ) { var lle: LinkedListEntry? = ll.pull() while(lle != null){ - if (!isDirectory(lle.file)) { + val lleAttrs = readAttributes(lle.file) + if (lleAttrs == null || !lleAttrs.isDirectory) { lle = ll.pull() continue } @@ -176,9 +192,19 @@ internal fun crawl_directories( dirFiles?.forEach { // ⚡ Bolt Performance Optimization: Short-circuit OS stat calls (isDirectory/isSymbolicLink) // by checking cheap in-memory string exclusion rules first - if(!it.name.startsWith(".") && it.name !in exclude && isDirectory(it) && !isSymbolicLink(it)) { - val childEntry = LinkedListEntry(it, currentLevel+1, readIdentity(it).key) - ll.push(childEntry) + if(!it.name.startsWith(".") && it.name !in exclude) { + val itAttrs = readAttributes(it) + if (itAttrs != null && itAttrs.isDirectory && !itAttrs.isSymbolicLink) { + val childIdentity = readIdentity(it) + if (childIdentity.readable) { + val childEntry = LinkedListEntry( + it, + currentLevel + 1, + childIdentity.key + ) + ll.push(childEntry) + } + } } } } @@ -405,4 +431,4 @@ fun process_dir(curr_dir: File, excludeSet: Set? = null, dirFiles: Array fun help() { println("ERROR: help has not been written yet!") -} +} \ No newline at end of file diff --git a/src/test/kotlin/html4tree/CrawlDirectoriesIdentityRegressionTest.kt b/src/test/kotlin/html4tree/CrawlDirectoriesIdentityRegressionTest.kt new file mode 100644 index 00000000..092968b9 --- /dev/null +++ b/src/test/kotlin/html4tree/CrawlDirectoriesIdentityRegressionTest.kt @@ -0,0 +1,91 @@ +package html4tree + +import java.io.File +import java.io.IOException +import java.nio.file.Files +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime +import org.junit.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** Regression coverage for fail-closed directory identity acquisition. */ +class CrawlDirectoriesIdentityRegressionTest { + /** Creates deterministic directory attributes without touching the host filesystem. */ + private fun directoryAttributes(): BasicFileAttributes { + return object : BasicFileAttributes { + override fun lastModifiedTime(): FileTime = FileTime.fromMillis(0) + override fun lastAccessTime(): FileTime = FileTime.fromMillis(0) + override fun creationTime(): FileTime = FileTime.fromMillis(0) + override fun isRegularFile(): Boolean = false + override fun isDirectory(): Boolean = true + override fun isSymbolicLink(): Boolean = false + override fun isOther(): Boolean = false + override fun size(): Long = 0 + override fun fileKey(): Any? = null + } + } + + /** + * A child whose identity cannot be read must not enter the queue, even when + * the same path later resolves to a readable replacement directory. + */ + @Test + fun unreadableChildIdentityIsNotEnqueuedBeforePathReplacement() { + val root = Files.createTempDirectory("html4tree-identity-root-").toFile() + val child = File(root, "child").apply { mkdir() } + val processed = mutableListOf() + val identityCalls = mutableMapOf() + val queue = LinkedList() + queue.push(LinkedListEntry(root, 0, "root-key")) + + try { + crawl_directories( + queue, + -1, + processDirectory = { file, _, _ -> processed.add(file) }, + processIgnoreFile = { _, _ -> emptySet() }, + listFiles = { file -> if (file == root) arrayOf(child) else emptyArray() }, + readAttributes = { directoryAttributes() }, + readIdentity = { file -> + val callCount = identityCalls.getOrDefault(file, 0) + identityCalls[file] = callCount + 1 + when (file) { + root -> FileIdentity("root-key", true) + child -> if (callCount == 0) { + FileIdentity(null, false) + } else { + FileIdentity("replacement-key", true) + } + else -> FileIdentity(null, false) + } + } + ) + + assertEquals(listOf(root), processed) + assertEquals(1, identityCalls[child], "unreadable child must never be dequeued") + } finally { + root.deleteRecursively() + } + } + + /** Expected I/O failures are converted to an absent attribute snapshot. */ + @Test + fun basicAttributeReaderSkipsIoFailures() { + val candidate = File("missing") + + assertNull(read_basic_file_attributes(candidate) { throw IOException("missing") }) + } + + /** Expected access-control failures are converted to an absent attribute snapshot. */ + @Test + fun basicAttributeReaderSkipsSecurityFailures() { + val candidate = File("restricted") + + assertNull( + read_basic_file_attributes(candidate) { + throw SecurityException("restricted") + } + ) + } +} diff --git a/src/test/kotlin/html4tree/MainTest.kt b/src/test/kotlin/html4tree/MainTest.kt index 83739c9c..bd70dacd 100644 --- a/src/test/kotlin/html4tree/MainTest.kt +++ b/src/test/kotlin/html4tree/MainTest.kt @@ -8,12 +8,28 @@ import java.io.ByteArrayOutputStream import java.io.File import java.io.PrintStream import java.nio.file.Files +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertNull import kotlin.test.assertTrue +fun createMockAttributes(isDirectory: Boolean, isSymbolicLink: Boolean): BasicFileAttributes { + return object : BasicFileAttributes { + override fun lastModifiedTime(): FileTime = FileTime.fromMillis(0) + override fun lastAccessTime(): FileTime = FileTime.fromMillis(0) + override fun creationTime(): FileTime = FileTime.fromMillis(0) + override fun isRegularFile(): Boolean = !isDirectory && !isSymbolicLink + override fun isDirectory(): Boolean = isDirectory + override fun isSymbolicLink(): Boolean = isSymbolicLink + override fun isOther(): Boolean = false + override fun size(): Long = 0 + override fun fileKey(): Any? = null + } +} + class MainTest { private lateinit var tempDir: File @@ -154,8 +170,7 @@ class MainTest { processDirectory = { file, _, _ -> processed.add(file) }, processIgnoreFile = { _, _ -> emptySet() }, listFiles = { emptyArray() }, - isDirectory = { true }, - isSymbolicLink = { false }, + readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) }, readIdentity = { FileIdentity("after-swap", true) } ) @@ -176,8 +191,7 @@ class MainTest { processDirectory = { file, _, _ -> processed.add(file) }, processIgnoreFile = { _, _ -> emptySet() }, listFiles = { emptyArray() }, - isDirectory = { true }, - isSymbolicLink = { false }, + readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) }, readIdentity = { FileIdentity(null, false) } ) @@ -200,8 +214,7 @@ class MainTest { processDirectory = { file, _, _ -> processed.add(file) }, processIgnoreFile = { _, _ -> emptySet() }, listFiles = { file -> if (file == root) arrayOf(child) else emptyArray() }, - isDirectory = { true }, - isSymbolicLink = { false }, + readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) }, readIdentity = { file -> val key = file.absolutePath val callCount = callsByPath.getOrDefault(key, 0) @@ -239,8 +252,7 @@ class MainTest { processDirectory = { file, _, _ -> processed.add(file) }, processIgnoreFile = { _, _ -> emptySet() }, listFiles = { emptyArray() }, - isDirectory = { it == directoryEntry }, - isSymbolicLink = { false }, + readAttributes = { createMockAttributes(isDirectory = it == directoryEntry, isSymbolicLink = false) }, readIdentity = { FileIdentity("directory-key", true) } ) @@ -698,12 +710,54 @@ class MainTest { listed = true emptyArray() }, - isDirectory = { true }, - isSymbolicLink = { false }, + readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) }, readIdentity = { FileIdentity("current-key", true) } ) assertFalse(processed, "fileKey mismatch should skip directory processing") assertFalse(listed, "fileKey mismatch should skip child listing") } + + @Test + fun testCrawlDirectoriesDefaultLambdaException() { + val missingDir = File(tempDir, "missing-dir") + val queue = LinkedList() + queue.push(LinkedListEntry(missingDir, 0, null)) + + val processedDirs = mutableListOf() + + crawl_directories( + ll = queue, + maxLevel = -1, + processDirectory = { file, _, _ -> processedDirs.add(file) }, + listFiles = { null } + // Using default readAttributes which will throw NoSuchFileException and return null + ) + + assertEquals(0, processedDirs.size) + } + + @Test + fun testCrawlDirectoriesDefaultLambdas() { + val root = File(tempDir, "default-root") + root.mkdir() + val child = File(root, "child") + child.mkdir() + val queue = LinkedList() + queue.push(LinkedListEntry(root, 0, read_file_identity(root).key)) + + val processedDirs = mutableListOf() + + crawl_directories( + ll = queue, + maxLevel = -1, + processDirectory = { file, _, _ -> processedDirs.add(file) }, + listFiles = { it.listFiles() } + // Using default readAttributes and readIdentity + ) + + assertEquals(2, processedDirs.size) + assertTrue(processedDirs.contains(root)) + assertTrue(processedDirs.contains(child)) + } }