⚡ Bolt: Optimize file stat OS calls in crawl_directories with a single readAttributes call

.jules/bolt.md

@@ -40,6 +40,3 @@
 ## 2025-01-24 - 단일 readAttributes 호출로 파일 속성 조회 최적화
 **학습:** `isDirectory`, `!it.isDirectory()`, `isSymbolicLink` 3개의 개별적인 파일 시스템 I/O 호출을 수행하면 성능 저하가 큽니다. 이를 단일 `Files.readAttributes` 호출로 변경하여 메타데이터를 한 번에 조회함으로써 I/O 오버헤드를 대폭 줄일 수 있음을 확인했습니다.
 **조치:** 디렉토리 순회 시 파일의 여러 속성을 확인할 때는 개별적인 stat 호출보다 `Files.readAttributes`를 사용하여 필요한 모든 속성을 한 번에 가져오는 방식을 우선적으로 고려해야 합니다.
-## 2025-01-24 - 단일 readAttributes 호출로 파일 속성 조회 최적화
-**학습:** `isDirectory`, `!it.isDirectory()`, `isSymbolicLink` 3개의 개별적인 파일 시스템 I/O 호출을 수행하면 성능 저하가 큽니다. 이를 단일 `Files.readAttributes` 호출로 변경하여 메타데이터를 한 번에 조회함으로써 I/O 오버헤드를 대폭 줄일 수 있음을 확인했습니다.
-**조치:** 디렉토리 순회 시 파일의 여러 속성을 확인할 때는 개별적인 stat 호출보다 `Files.readAttributes`를 사용하여 필요한 모든 속성을 한 번에 가져오는 방식을 우선적으로 고려해야 합니다.

src/main/kotlin/html4tree/main.kt

@@ -1,6 +1,7 @@
 package html4tree
 
 import java.io.File
+import java.io.IOException
 import java.security.MessageDigest
 import java.nio.file.Files
 import java.nio.file.LinkOption
@@ -119,6 +120,21 @@ internal fun read_file_identity(file: File): FileIdentity {
     }
 }
 
+internal fun read_basic_file_attributes(
+    file: File,
+    reader: (File) -> BasicFileAttributes = {
+        Files.readAttributes(it.toPath(), BasicFileAttributes::class.java, LinkOption.NOFOLLOW_LINKS)
+    }
+): BasicFileAttributes? {
+    return try {
+        reader(file)
+    } catch (e: IOException) {
+        null
+    } catch (e: SecurityException) {
+        null
+    }
+}
+
 fun go(topDir: String, maxLevel: Int)  {
     require(topDir.isNotBlank())
     require(!topDir.contains("..")) { "Path traversal sequences are not allowed." }
@@ -144,14 +160,14 @@ internal fun crawl_directories(
     processDirectory: (File, Set<String>, Array<File>?) -> Unit = { file, exclude, files -> process_dir(file, exclude, files) },
     processIgnoreFile: (File, Array<String>?) -> Set<String> = { file, names -> process_ignore_file(file, names) },
     listFiles: (File) -> Array<File>? = { it.listFiles() },
-    isDirectory: (File) -> Boolean = { Files.isDirectory(it.toPath(), LinkOption.NOFOLLOW_LINKS) },
-    isSymbolicLink: (File) -> Boolean = { Files.isSymbolicLink(it.toPath()) },
+    readAttributes: (File) -> BasicFileAttributes? = { read_basic_file_attributes(it) },
     readIdentity: (File) -> FileIdentity = ::read_file_identity
 ) {
     var lle: LinkedListEntry? = ll.pull()
 
     while(lle != null){
-        if (!isDirectory(lle.file)) {
+        val lleAttrs = readAttributes(lle.file)
+        if (lleAttrs == null || !lleAttrs.isDirectory) {
             lle = ll.pull()
             continue
         }
@@ -176,9 +192,19 @@ internal fun crawl_directories(
             dirFiles?.forEach {
                 // ⚡ Bolt Performance Optimization: Short-circuit OS stat calls (isDirectory/isSymbolicLink)
                 // by checking cheap in-memory string exclusion rules first
-                if(!it.name.startsWith(".") && it.name !in exclude && isDirectory(it) && !isSymbolicLink(it)) {
-                    val childEntry = LinkedListEntry(it, currentLevel+1, readIdentity(it).key)
-                    ll.push(childEntry)
+                if(!it.name.startsWith(".") && it.name !in exclude) {
+                    val itAttrs = readAttributes(it)
+                    if (itAttrs != null && itAttrs.isDirectory && !itAttrs.isSymbolicLink) {
+                        val childIdentity = readIdentity(it)
+                        if (childIdentity.readable) {
+                            val childEntry = LinkedListEntry(
+                                it,
+                                currentLevel + 1,
+                                childIdentity.key
+                            )
+                            ll.push(childEntry)
+                        }
+                    }
                 }
             }
         }
@@ -405,4 +431,4 @@ fun process_dir(curr_dir: File, excludeSet: Set<String>? = null, dirFiles: Array
 
 fun help() {
     println("ERROR: help has not been written yet!")
-}
+}

src/test/kotlin/html4tree/CrawlDirectoriesIdentityRegressionTest.kt

@@ -0,0 +1,91 @@
+package html4tree
+
+import java.io.File
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.attribute.FileTime
+import org.junit.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNull
+
+/** Regression coverage for fail-closed directory identity acquisition. */
+class CrawlDirectoriesIdentityRegressionTest {
+    /** Creates deterministic directory attributes without touching the host filesystem. */
+    private fun directoryAttributes(): BasicFileAttributes {
+        return object : BasicFileAttributes {
+            override fun lastModifiedTime(): FileTime = FileTime.fromMillis(0)
+            override fun lastAccessTime(): FileTime = FileTime.fromMillis(0)
+            override fun creationTime(): FileTime = FileTime.fromMillis(0)
+            override fun isRegularFile(): Boolean = false
+            override fun isDirectory(): Boolean = true
+            override fun isSymbolicLink(): Boolean = false
+            override fun isOther(): Boolean = false
+            override fun size(): Long = 0
+            override fun fileKey(): Any? = null
+        }
+    }
+
+    /**
+     * A child whose identity cannot be read must not enter the queue, even when
+     * the same path later resolves to a readable replacement directory.
+     */
+    @Test
+    fun unreadableChildIdentityIsNotEnqueuedBeforePathReplacement() {
+        val root = Files.createTempDirectory("html4tree-identity-root-").toFile()
+        val child = File(root, "child").apply { mkdir() }
+        val processed = mutableListOf<File>()
+        val identityCalls = mutableMapOf<File, Int>()
+        val queue = LinkedList()
+        queue.push(LinkedListEntry(root, 0, "root-key"))
+
+        try {
+            crawl_directories(
+                queue,
+                -1,
+                processDirectory = { file, _, _ -> processed.add(file) },
+                processIgnoreFile = { _, _ -> emptySet() },
+                listFiles = { file -> if (file == root) arrayOf(child) else emptyArray() },
+                readAttributes = { directoryAttributes() },
+                readIdentity = { file ->
+                    val callCount = identityCalls.getOrDefault(file, 0)
+                    identityCalls[file] = callCount + 1
+                    when (file) {
+                        root -> FileIdentity("root-key", true)
+                        child -> if (callCount == 0) {
+                            FileIdentity(null, false)
+                        } else {
+                            FileIdentity("replacement-key", true)
+                        }
+                        else -> FileIdentity(null, false)
+                    }
+                }
+            )
+
+            assertEquals(listOf(root), processed)
+            assertEquals(1, identityCalls[child], "unreadable child must never be dequeued")
+        } finally {
+            root.deleteRecursively()
+        }
+    }
+
+    /** Expected I/O failures are converted to an absent attribute snapshot. */
+    @Test
+    fun basicAttributeReaderSkipsIoFailures() {
+        val candidate = File("missing")
+
+        assertNull(read_basic_file_attributes(candidate) { throw IOException("missing") })
+    }
+
+    /** Expected access-control failures are converted to an absent attribute snapshot. */
+    @Test
+    fun basicAttributeReaderSkipsSecurityFailures() {
+        val candidate = File("restricted")
+
+        assertNull(
+            read_basic_file_attributes(candidate) {
+                throw SecurityException("restricted")
+            }
+        )
+    }
+}

src/test/kotlin/html4tree/MainTest.kt

@@ -8,12 +8,28 @@ import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.PrintStream
 import java.nio.file.Files
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.attribute.FileTime
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 import kotlin.test.assertFalse
 import kotlin.test.assertNull
 import kotlin.test.assertTrue
 
+fun createMockAttributes(isDirectory: Boolean, isSymbolicLink: Boolean): BasicFileAttributes {
+    return object : BasicFileAttributes {
+        override fun lastModifiedTime(): FileTime = FileTime.fromMillis(0)
+        override fun lastAccessTime(): FileTime = FileTime.fromMillis(0)
+        override fun creationTime(): FileTime = FileTime.fromMillis(0)
+        override fun isRegularFile(): Boolean = !isDirectory && !isSymbolicLink
+        override fun isDirectory(): Boolean = isDirectory
+        override fun isSymbolicLink(): Boolean = isSymbolicLink
+        override fun isOther(): Boolean = false
+        override fun size(): Long = 0
+        override fun fileKey(): Any? = null
+    }
+}
+
 class MainTest {
     private lateinit var tempDir: File
 
@@ -154,8 +170,7 @@ class MainTest {
             processDirectory = { file, _, _ -> processed.add(file) },
             processIgnoreFile = { _, _ -> emptySet() },
             listFiles = { emptyArray() },
-            isDirectory = { true },
-            isSymbolicLink = { false },
+            readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) },
             readIdentity = { FileIdentity("after-swap", true) }
         )
 
@@ -176,8 +191,7 @@ class MainTest {
             processDirectory = { file, _, _ -> processed.add(file) },
             processIgnoreFile = { _, _ -> emptySet() },
             listFiles = { emptyArray() },
-            isDirectory = { true },
-            isSymbolicLink = { false },
+            readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) },
             readIdentity = { FileIdentity(null, false) }
         )
 
@@ -200,8 +214,7 @@ class MainTest {
             processDirectory = { file, _, _ -> processed.add(file) },
             processIgnoreFile = { _, _ -> emptySet() },
             listFiles = { file -> if (file == root) arrayOf(child) else emptyArray() },
-            isDirectory = { true },
-            isSymbolicLink = { false },
+            readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) },
             readIdentity = { file ->
                 val key = file.absolutePath
                 val callCount = callsByPath.getOrDefault(key, 0)
@@ -239,8 +252,7 @@ class MainTest {
             processDirectory = { file, _, _ -> processed.add(file) },
             processIgnoreFile = { _, _ -> emptySet() },
             listFiles = { emptyArray() },
-            isDirectory = { it == directoryEntry },
-            isSymbolicLink = { false },
+            readAttributes = { createMockAttributes(isDirectory = it == directoryEntry, isSymbolicLink = false) },
             readIdentity = { FileIdentity("directory-key", true) }
         )
 
@@ -698,12 +710,54 @@ class MainTest {
                 listed = true
                 emptyArray()
             },
-            isDirectory = { true },
-            isSymbolicLink = { false },
+            readAttributes = { createMockAttributes(isDirectory = true, isSymbolicLink = false) },
             readIdentity = { FileIdentity("current-key", true) }
         )
 
         assertFalse(processed, "fileKey mismatch should skip directory processing")
         assertFalse(listed, "fileKey mismatch should skip child listing")
     }
+
+    @Test
+    fun testCrawlDirectoriesDefaultLambdaException() {
+        val missingDir = File(tempDir, "missing-dir")
+        val queue = LinkedList()
+        queue.push(LinkedListEntry(missingDir, 0, null))
+
+        val processedDirs = mutableListOf<File>()
+
+        crawl_directories(
+            ll = queue,
+            maxLevel = -1,
+            processDirectory = { file, _, _ -> processedDirs.add(file) },
+            listFiles = { null }
+            // Using default readAttributes which will throw NoSuchFileException and return null
+        )
+
+        assertEquals(0, processedDirs.size)
+    }
+
+    @Test
+    fun testCrawlDirectoriesDefaultLambdas() {
+        val root = File(tempDir, "default-root")
+        root.mkdir()
+        val child = File(root, "child")
+        child.mkdir()
+        val queue = LinkedList()
+        queue.push(LinkedListEntry(root, 0, read_file_identity(root).key))
+
+        val processedDirs = mutableListOf<File>()
+
+        crawl_directories(
+            ll = queue,
+            maxLevel = -1,
+            processDirectory = { file, _, _ -> processedDirs.add(file) },
+            listFiles = { it.listFiles() }
+            // Using default readAttributes and readIdentity
+        )
+
+        assertEquals(2, processedDirs.size)
+        assertTrue(processedDirs.contains(root))
+        assertTrue(processedDirs.contains(child))
+    }
 }