Skip to content

⚡ Bolt: 불필요한 파일 시스템 I/O 호출 최소화 (listFiles 캐싱) #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
seonghobae wants to merge 1 commit into from
Closed

⚡ Bolt: 불필요한 파일 시스템 I/O 호출 최소화 (listFiles 캐싱) #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .jules/bolt.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,6 @@
## 2024-07-28 - 디렉토리 목록 불필요한 정렬 오버헤드
**학습:** 디렉토리 목록(`list()` 또는 `listFiles()`)을 단순히 필터링하여 `Set`에 추가하는 경우처럼 특정 순서가 필요하지 않은 작업에서 `.sorted()`를 호출하면 불필요한 O(N log N) 오버헤드가 발생합니다.
**조치:** `Set`과 같은 순서에 무관한 자료구조에 요소를 추가하기 위한 필터링 작업에서는 디렉토리 목록에서 `.sorted()` 호출을 제거하여 성능을 최적화합니다.
## 2024-07-12 - [Filesystem Access Optimization]
**Learning:** Calling `File.listFiles()` multiple times for the same directory is a significant performance bottleneck in deeply nested directory trees, causing expensive OS-level I/O operations per file or directory. This overhead compounds geometrically with the depth and width of the tree.
**Action:** Always fetch the directory contents once and pass the resulting array (`Array<File>?`) down to helper methods (like sorting and filtering) to eliminate redundant filesystem reads.
24 changes: 13 additions & 11 deletions src/main/kotlin/html4tree/main.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,14 @@ fun go(topDir: String, maxLevel: Int) {

while(lle != null && Files.isDirectory(lle.file.toPath(), LinkOption.NOFOLLOW_LINKS)){
val currentLevel: Int = lle.level
val dirFiles: Array<File>? = lle.file.listFiles()

if(maxLevel == -1 || currentLevel <= maxLevel)
process_dir(lle.file)
process_dir(lle.file, dirFiles)

if(maxLevel == -1 || currentLevel < maxLevel) {
val exclude = process_ignore_file(lle.file)
lle.file.listFiles()?.forEach {
val exclude = process_ignore_file(lle.file, dirFiles)
dirFiles?.forEach {
if(Files.isDirectory(it.toPath(), LinkOption.NOFOLLOW_LINKS) && !Files.isSymbolicLink(it.toPath()) && it.name !in exclude) {
ll.push( LinkedListEntry(it, currentLevel+1))
}
Expand Down Expand Up @@ -121,7 +123,7 @@ fun String.urlEncodePath(): String {
return encoded.toString()
}

fun process_ignore_file(curr_dir: File): Set<String> {
fun process_ignore_file(curr_dir: File, dirFiles: Array<File>?): Set<String> {

val ignore_filename = ".html4ignore"

Expand Down Expand Up @@ -152,8 +154,8 @@ fun process_ignore_file(curr_dir: File): Set<String> {
}

// ⚡ Bolt Performance Optimization: 디렉토리 목록을 Set에 추가하기 위해 필터링만 할 때는 정렬이 불필요하므로 .sorted()를 제거하여 O(N log N) 오버헤드를 방지합니다.
curr_dir.list()?.forEach {
val current = it
dirFiles?.forEach { file ->
val current = file.name
val pathCurrent = java.nio.file.Paths.get(current)
for (matcher in ignored_matchers) {
if (matcher.matches(pathCurrent)) {
Expand Down Expand Up @@ -185,9 +187,9 @@ fun write_index_file(curr_dir: File, content: String) {
}
}

fun process_dir(curr_dir: File){
fun process_dir(curr_dir: File, dirFiles: Array<File>?){

val exclude: Set<String> = process_ignore_file(curr_dir)
val exclude: Set<String> = process_ignore_file(curr_dir, dirFiles)
val styleNonce = generate_csp_nonce()

val css = """
Expand Down Expand Up @@ -265,9 +267,9 @@ fun process_dir(curr_dir: File){
val index_middle = fun():String{
val l = StringBuilder()

val dir_files: MutableList<File> = curr_dir.listFiles()?.toMutableList() ?: mutableListOf()
dir_files.sortWith(compareBy ({it.name}) )
dir_files.forEach {
val dir_files_list: MutableList<File> = dirFiles?.toMutableList() ?: mutableListOf()
dir_files_list.sortWith(compareBy ({it.name}) )
dir_files_list.forEach {
val fileName = it.getName()
// ⚡ Bolt Performance Optimization: Short-circuit string match before expensive OS filesystem calls
if (fileName !in exclude) {
Expand Down
2 changes: 1 addition & 1 deletion src/test/kotlin/html4tree/CoverageTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class CoverageTest {
readOnlyDir.mkdir()
readOnlyDir.setWritable(false, false)
try {
process_dir(readOnlyDir)
process_dir(readOnlyDir, readOnlyDir.listFiles())
// It should be handled securely
assertTrue(true)
} finally {
Expand Down
30 changes: 15 additions & 15 deletions src/test/kotlin/html4tree/MainTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ class MainTest {
File(tempDir, "test.log").createNewFile()
File(tempDir, "test.md").createNewFile()

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())

assertTrue(excluded.contains("test.txt"))
assertTrue(excluded.contains("test.log"))
Expand All @@ -123,7 +123,7 @@ class MainTest {

@Test
fun testProcessIgnoreFileNoIgnore() {
val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertTrue(excluded.contains("index.html"))
assertEquals(9, excluded.size) // index.html + 8 default sensitive files
}
Expand All @@ -136,7 +136,7 @@ class MainTest {
File(tempDir, "test.log").createNewFile()
File(tempDir, "test.txt").createNewFile()

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())

assertTrue(excluded.contains("test.log"))
assertFalse(excluded.contains("test.txt"))
Expand All @@ -150,7 +150,7 @@ class MainTest {
File(tempDir, "test.ignore").createNewFile()
File(tempDir, ".html4ignore").writeText("*.ignore")

process_dir(tempDir)
process_dir(tempDir, tempDir.listFiles())

val indexFile = File(tempDir, "index.html")
assertTrue(indexFile.exists())
Expand Down Expand Up @@ -214,7 +214,7 @@ class MainTest {
Assume.assumeTrue("Symlink creation not supported in this environment", false)
}

process_dir(tempDir)
process_dir(tempDir, tempDir.listFiles())

assertEquals("original content", targetFile.readText())
assertTrue(indexFile.exists())
Expand Down Expand Up @@ -287,7 +287,7 @@ class MainTest {
val notADirectory = File(tempDir, "not-a-directory")
notADirectory.writeText("content")

process_dir(notADirectory)
process_dir(notADirectory, notADirectory.listFiles())

assertEquals("content", notADirectory.readText())
}
Expand Down Expand Up @@ -349,14 +349,14 @@ class MainTest {
val ignoreFile = File(tempDir, ".html4ignore")
ignoreFile.writeText("index.html")
File(tempDir, "index.html").writeText("existing")
val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertTrue(excluded.contains("index.html"))
}

@Test
fun testProcessDirItEqualsCurrDir() {
File(tempDir, "tempDir").mkdir()
process_dir(tempDir)
process_dir(tempDir, tempDir.listFiles())
}

@Test(expected = IllegalArgumentException::class)
Expand Down Expand Up @@ -389,7 +389,7 @@ class MainTest {

File(tempDir, "test.txt").createNewFile()

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertTrue(excluded.contains("test.txt"))
}

Expand All @@ -399,7 +399,7 @@ class MainTest {
ignoreDir.mkdir()

// This should not crash or parse the directory
val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertTrue(excluded.contains("index.html"))
}

Expand All @@ -421,7 +421,7 @@ class MainTest {
File(tempDir, "pattern1005").createNewFile() // Should not be ignored as we stop at 1000
File(tempDir, longPattern).createNewFile() // Should not be ignored as length > 100

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())

assertTrue(excluded.contains("pattern500"))
assertFalse(excluded.contains("pattern1005"))
Expand All @@ -443,7 +443,7 @@ class MainTest {
File(tempDir, "test.txt").createNewFile()

// Should ignore the symlink and NOT parse it
val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertFalse(excluded.contains("test.txt"))
assertTrue(excluded.contains("index.html"))
}
Expand All @@ -458,7 +458,7 @@ class MainTest {
File(tempDir, "test.txt").createNewFile()

// Should ignore the file because it's too large
val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
assertFalse(excluded.contains("test.txt"))
assertTrue(excluded.contains("index.html"))
}
Expand All @@ -472,7 +472,7 @@ class MainTest {
File(tempDir, "test.log").createNewFile()
File(tempDir, "test.txt").createNewFile()

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
// .log is excluded because it's valid
assertTrue(excluded.contains("test.log"))
// test.txt is not excluded because long regex was ignored
Expand All @@ -492,7 +492,7 @@ class MainTest {
File(tempDir, "test.txt1000").createNewFile()
File(tempDir, "test.txt1001").createNewFile()

val excluded = process_ignore_file(tempDir)
val excluded = process_ignore_file(tempDir, tempDir.listFiles())
// Line 1000 should be processed
assertTrue(excluded.contains("test.txt1000"))
// Line 1001 should be ignored due to line limit
Expand Down
Loading