Reduce memory usage by using Reader and Writer instead of ByteArrays (#767)

* Reduce memory usage by using Reader and Writer instead of ByteArrays where possible.

Author: jan-goral

release_notes.md

@@ -1,4 +1,5 @@
 ## next (unreleased)
+- [#757](https://github.com/Flank/flank/pull/767) Reduce memory usage by using Reader and Writer instead of ByteArrays. ([jan-gogo](https://github.com/jan-gogo))
 - [#763](https://github.com/Flank/flank/pull/763) Use "localhost" as default for hostname to fix backward compatibility. ([jan-gogo](https://github.com/jan-gogo))
 - [#757](https://github.com/Flank/flank/pull/757) Print version and revision before each command. ([jan-gogo](https://github.com/jan-gogo))
 - [#759](https://github.com/Flank/flank/pull/759) Add shard name for uploaded xctestrun files. ([pawelpasterz](https://github.com/pawelpasterz))

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -1,5 +1,6 @@
 package ftl.args
 
+import com.google.common.annotations.VisibleForTesting
 import ftl.android.AndroidCatalog
 import ftl.android.IncompatibleModelVersion
 import ftl.android.SupportedDeviceConfig
@@ -29,6 +30,7 @@ import ftl.config.Device
 import ftl.config.FtlConstants
 import ftl.config.parseRoboDirectives
 import ftl.util.FlankFatalError
+import java.io.Reader
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -187,11 +189,11 @@ AndroidArgs
             mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml, AndroidFlankYml)
         }
 
-        fun load(data: Path, cli: AndroidRunCommand? = null): AndroidArgs =
-            load(String(Files.readAllBytes(data)), cli)
+        fun load(yamlPath: Path, cli: AndroidRunCommand? = null): AndroidArgs = load(Files.newBufferedReader(yamlPath), cli)
 
-        fun load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs {
-            val data = YamlDeprecated.modifyAndThrow(yamlData, android = true)
+        @VisibleForTesting
+        internal fun load(yamlReader: Reader, cli: AndroidRunCommand? = null): AndroidArgs {
+            val data = YamlDeprecated.modifyAndThrow(yamlReader, android = true)
 
             val flankYml = yamlMapper.readValue(data, FlankYml::class.java)
             val gcloudYml = yamlMapper.readValue(data, GcloudYml::class.java)
@@ -215,7 +217,8 @@ AndroidArgs
                 FlankYml(),
                 AndroidFlankYml(),
                 "",
-                AndroidRunCommand())
+                AndroidRunCommand()
+            )
         }
     }
 }

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -1,5 +1,6 @@
 package ftl.args
 
+import com.google.common.annotations.VisibleForTesting
 import ftl.args.ArgsHelper.assertCommonProps
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
@@ -24,6 +25,7 @@ import ftl.ios.IosCatalog
 import ftl.ios.Xctestrun
 import ftl.util.FlankTestMethod
 import ftl.util.FlankFatalError
+import java.io.Reader
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -156,10 +158,11 @@ IosArgs
             mergeYmlMaps(GcloudYml, IosGcloudYml, FlankYml, IosFlankYml)
         }
 
-        fun load(data: Path, cli: IosRunCommand? = null): IosArgs = load(String(Files.readAllBytes(data)), cli)
+        fun load(yamlPath: Path, cli: IosRunCommand? = null): IosArgs = load(Files.newBufferedReader(yamlPath), cli)
 
-        fun load(yamlData: String, cli: IosRunCommand? = null): IosArgs {
-            val data = YamlDeprecated.modifyAndThrow(yamlData, android = false)
+        @VisibleForTesting
+        internal fun load(yamlReader: Reader, cli: IosRunCommand? = null): IosArgs {
+            val data = YamlDeprecated.modifyAndThrow(yamlReader, android = false)
 
             val flankYml = yamlMapper.readValue(data, FlankYml::class.java)
             val iosFlankYml = yamlMapper.readValue(data, IosFlankYml::class.java)
@@ -183,7 +186,8 @@ IosArgs
                 FlankYml(),
                 IosFlankYml(),
                 "",
-                IosRunCommand())
+                IosRunCommand()
+            )
         }
     }
 }

test_runner/src/main/kotlin/ftl/args/yml/YamlDeprecated.kt

@@ -4,8 +4,10 @@ import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.databind.node.JsonNodeFactory
 import com.fasterxml.jackson.databind.node.MissingNode
 import com.fasterxml.jackson.databind.node.ObjectNode
+import com.google.common.annotations.VisibleForTesting
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.util.FlankFatalError
+import java.io.Reader
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -123,8 +125,7 @@ object YamlDeprecated {
     fun modify(yamlPath: Path): Boolean {
         if (yamlPath.toFile().exists().not()) throw FlankFatalError("Flank yml doesn't exist at path $yamlPath")
 
-        val data = String(Files.readAllBytes(yamlPath))
-        val (errorDetected, string) = modify(data)
+        val (errorDetected, string) = modify(Files.newBufferedReader(yamlPath))
 
         Files.write(yamlPath, string.toByteArray())
         println("\nUpdated ${yamlPath.fileName} file")
@@ -133,8 +134,8 @@ object YamlDeprecated {
     }
 
     // Throw exception when Level.Error modified key is found.
-    fun modifyAndThrow(yamlData: String, android: Boolean): String {
-        val (error, data) = YamlDeprecated.modify(yamlData)
+    fun modifyAndThrow(yamlReader: Reader, android: Boolean): String {
+        val (error, data) = modify(yamlReader)
 
         if (error) {
             val platform = if (android) "android" else "ios"
@@ -144,7 +145,8 @@ object YamlDeprecated {
         return data
     }
 
-    fun modify(yamlData: String): Pair<Boolean, String> {
+    @VisibleForTesting
+    internal fun modify(yamlData: Reader): Pair<Boolean, String> {
         val mappedYaml = yamlMapper.readTree(yamlData)
 
         val parsed = if (mappedYaml == null || mappedYaml is MissingNode) {

test_runner/src/main/kotlin/ftl/doctor/Doctor.kt

@@ -1,17 +1,20 @@
 package ftl.doctor
 
+import com.google.common.annotations.VisibleForTesting
 import ftl.args.ArgsHelper
 import ftl.args.IArgsCompanion
+import java.io.Reader
 import java.nio.file.Files
 import java.nio.file.Path
 
 object Doctor {
     fun validateYaml(args: IArgsCompanion, data: Path): String {
         if (!data.toFile().exists()) return "Skipping yaml validation. No file at path $data"
-        return validateYaml(args, String(Files.readAllBytes(data)))
+        return validateYaml(args, Files.newBufferedReader(data))
     }
 
-    fun validateYaml(args: IArgsCompanion, data: String): String {
+    @VisibleForTesting
+    internal fun validateYaml(args: IArgsCompanion, data: Reader): String {
         var result = ""
         val parsed = ArgsHelper.yamlMapper.readTree(data)
 

test_runner/src/main/kotlin/ftl/mock/MockServer.kt

@@ -57,9 +57,8 @@ object MockServer {
 
     private inline fun <reified T> loadCatalog(fileName: String): T {
         val jsonPath = Paths.get("./src/test/kotlin/ftl/fixtures/$fileName")
-        if (!jsonPath.toFile().exists()) throw RuntimeException("Path doesn't exist: $fileName")
-        val jsonString = String(Files.readAllBytes(jsonPath))
-        return JSON_FACTORY.fromString(jsonString, T::class.java)
+        if (!Files.exists(jsonPath)) throw RuntimeException("Path doesn't exist: $fileName")
+        return JSON_FACTORY.fromReader(Files.newBufferedReader(jsonPath), T::class.java)
     }
 
     private val androidCatalog by lazy { loadCatalog<AndroidDeviceCatalog>("android_catalog.json") }

test_runner/src/main/kotlin/ftl/reports/xml/JUnitXml.kt

@@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PRO
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.reports.xml.model.JUnitTestSuite
 import java.io.File
-import java.nio.file.Files
 import java.nio.file.Path
 
 private val xmlModule = JacksonXmlModule().apply { setDefaultUseWrapper(false) }
@@ -17,47 +16,36 @@ private val xmlMapper = XmlMapper(xmlModule)
 
 internal val xmlPrettyWriter = xmlMapper.writerWithDefaultPrettyPrinter()
 
-private fun xmlBytes(path: Path): ByteArray {
-    if (!path.toFile().exists()) throw RuntimeException("$path doesn't exist!")
-    return Files.readAllBytes(path)
-}
-
 fun JUnitTestResult?.xmlToString(): String {
     if (this == null) return ""
     val prefix = "<?xml version='1.0' encoding='UTF-8' ?>\n"
     return prefix + xmlPrettyWriter.writeValueAsString(this)
 }
 
-fun parseOneSuiteXml(bytes: ByteArray): JUnitTestResult {
-    return JUnitTestResult(mutableListOf(xmlMapper.readValue(bytes, JUnitTestSuite::class.java)))
-}
-
 fun parseOneSuiteXml(path: Path): JUnitTestResult {
-    return parseOneSuiteXml(xmlBytes(path))
+    return parseOneSuiteXml(path.toFile())
 }
 
-fun parseOneSuiteXml(path: File): JUnitTestResult {
-    return parseOneSuiteXml(xmlBytes(path.toPath()))
+fun parseOneSuiteXml(file: File): JUnitTestResult {
+    if (!file.exists()) throw RuntimeException("$file doesn't exist!")
+    return JUnitTestResult(mutableListOf(xmlMapper.readValue(file, JUnitTestSuite::class.java)))
 }
 
 fun parseOneSuiteXml(data: String): JUnitTestResult {
-    return parseOneSuiteXml(data.toByteArray())
+    return JUnitTestResult(mutableListOf(xmlMapper.readValue(data, JUnitTestSuite::class.java)))
 }
 
 // --
 
-fun parseAllSuitesXml(bytes: ByteArray): JUnitTestResult {
-    return xmlMapper.readValue(bytes, JUnitTestResult::class.java)
-}
-
 fun parseAllSuitesXml(path: Path): JUnitTestResult {
-    return parseAllSuitesXml(xmlBytes(path))
+    return parseAllSuitesXml(path.toFile())
 }
 
-fun parseAllSuitesXml(path: File): JUnitTestResult {
-    return parseAllSuitesXml(path.toPath())
+fun parseAllSuitesXml(file: File): JUnitTestResult {
+    if (!file.exists()) throw RuntimeException("$file doesn't exist!")
+    return xmlMapper.readValue(file, JUnitTestResult::class.java)
 }
 
 fun parseAllSuitesXml(data: String): JUnitTestResult {
-    return parseAllSuitesXml(data.toByteArray())
+    return xmlMapper.readValue(data, JUnitTestResult::class.java)
 }

test_runner/src/main/kotlin/ftl/util/Utils.kt

@@ -120,8 +120,10 @@ fun copyBinaryResource(name: String) {
     destinationPath.parent.toFile().mkdirs()
 
     // "binaries/" folder prefix is required for Linux to find the resource.
-    val bytes = getResource("binaries/$name").use { it.readBytes() }
-    Files.write(destinationPath, bytes)
+    Files.copy(
+        getResource("binaries/$name"),
+        destinationPath
+    )
     destinationFile.setExecutable(true)
 }
 

test_runner/src/test/kotlin/Debug.kt

@@ -10,8 +10,8 @@ fun main() {
     // run "gradle check" to generate required fixtures
     val projectId = System.getenv("GOOGLE_CLOUD_PROJECT")
         ?: "YOUR PROJECT ID"
-    val quantity = "single"
-    val type = "robo"
+    val quantity = "multiple"
+    val type = "error"
 
     // Bugsnag keeps the process alive so we must call exitProcess
     // https://github.com/bugsnag/bugsnag-java/issues/151

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -26,6 +26,7 @@ import org.junit.Test
 import org.junit.rules.ExpectedException
 import org.junit.runner.RunWith
 import picocli.CommandLine
+import java.io.StringReader
 
 @Suppress("TooManyFunctions")
 @RunWith(FlankTestRunner::class)
@@ -1242,3 +1243,5 @@ AndroidArgs
         runBlocking { runAndroidTests(parsedYml) }
     }
 }
+
+private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = load(StringReader(yamlData), cli)