diff --git a/Documentation/building/configuration.md b/Documentation/building/configuration.md index 9e409096049..c605a3dc277 100644 --- a/Documentation/building/configuration.md +++ b/Documentation/building/configuration.md @@ -118,6 +118,12 @@ Overridable MSBuild properties include: assemblies placed in the APK will be compressed in `Release` builds. `Debug` builds are not affected. + * `$(AndroidEnableAssemblyStoreDecompressionCache)`: Defaults to `False`. When + enabled for a CoreCLR `Release` build, decompressed assemblies are cached in + the app's Android code-cache directory and mapped from there on subsequent + launches. The cache consumes additional on-device storage and is rebuilt + after app or platform updates. + ## Options suitable for local development ### Native runtime (`src/native`) diff --git a/Documentation/project-docs/AssemblyStores.md b/Documentation/project-docs/AssemblyStores.md index a18e126ca39..117d8760f04 100644 --- a/Documentation/project-docs/AssemblyStores.md +++ b/Documentation/project-docs/AssemblyStores.md @@ -111,6 +111,7 @@ The header is a fixed-size structure at the beginning of each assembly store fil - **ENTRY_COUNT** (`uint32_t`) - Number of assemblies in the store - **INDEX_ENTRY_COUNT** (`uint32_t`) - Number of entries in the index (typically `ENTRY_COUNT * 2`) - **INDEX_SIZE** (`uint32_t`) - Index size in bytes +- **CONTENT_ID** (`uint64_t`) - Deterministic xxHash3 of everything after the header ## [INDEX] @@ -168,6 +169,7 @@ All kinds of stores share the following header format: uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; Individual fields have the following meanings: @@ -179,6 +181,7 @@ Individual fields have the following meanings: table, see below) - `index_entry_count`: number of entries in the index - `index_size`: index size in bytes + - `content_id`: deterministic xxHash3 of the index, descriptors, names, and assembly data ## Assembly descriptor table @@ -284,6 +287,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; ``` diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs index dd2788f30a1..06c70e6f479 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs @@ -59,6 +59,7 @@ public class GenerateNativeApplicationConfigSources : AndroidTask public bool EnableMarshalMethods { get; set; } public bool EnableManagedMarshalMethodsLookup { get; set; } + public bool AndroidEnableAssemblyStoreDecompressionCache { get; set; } public string? RuntimeConfigBinFilePath { get; set; } public string ProjectRuntimeConfigFilePath { get; set; } = String.Empty; public string? BoundExceptionType { get; set; } @@ -286,6 +287,7 @@ static bool ShouldSkipAssembly (ITaskItem assembly) ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), HaveAssemblyStore = UseAssemblyStore, + AssemblyStoreDecompressionCacheEnabled = AndroidEnableAssemblyStoreDecompressionCache, }; } else { appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (envBuilder.EnvironmentVariables, envBuilder.SystemProperties, Log) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs new file mode 100644 index 00000000000..466f52e15df --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs @@ -0,0 +1,57 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Hashing; +using System.Linq; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NUnit.Framework; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Build.Tests.Tasks; + +[TestFixture] +public class CreateAssemblyStoreTests : BaseTest +{ + [Test] + public void ContentIdMatchesStoreContents () + { + string testDirectory = Path.Combine (Root, "temp", nameof (ContentIdMatchesStoreContents)); + Directory.CreateDirectory (testDirectory); + + string assemblyPath = Path.Combine (testDirectory, "Example.dll.zst"); + byte [] assemblyData = [1, 3, 3, 7, 9, 11, 17, 23]; + File.WriteAllBytes (assemblyPath, assemblyData); + + var metadata = new Dictionary { + ["Abi"] = "arm64-v8a", + }; + var task = new CreateAssemblyStore { + BuildEngine = new MockBuildEngine (TestContext.Out), + AppSharedLibrariesDir = Path.Combine (testDirectory, "stores"), + ResolvedFrameworkAssemblies = [], + ResolvedUserAssemblies = [new TaskItem (assemblyPath, metadata)], + SupportedAbis = ["arm64-v8a"], + TargetRuntime = "CoreCLR", + UseAssemblyStore = true, + }; + + Assert.IsTrue (task.Execute (), "CreateAssemblyStore should succeed."); + + string storePath = task.AssembliesToAddToArchive.Single ().ItemSpec; + byte [] store = File.ReadAllBytes (storePath); + using var reader = new BinaryReader (new MemoryStream (store)); + Assert.AreEqual (0x41424158u, reader.ReadUInt32 (), "Unexpected assembly store magic."); + Assert.AreEqual (0x80010004u, reader.ReadUInt32 (), "Unexpected arm64 assembly store version."); + reader.BaseStream.Seek (3 * sizeof (uint), SeekOrigin.Current); + ulong contentId = reader.ReadUInt64 (); + + Assert.AreEqual ( + XxHash3.HashToUInt64 (store.AsSpan (5 * sizeof (uint) + sizeof (ulong))), + contentId, + "The content ID should hash everything after the assembly store header." + ); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs index 803cc00bc32..04cd7982543 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs @@ -42,4 +42,37 @@ public void HaveAssemblyStoreIsEmittedForCoreCLR (bool haveAssemblyStore) var config = (EnvironmentHelper.ApplicationConfig_CoreCLR)EnvironmentHelper.ReadApplicationConfig (environmentFiles, AndroidRuntime.CoreCLR); Assert.AreEqual (haveAssemblyStore, config.have_assembly_store); } + + [TestCase (false)] + [TestCase (true)] + public void AssemblyStoreDecompressionCacheSettingIsEmitted (bool enabled) + { + string outputRoot = Path.Combine (Root, "temp", $"{nameof (AssemblyStoreDecompressionCacheSettingIsEmitted)}-{enabled}"); + string monoAndroidPath = Path.Combine (TestEnvironment.MonoAndroidFrameworkDirectory, "Mono.Android.dll"); + FileAssert.Exists (monoAndroidPath); + + var task = new GenerateNativeApplicationConfigSources { + BuildEngine = new MockBuildEngine (TestContext.Out), + ResolvedAssemblies = [new TaskItem (monoAndroidPath)], + EnvironmentOutputDirectory = Path.Combine (outputRoot, "android"), + SupportedAbis = ["arm64-v8a"], + AndroidPackageName = "com.microsoft.android.cachetest", + EnablePreloadAssembliesDefault = false, + TargetsCLR = true, + AndroidRuntime = "CoreCLR", + UseAssemblyStore = true, + AndroidEnableAssemblyStoreDecompressionCache = enabled, + }; + + Assert.IsTrue (task.Execute (), "GenerateNativeApplicationConfigSources should succeed."); + + var environmentFiles = EnvironmentHelper.GatherEnvironmentFiles ( + outputRoot, + "arm64-v8a", + required: true, + runtime: AndroidRuntime.CoreCLR + ); + var config = (EnvironmentHelper.ApplicationConfig_CoreCLR)EnvironmentHelper.ReadApplicationConfig (environmentFiles, AndroidRuntime.CoreCLR); + Assert.AreEqual (enabled, config.assembly_store_decompression_cache_enabled); + } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index d8c2ab96b50..8c511a1311f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -62,9 +62,10 @@ public sealed class ApplicationConfig_CoreCLR : IApplicationConfig public string android_package_name = String.Empty; public bool managed_marshal_methods_lookup_enabled; public bool have_assembly_store; + public bool assembly_store_decompression_cache_enabled; } - const uint ApplicationConfigFieldCount_CoreCLR = 20; + const uint ApplicationConfigFieldCount_CoreCLR = 21; // This must be identical to the ApplicationConfig structure in src/native/mono/xamarin-app-stub/xamarin-app.hh public sealed class ApplicationConfig_MonoVM : IApplicationConfig @@ -407,6 +408,11 @@ static IApplicationConfig ReadApplicationConfig_CoreCLR (EnvironmentFile envFile AssertFieldType (envFile.Path, parser.SourceFilePath, ".byte", field [0], item.LineNumber); ret.have_assembly_store = ConvertFieldToBool ("have_assembly_store", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; + + case 20: // assembly_store_decompression_cache_enabled: bool / .byte + AssertFieldType (envFile.Path, parser.SourceFilePath, ".byte", field [0], item.LineNumber); + ret.assembly_store_decompression_cache_enabled = ConvertFieldToBool ("assembly_store_decompression_cache_enabled", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + break; } fieldCount++; } @@ -772,6 +778,7 @@ static void AssertApplicationConfigIsIdentical (ApplicationConfig_CoreCLR firstA Assert.AreEqual (firstAppConfig.system_property_count, secondAppConfig.system_property_count, $"Field 'system_property_count' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); Assert.AreEqual (firstAppConfig.android_package_name, secondAppConfig.android_package_name, $"Field 'android_package_name' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); Assert.AreEqual (firstAppConfig.have_assembly_store, secondAppConfig.have_assembly_store, $"Field 'have_assembly_store' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); + Assert.AreEqual (firstAppConfig.assembly_store_decompression_cache_enabled, secondAppConfig.assembly_store_decompression_cache_enabled, $"Field 'assembly_store_decompression_cache_enabled' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); } static void AssertApplicationConfigIsIdentical (ApplicationConfig_MonoVM firstAppConfig, string firstEnvFile, ApplicationConfig_MonoVM secondAppConfig, string secondEnvFile) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs index 832c37299d7..5a0d0f4cee5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs @@ -51,4 +51,5 @@ sealed class ApplicationConfigCLR public string android_package_name = String.Empty; public bool managed_marshal_methods_lookup_enabled; public bool have_assembly_store; + public bool assembly_store_decompression_cache_enabled; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs index 4f977d2d71f..e9a4475bee4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs @@ -198,6 +198,7 @@ sealed class DsoCacheState public bool ManagedMarshalMethodsLookupEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } public bool HaveAssemblyStore { get; set; } + public bool AssemblyStoreDecompressionCacheEnabled { get; set; } public ApplicationConfigNativeAssemblyGeneratorCLR (IDictionary environmentVariables, IDictionary systemProperties, IDictionary? runtimeProperties, TaskLoggingHelper log) @@ -284,6 +285,7 @@ protected override void Construct (LlvmIrModule module) jni_remapping_replacement_method_index_entry_count = (uint)JniRemappingReplacementMethodIndexEntryCount, android_package_name = AndroidPackageName, have_assembly_store = HaveAssemblyStore, + assembly_store_decompression_cache_enabled = AssemblyStoreDecompressionCacheEnabled, }; application_config = new StructureInstance (applicationConfigStructureInfo, app_cfg); module.AddGlobalVariable ("application_config", application_config); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs index 9df30db6303..7a30ec231b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -5,7 +5,7 @@ partial class AssemblyStoreGenerator { sealed class AssemblyStoreHeader { - public const uint NativeSize = 5 * sizeof (uint); + public const uint NativeSize = 5 * sizeof (uint) + sizeof (ulong); public readonly uint magic = ASSEMBLY_STORE_MAGIC; public readonly uint version; @@ -14,17 +14,19 @@ sealed class AssemblyStoreHeader // Index size in bytes public readonly uint index_size; + public readonly ulong content_id; - public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size) + public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) { this.version = version; this.entry_count = entry_count; this.index_entry_count = index_entry_count; this.index_size = index_size; + this.content_id = content_id; } #if XABT_TESTS - public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) - : this (version, entry_count, index_entry_count, index_size) + public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) + : this (version, entry_count, index_entry_count, index_size, content_id) { this.magic = magic; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 42dace7c726..1241a805f25 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Hashing; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; @@ -28,6 +29,7 @@ namespace Xamarin.Android.Tasks; // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint CRC32 for CoreCLR; uint/ulong xxhash for MonoVM depending on target bitness @@ -53,8 +55,8 @@ partial class AssemblyStoreGenerator const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones - const uint ASSEMBLY_STORE_FORMAT_VERSION_MONOVM_64BIT = 0x80000003; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant - const uint ASSEMBLY_STORE_FORMAT_VERSION_MONOVM_32BIT = 0x00000003; + const uint ASSEMBLY_STORE_FORMAT_VERSION_MONOVM_64BIT = 0x80000004; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_MONOVM_32BIT = 0x00000004; const uint ASSEMBLY_STORE_FORMAT_VERSION_CORECLR_64BIT = 0x80000004; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant const uint ASSEMBLY_STORE_FORMAT_VERSION_CORECLR_32BIT = 0x00000004; @@ -129,7 +131,7 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List 0) { + hash.Append (buffer.AsSpan (0, bytesRead)); + } + + return hash.GetCurrentHashAsUInt64 (); + } + void CopyData (FileInfo? src, Stream dest, string storePath) { if (src == null) { @@ -262,6 +285,7 @@ void WriteHeader (BinaryWriter writer, AssemblyStoreHeader header) writer.Write (header.entry_count); writer.Write (header.index_entry_count); writer.Write (header.index_size); + writer.Write (header.content_id); } #if XABT_TESTS AssemblyStoreHeader ReadHeader (BinaryReader reader) @@ -272,8 +296,9 @@ AssemblyStoreHeader ReadHeader (BinaryReader reader) uint entry_count = reader.ReadUInt32 (); uint index_entry_count = reader.ReadUInt32 (); uint index_size = reader.ReadUInt32 (); + ulong content_id = reader.ReadUInt64 (); - return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size); + return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size, content_id); } #endif diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 8e0b5004150..c0235f7c06c 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -167,6 +167,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True <_AndroidAssemblyStoreCompressionLevel Condition=" '$(_AndroidAssemblyStoreCompressionLevel)' == '' ">3 + False False <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> @@ -1809,6 +1810,7 @@ because xbuild doesn't support framework reference assemblies. BoundExceptionType="$(AndroidBoundExceptionType)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UseAssemblyStore="$(_AndroidUseAssemblyStore)" + AndroidEnableAssemblyStoreDecompressionCache="$(AndroidEnableAssemblyStoreDecompressionCache)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" EnableManagedMarshalMethodsLookup="$(_AndroidUseManagedMarshalMethodsLookup)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" diff --git a/src/java-runtime/java/mono/android/clr/MonoPackageManager.java b/src/java-runtime/java/mono/android/clr/MonoPackageManager.java index 4cfa0ea2169..5592cc2b68e 100644 --- a/src/java-runtime/java/mono/android/clr/MonoPackageManager.java +++ b/src/java-runtime/java/mono/android/clr/MonoPackageManager.java @@ -52,6 +52,7 @@ public static void LoadApplication (Context context) String language = locale.getLanguage () + "-" + locale.getCountry (); String filesDir = context.getFilesDir ().getAbsolutePath (); String cacheDir = context.getCacheDir ().getAbsolutePath (); + String codeCacheDir = context.getCodeCacheDir ().getAbsolutePath (); String dataDir = getNativeLibraryPath (context); ClassLoader loader = context.getClassLoader (); String runtimeDir = getNativeLibraryPath (runtimePackage); @@ -66,7 +67,7 @@ public static void LoadApplication (Context context) // // Should the order change here, src/native/clr/include/constants.hh must be updated accordingly // - String[] appDirs = new String[] {filesDir, cacheDir, dataDir}; + String[] appDirs = new String[] {filesDir, cacheDir, dataDir, codeCacheDir}; boolean haveSplitApks = runtimePackage.splitSourceDirs != null && runtimePackage.splitSourceDirs.length > 0; System.loadLibrary("monodroid"); diff --git a/src/native/clr/host/assembly-store.cc b/src/native/clr/host/assembly-store.cc index c000b5679f2..3b34cf87092 100644 --- a/src/native/clr/host/assembly-store.cc +++ b/src/native/clr/host/assembly-store.cc @@ -1,8 +1,21 @@ +#include +#include #include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include + #include #include +#include #include #include #include @@ -33,8 +46,461 @@ namespace { return false; } -} + namespace asm_cache { + constexpr std::string_view CACHE_DIR_NAME = "decompressed-assembly-cache-v1"sv; + constexpr uint32_t CACHE_FILE_MAGIC = 0x43434158; // 'XACC', little-endian + constexpr uint32_t CACHE_FILE_FORMAT_VERSION = 1; + constexpr size_t MAX_QUEUED_BYTES = 32uz * 1024uz * 1024uz; + + struct [[gnu::packed]] CacheFileFooter final + { + uint32_t magic; + uint32_t version; + uint64_t store_id; + uint64_t payload_hash; + uint32_t descriptor_index; + uint32_t payload_size; + }; + + static_assert (sizeof (CacheFileFooter) == 32uz); + + struct WriteRequest final + { + std::string path; + std::unique_ptr data; + size_t size; + }; + + enum class WriteResult + { + Succeeded, + Failed, + }; + + std::mutex state_lock; + std::deque write_queue; + std::string cache_dir; + std::unique_ptr tracking; + size_t queued_bytes = 0; + uint64_t store_id = 0; + bool initialized = false; + bool enabled = false; + bool writes_enabled = false; + bool writer_running = false; + + auto hash_payload (const uint8_t *data, size_t size) noexcept -> uint64_t + { + return static_cast(crc32_hash (reinterpret_cast(data), size)); + } + + bool write_fully (int fd, const uint8_t *buf, size_t len) noexcept + { + size_t off = 0; + while (off < len) { + ssize_t n = write (fd, buf + off, len - off); + if (n < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (n == 0) { + errno = EIO; + return false; + } + off += static_cast(n); + } + return true; + } + + void log_file_error (std::string_view operation, std::string const& path, int error) noexcept + { + log_debug (LOG_ASSEMBLY, "Decompressed-assembly cache {} failed for '{}': {}"sv, operation, path, std::strerror (error)); + } + + auto write_cache_file (WriteRequest const& req) noexcept -> WriteResult + { + std::string tmp_path = req.path; + tmp_path.append (".tmp."sv); + tmp_path.append (std::to_string (getpid ())); + + int fd; + do { + fd = open (tmp_path.c_str (), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + log_file_error ("temporary-file creation"sv, req.path, errno); + return WriteResult::Failed; + } + + bool ok = write_fully (fd, req.data.get (), req.size); + int error = ok ? 0 : errno; + if (close (fd) != 0 && ok) { + ok = false; + error = errno; + } + + if (!ok) { + log_file_error ("write"sv, req.path, error); + unlink (tmp_path.c_str ()); + return WriteResult::Failed; + } + + int rename_result; + do { + rename_result = rename (tmp_path.c_str (), req.path.c_str ()); + } while (rename_result != 0 && errno == EINTR); + + if (rename_result != 0) { + error = errno; + log_file_error ("publish"sv, req.path, error); + unlink (tmp_path.c_str ()); + return WriteResult::Failed; + } + + return WriteResult::Succeeded; + } + + void clear_write_queue_locked () noexcept + { + for (WriteRequest const& request : write_queue) { + queued_bytes -= request.size; + } + write_queue.clear (); + } + + [[gnu::cold]] + auto writer_loop ([[maybe_unused]] void *arg) noexcept -> void* + { + while (true) { + WriteRequest request; + { + std::lock_guard lock (state_lock); + if (write_queue.empty ()) { + writer_running = false; + return nullptr; + } + + request = std::move (write_queue.front ()); + write_queue.pop_front (); + } + + size_t request_size = request.size; + WriteResult write_result = write_cache_file (request); + request.data.reset (); + + { + std::lock_guard lock (state_lock); + queued_bytes -= request_size; + if (write_result == WriteResult::Failed) { + writes_enabled = false; + clear_write_queue_locked (); + writer_running = false; + log_debug (LOG_ASSEMBLY, "Disabling decompressed-assembly cache writes after a persistence failure"sv); + return nullptr; + } + } + } + } + + bool start_writer_locked () noexcept + { + pthread_attr_t attributes; + int result = pthread_attr_init (&attributes); + bool attributes_initialized = result == 0; + if (result == 0) { + result = pthread_attr_setdetachstate (&attributes, PTHREAD_CREATE_DETACHED); + } + + pthread_t writer_thread; + if (result == 0) { + result = pthread_create (&writer_thread, &attributes, writer_loop, nullptr); + } + + if (attributes_initialized) { + pthread_attr_destroy (&attributes); + } + if (result != 0) { + log_debug (LOG_ASSEMBLY, "Failed to start decompressed-assembly cache writer: {}"sv, std::strerror (result)); + return false; + } + + return true; + } + + bool ensure_directory (std::string const& path) noexcept + { + if (mkdir (path.c_str (), 0700) == 0) { + return true; + } + + int error = errno; + if (error != EEXIST) { + log_file_error ("directory creation"sv, path, error); + return false; + } + + struct stat st {}; + if (lstat (path.c_str (), &st) != 0) { + log_file_error ("directory validation"sv, path, errno); + return false; + } + if (!S_ISDIR (st.st_mode)) { + log_file_error ("directory validation"sv, path, ENOTDIR); + return false; + } + + return true; + } + + // Best-effort removal of staging files left behind by a previous process whose writer was + // killed (e.g. by Android) between creating a `.tmp.` file and renaming it into place. + // Such files are never reclaimed otherwise and would accumulate outside the queue bound. + void remove_stale_temp_files (std::string const& dir) noexcept + { + DIR *handle = opendir (dir.c_str ()); + if (handle == nullptr) { + return; + } + + std::string prefix = dir; + prefix.append ("/"); + for (dirent *entry = readdir (handle); entry != nullptr; entry = readdir (handle)) { + std::string_view name { entry->d_name }; + if (name.find (".tmp."sv) == std::string_view::npos) { + continue; + } + + std::string path = prefix; + path.append (name); + unlink (path.c_str ()); + } + + closedir (handle); + } + + void ensure_initialized (uint64_t assembly_store_id) noexcept + { + if (initialized) { + return; + } + initialized = true; + + bool cache_requested = application_config.assembly_store_decompression_cache_enabled; + + // Allow overriding the build setting at runtime for A/B benchmarking: + // adb shell setprop debug.net.asmcache 0 # off + // adb shell setprop debug.net.asmcache 1 # on + if (getenv ("XA_DISABLE_ASSEMBLY_CACHE") != nullptr) { + return; + } + { + dynamic_local_property_string prop_value; + if (AndroidSystem::monodroid_get_system_property ("debug.net.asmcache"sv, prop_value) > 0 && prop_value.get () != nullptr) { + if (prop_value.get ()[0] == '0') { + cache_requested = false; + } else if (prop_value.get ()[0] == '1') { + cache_requested = true; + } + } + } + + if (!cache_requested) { + return; + } + + std::string const& code_cache_dir = AndroidSystem::get_app_code_cache_dir (); + if (code_cache_dir.empty ()) { + return; + } + + cache_dir.assign (code_cache_dir); + cache_dir.append ("/"); + cache_dir.append (CACHE_DIR_NAME); + if (!ensure_directory (cache_dir)) { + return; + } + + store_id = assembly_store_id; + cache_dir.append ("/"); + cache_dir.append (std::format ("{:x}", store_id)); + if (!ensure_directory (cache_dir)) { + return; + } + + remove_stale_temp_files (cache_dir); + + if (compressed_assembly_count > 0) { + tracking.reset (new (std::nothrow) uint8_t*[compressed_assembly_count]()); + } + + enabled = (tracking != nullptr); + if (!enabled) { + return; + } + + { + std::lock_guard lock (state_lock); + writes_enabled = true; + } + + log_debug ( + LOG_ASSEMBLY, + "Enabled decompressed-assembly cache at '{}'; store ID 0x{:x}; write queue limit {} bytes"sv, + cache_dir, + store_id, + MAX_QUEUED_BYTES + ); + } + + auto build_path (uint32_t descriptor_index) noexcept -> std::string + { + std::string path = cache_dir; + path.append ("/"); + path.append (std::to_string (descriptor_index)); + path.append (".bin"sv); + return path; + } + + auto try_load (uint32_t descriptor_index, std::string_view name, uint32_t expected_size) noexcept -> uint8_t* + { + if (!enabled) { + return nullptr; + } + + std::string path = build_path (descriptor_index); + int fd = open (path.c_str (), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + return nullptr; + } + + struct stat st {}; + if (fstat (fd, &st) != 0 || + !S_ISREG (st.st_mode) || + static_cast(st.st_size) != static_cast(expected_size) + sizeof (CacheFileFooter)) { + close (fd); + return nullptr; + } + + size_t map_size = static_cast(expected_size) + sizeof (CacheFileFooter); + // The runtime may modify the image, so keep those changes private while + // retaining clean file-backed pages until they are actually written. + void *mapped = mmap (nullptr, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + close (fd); + if (mapped == MAP_FAILED) { + return nullptr; + } + + CacheFileFooter footer {}; + memcpy (&footer, static_cast(mapped) + expected_size, sizeof (footer)); + if (footer.magic != CACHE_FILE_MAGIC || + footer.version != CACHE_FILE_FORMAT_VERSION || + footer.store_id != store_id || + footer.descriptor_index != descriptor_index || + footer.payload_size != expected_size || + footer.payload_hash != hash_payload (static_cast(mapped), expected_size)) { + munmap (mapped, map_size); + log_debug (LOG_ASSEMBLY, "Ignoring invalid decompressed-assembly cache entry for '{}'"sv, name); + return nullptr; + } + + return static_cast(mapped); + } + + void enqueue_write (uint32_t descriptor_index, std::string_view name, const uint8_t *data, size_t size) noexcept + { + if (!enabled) { + return; + } + + if (size > SIZE_MAX - sizeof (CacheFileFooter)) { + return; + } + size_t total = size + sizeof (CacheFileFooter); + + size_t bytes_queued = 0; + bool queue_full = false; + { + std::lock_guard lock (state_lock); + if (!writes_enabled) { + return; + } + if (total > MAX_QUEUED_BYTES || queued_bytes > MAX_QUEUED_BYTES - total) { + queue_full = true; + bytes_queued = queued_bytes; + } else { + queued_bytes += total; + } + } + + if (queue_full) { + if (total > MAX_QUEUED_BYTES) { + log_debug ( + LOG_ASSEMBLY, + "Skipping decompressed-assembly cache write for '{}': {} bytes exceed the {}-byte queue limit"sv, + name, + total, + MAX_QUEUED_BYTES + ); + } else { + log_debug ( + LOG_ASSEMBLY, + "Skipping decompressed-assembly cache write for '{}': {} of {} queue bytes are in use"sv, + name, + bytes_queued, + MAX_QUEUED_BYTES + ); + } + return; + } + + auto snapshot = std::unique_ptr (new (std::nothrow) uint8_t[total]); + if (snapshot == nullptr) { + std::lock_guard lock (state_lock); + queued_bytes -= total; + return; + } + // The runtime can modify the shared decompression buffer after this + // method returns, so the background writer needs an immutable copy. + memcpy (snapshot.get (), data, size); + + CacheFileFooter footer { + .magic = CACHE_FILE_MAGIC, + .version = CACHE_FILE_FORMAT_VERSION, + .store_id = store_id, + .payload_hash = hash_payload (snapshot.get (), size), + .descriptor_index = descriptor_index, + .payload_size = static_cast(size), + }; + memcpy (snapshot.get () + size, &footer, sizeof (footer)); + + WriteRequest req { + .path = build_path (descriptor_index), + .data = std::move (snapshot), + .size = total, + }; + + { + std::lock_guard lock (state_lock); + if (!writes_enabled) { + queued_bytes -= total; + return; + } + + write_queue.push_back (std::move (req)); + if (!writer_running) { + writer_running = true; + if (!start_writer_locked ()) { + writer_running = false; + writes_enabled = false; + clear_write_queue_locked (); + } + } + } + } + } // namespace asm_cache +} // anonymous namespace [[gnu::always_inline]] void AssemblyStore::set_assembly_data_and_size (uint8_t* source_assembly_data, uint32_t source_assembly_data_size, uint8_t*& dest_assembly_data, uint32_t& dest_assembly_data_size) noexcept { @@ -51,7 +517,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co #if defined (RELEASE) auto header = reinterpret_cast(e.image_data); if (header->magic == COMPRESSED_DATA_MAGIC) { - log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); + log_debug (LOG_ASSEMBLY, "Resolving compressed assembly '{}' from the assembly store"sv, name); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.start_event (TimingEventKind::AssemblyDecompression); @@ -101,11 +567,25 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } uint8_t *data_buffer = uncompressed_assemblies_data_buffer + cad.buffer_offset; - if (!cad.loaded) { + uint32_t const descriptor_index = header->descriptor_index; + auto is_loaded = [&cad]() noexcept -> bool { + return __atomic_load_n (&cad.loaded, __ATOMIC_ACQUIRE); + }; + + // Resolves to the mmap'd cache file when this assembly was loaded from + // the on-device cache, otherwise to the shared decompression buffer. + auto resolve_data = [descriptor_index, data_buffer]() noexcept -> uint8_t* { + if (asm_cache::tracking != nullptr && asm_cache::tracking[descriptor_index] != nullptr) { + return asm_cache::tracking[descriptor_index]; + } + return data_buffer; + }; + + if (!is_loaded ()) { StartupAwareLock decompress_lock (assembly_decompress_mutex); - if (cad.loaded) { - set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); + if (is_loaded ()) { + set_assembly_data_and_size (resolve_data (), cad.uncompressed_file_size, assembly_data, assembly_data_size); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.end_event (true /* uses_more_info */); @@ -118,6 +598,8 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co return {assembly_data, assembly_data_size}; } + asm_cache::ensure_initialized (assembly_store_content_id); + if (header->uncompressed_length != cad.uncompressed_file_size) { if (header->uncompressed_length > cad.uncompressed_file_size) { Helpers::abort_application ( @@ -136,38 +618,59 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } const char *data_start = pointer_add(e.image_data, sizeof(CompressedAssemblyHeader)); - size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ZSTD_isError (ret)) { - Helpers::abort_application ( - LOG_ASSEMBLY, - std::format ( - "Decompression of assembly {} failed: {}"sv, - name, - ZSTD_getErrorName (ret) - ) - ); - } + bool loaded_from_cache = false; + uint8_t *cached = asm_cache::try_load (descriptor_index, name, cad.uncompressed_file_size); + if (cached != nullptr) { + loaded_from_cache = true; + log_debug (LOG_ASSEMBLY, "Loaded decompressed assembly '{}' from the on-device cache"sv, name); + if (asm_cache::tracking != nullptr) { + asm_cache::tracking[descriptor_index] = cached; + } + } else { + log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret != cad.uncompressed_file_size) { - Helpers::abort_application ( - LOG_ASSEMBLY, - std::format ( - "Decompression of assembly {} yielded a different size (expected {}, got {})"sv, - name, - cad.uncompressed_file_size, - static_cast(ret) - ) - ); + if (ZSTD_isError (ret)) { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Decompression of assembly {} failed: {}"sv, + name, + ZSTD_getErrorName (ret) + ) + ); + } + + if (ret != cad.uncompressed_file_size) { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Decompression of assembly {} yielded a different size (expected {}, got {})"sv, + name, + cad.uncompressed_file_size, + static_cast(ret) + ) + ); + } + + asm_cache::enqueue_write (descriptor_index, name, data_buffer, cad.uncompressed_file_size); } - cad.loaded = true; + + __atomic_store_n (&cad.loaded, true, __ATOMIC_RELEASE); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.end_event (true /* uses_more_info */); - internal_timing.add_more_info (name); + + dynamic_local_string msg; + msg.append (name); + if (loaded_from_cache) { + msg.append (" (decompressed cache hit)"sv); + } + internal_timing.add_more_info (msg); } } - set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); + set_assembly_data_and_size (resolve_data (), cad.uncompressed_file_size, assembly_data, assembly_data_size); } else #endif // def RELEASE { @@ -318,6 +821,7 @@ void AssemblyStore::configure_from_payload (const void *payload_start, const std constexpr size_t header_size = sizeof(AssemblyStoreHeader); + assembly_store_content_id = header->content_id; assembly_store.data_start = static_cast(payload_start); assembly_store.assembly_count = header->entry_count; assembly_store.index_entry_count = header->index_entry_count; @@ -340,4 +844,6 @@ void AssemblyStore::configure_from_payload (const void *payload_start, const std assembly_store_names[i] = std::string_view (reinterpret_cast(names_cursor), name_length); names_cursor += name_length; } + + log_debug (LOG_ASSEMBLY, "Mapped assembly store {}; content ID 0x{:x}"sv, get_full_store_path (), assembly_store_content_id); } diff --git a/src/native/clr/host/host.cc b/src/native/clr/host/host.cc index 65645dff0b0..033f8425cf6 100644 --- a/src/native/clr/host/host.cc +++ b/src/native/clr/host/host.cc @@ -325,6 +325,7 @@ void Host::Java_mono_android_Runtime_initInternal ( AndroidSystem::detect_embedded_dso_mode (applicationDirs); AndroidSystem::set_running_in_emulator (isEmulator); AndroidSystem::set_primary_override_dir (files_dir); + AndroidSystem::set_app_code_cache_dir (applicationDirs[Constants::APP_DIRS_CODE_CACHE_DIR_INDEX]); AndroidSystem::create_update_dir (AndroidSystem::get_primary_override_dir ()); AndroidSystem::setup_environment (); Logger::init_reference_logging (AndroidSystem::get_primary_override_dir ()); diff --git a/src/native/clr/include/constants.hh b/src/native/clr/include/constants.hh index d9764c9cefb..5de37b8cfd8 100644 --- a/src/native/clr/include/constants.hh +++ b/src/native/clr/include/constants.hh @@ -114,6 +114,7 @@ namespace xamarin::android { static constexpr size_t APP_DIRS_FILES_DIR_INDEX = 0uz; static constexpr size_t APP_DIRS_CACHE_DIR_INDEX = 1uz; static constexpr size_t APP_DIRS_DATA_DIR_INDEX = 2uz; + static constexpr size_t APP_DIRS_CODE_CACHE_DIR_INDEX = 3uz; static inline constexpr size_t PROPERTY_VALUE_BUFFER_LEN = PROP_VALUE_MAX + 1uz; diff --git a/src/native/clr/include/host/assembly-store.hh b/src/native/clr/include/host/assembly-store.hh index 973dc37e917..6a529a11376 100644 --- a/src/native/clr/include/host/assembly-store.hh +++ b/src/native/clr/include/host/assembly-store.hh @@ -36,6 +36,7 @@ namespace xamarin::android { // Assembly names indexed by `AssemblyStoreIndexEntry::descriptor_index`, used to disambiguate // CRC32 hash collisions in the store index. Built once when the store is mapped. static inline std::string_view *assembly_store_names = nullptr; + static inline uint64_t assembly_store_content_id = 0; static inline std::mutex assembly_decompress_mutex {}; }; } diff --git a/src/native/clr/include/runtime-base/android-system.hh b/src/native/clr/include/runtime-base/android-system.hh index b60871a93c4..3ddaee861b6 100644 --- a/src/native/clr/include/runtime-base/android-system.hh +++ b/src/native/clr/include/runtime-base/android-system.hh @@ -70,6 +70,16 @@ namespace xamarin::android { primary_override_dir = determine_primary_override_dir (home); } + static auto get_app_code_cache_dir () noexcept -> std::string const& + { + return app_code_cache_dir; + } + + static void set_app_code_cache_dir (jstring_wrapper& code_cache_dir) noexcept + { + app_code_cache_dir.assign (code_cache_dir.get_cstr ()); + } + static auto get_native_libraries_dir () noexcept -> std::string const& { return native_libraries_dir; @@ -146,6 +156,7 @@ namespace xamarin::android { static inline bool embedded_dso_mode_enabled = false; static inline std::string primary_override_dir; static inline std::string native_libraries_dir; + static inline std::string app_code_cache_dir; #if defined (DEBUG) static inline std::unordered_map bundled_properties; diff --git a/src/native/clr/include/xamarin-app.hh b/src/native/clr/include/xamarin-app.hh index fd0ea6a12fd..30eed68978c 100644 --- a/src/native/clr/include/xamarin-app.hh +++ b/src/native/clr/include/xamarin-app.hh @@ -137,6 +137,7 @@ struct CompressedAssemblyDescriptor // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint; CRC32 of the assembly name @@ -168,6 +169,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; struct [[gnu::packed]] AssemblyStoreIndexEntry final @@ -232,6 +234,7 @@ struct ApplicationConfig const char *android_package_name; bool managed_marshal_methods_lookup_enabled; bool have_assembly_store; + bool assembly_store_decompression_cache_enabled; }; struct DSOCacheEntry diff --git a/src/native/clr/xamarin-app-stub/application_dso_stub.cc b/src/native/clr/xamarin-app-stub/application_dso_stub.cc index 3d2f2eae958..147ea889d0d 100644 --- a/src/native/clr/xamarin-app-stub/application_dso_stub.cc +++ b/src/native/clr/xamarin-app-stub/application_dso_stub.cc @@ -69,6 +69,7 @@ const ApplicationConfig application_config = { .android_package_name = android_package_name, .managed_marshal_methods_lookup_enabled = false, .have_assembly_store = false, + .assembly_store_decompression_cache_enabled = false, }; // TODO: migrate to std::string_view for these two diff --git a/src/native/mono/xamarin-app-stub/xamarin-app.hh b/src/native/mono/xamarin-app-stub/xamarin-app.hh index 2a21c1e827b..1a912d272d5 100644 --- a/src/native/mono/xamarin-app-stub/xamarin-app.hh +++ b/src/native/mono/xamarin-app-stub/xamarin-app.hh @@ -34,7 +34,7 @@ static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000; #endif // Increase whenever an incompatible change is made to the assembly store format -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 3 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 4 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian @@ -145,6 +145,7 @@ struct XamarinAndroidBundledAssembly // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name @@ -176,6 +177,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; struct [[gnu::packed]] AssemblyStoreIndexEntry final diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 27bcdd668b7..6bfdb7d5b63 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -660,6 +660,98 @@ public void DeployToDevice ([Values] bool isRelease, [Values (AndroidRuntime.Cor Assert.IsTrue (didLaunch, "Activity should have started."); } + [Test] + public void AssemblyStoreDecompressionCacheMapsPersistedAssemblies () + { + if (IgnoreUnsupportedConfiguration (AndroidRuntime.CoreCLR, release: true)) { + return; + } + + var app = new XamarinAndroidApplicationProject (packageName: PackageUtils.MakePackageName (AndroidRuntime.CoreCLR, "assemblycache")) { + IsRelease = true, + }; + app.SetRuntime (AndroidRuntime.CoreCLR); + app.SetRuntimeIdentifiers (new [] { DeviceAbi }); + app.SetProperty ("AndroidEnableAssemblyStoreDecompressionCache", "true"); + app.AndroidManifest = app.AndroidManifest.Replace (" line.EndsWith (".bin", StringComparison.Ordinal)) + .ToArray (); + } + Assert.That (cacheFiles.Length, Is.GreaterThanOrEqualTo (2), "The first launch should persist multiple decompressed assemblies."); + + RunAdbCommand ($"shell am force-stop --user all {app.PackageName}"); + string cacheFileToCorrupt = cacheFiles.First (); + string ValidFileHash () => RunAdbCommand ( + $"shell run-as {app.PackageName} md5sum {cacheFileToCorrupt}" + ).Split (new [] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault () ?? ""; + + string validHash = ValidFileHash (); + Assert.That (validHash, Is.Not.Empty, $"Should be able to hash the persisted cache file '{cacheFileToCorrupt}'."); + + RunAdbCommand ( + $"shell run-as {app.PackageName} dd if=/dev/zero of={cacheFileToCorrupt} bs=1 count=1 conv=notrunc" + ); + Assert.That (ValidFileHash (), Is.Not.EqualTo (validHash), "Corrupting the cache file should change its contents."); + + ClearAdbLogcat (); + AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); + Assert.IsTrue ( + WaitForActivityToStart ( + app.PackageName, + "MainActivity", + Path.Combine (Root, appBuilder.ProjectDirectory, "assembly-cache-second-launch.log"), + ActivityStartTimeoutInSeconds + ), + "Second launch should succeed." + ); + + // A corrupted entry must be rejected (footer hash mismatch) and re-decompressed, which + // re-persists a byte-identical file. Verify the *exact* corrupted file is healed rather + // than merely checking that some other valid entry is still mapped. + bool rewritten = false; + for (int attempt = 0; attempt < 40 && !rewritten; attempt++) { + Thread.Sleep (250); + rewritten = ValidFileHash () == validHash; + } + Assert.IsTrue (rewritten, $"The corrupted cache file '{cacheFileToCorrupt}' should be rewritten with valid contents after fallback."); + + string [] pids = RunAdbCommand ($"shell pidof {app.PackageName}") + .Split (new [] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); + Assert.IsNotEmpty (pids, "The application process should be running after the second launch."); + var maps = new StringBuilder (); + foreach (string pid in pids) { + maps.Append (RunAdbCommand ($"shell run-as {app.PackageName} cat /proc/{pid}/maps")); + } + StringAssert.Contains ( + "/code_cache/decompressed-assembly-cache-v1/", + maps.ToString (), + "The second launch should map persisted decompressed assemblies." + ); + } + [Test] public void ActivityAliasRuns ([Values] bool isRelease, [Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) { diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs index 480a15d8fca..e8a77a1e102 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs @@ -8,8 +8,6 @@ partial class StoreReader_V2 { sealed class Header { - public const uint NativeSize = 5 * sizeof (uint); - public readonly uint magic; public readonly uint version; public readonly uint entry_count; @@ -17,14 +15,18 @@ sealed class Header // Index size in bytes public readonly uint index_size; + public readonly ulong content_id; + + public uint NativeSize => (uint)(5 * sizeof (uint) + ((version & ASSEMBLY_STORE_FORMAT_NUMBER_MASK) >= 4 ? sizeof (ulong) : 0)); - public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) { this.magic = magic; this.version = version; this.entry_count = entry_count; this.index_entry_count = index_entry_count; this.index_size = index_size; + this.content_id = content_id; } } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index f31e44c7a70..d8de129f727 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -16,6 +16,7 @@ partial class StoreReader_V2 : AssemblyStoreReader const uint ASSEMBLY_STORE_FORMAT_VERSION_CORECLR_64BIT = 0x80000004; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant const uint ASSEMBLY_STORE_FORMAT_VERSION_CORECLR_32BIT = 0x00000004; const uint ASSEMBLY_STORE_FORMAT_VERSION_MASK = 0xF0000000; + const uint ASSEMBLY_STORE_FORMAT_NUMBER_MASK = 0x0000FFFF; const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; @@ -130,8 +131,9 @@ protected override bool IsSupported () uint entry_count = reader.ReadUInt32 (); uint index_entry_count = reader.ReadUInt32 (); uint index_size = reader.ReadUInt32 (); + ulong content_id = (version & ASSEMBLY_STORE_FORMAT_NUMBER_MASK) >= 4 ? reader.ReadUInt64 () : 0; - header = new Header (magic, version, entry_count, index_entry_count, index_size); + header = new Header (magic, version, entry_count, index_entry_count, index_size, content_id); return true; } @@ -153,7 +155,7 @@ protected override void Prepare () AssemblyCount = header.entry_count; IndexEntryCount = header.index_entry_count; - StoreStream.Seek ((long)elfOffset + Header.NativeSize, SeekOrigin.Begin); + StoreStream.Seek ((long)elfOffset + header.NativeSize, SeekOrigin.Begin); using var reader = CreateReader (); uint indexEntrySize = GetIndexEntrySize ();