From cfaeb6cebcebc2c1dc0f48b7aed7c2a4e4af09ca Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 2 Sep 2021 20:37:35 +0200 Subject: [PATCH 01/54] [WIP] Blob generation + infrastructure --- .../Tasks/BuildApk.cs | 130 ++++----- .../Tasks/GeneratePackageManagerJava.cs | 60 ++++- .../Tasks/ProcessAssemblies.cs | 2 + .../Utilities/EnvironmentHelper.cs | 34 ++- ...pplicationConfigNativeAssemblyGenerator.cs | 47 +++- .../Utilities/ApplicationConfigTaskState.cs | 1 + .../Utilities/AssemblyBlobGenerator.cs | 193 ++++++++++++++ .../Utilities/BlobAssemblyInfo.cs | 49 ++++ .../Xamarin.Android.Build.Tasks.csproj | 1 + src/monodroid/jni/application_dso_stub.cc | 15 +- src/monodroid/jni/basic-utilities.hh | 14 + src/monodroid/jni/cpp-util.hh | 22 ++ src/monodroid/jni/embedded-assemblies-zip.cc | 246 +++++++++++------- src/monodroid/jni/embedded-assemblies.hh | 41 ++- src/monodroid/jni/xamarin-app.hh | 54 ++++ 15 files changed, 735 insertions(+), 174 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index f37a98b0a15..bf64bafa6aa 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -324,82 +324,96 @@ public override bool RunTask () void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo) { + var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); + bool useAssembliesBlob = appConfState != null ? appConfState.UseAssembliesBlob : false; string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); + AssemblyBlobGenerator blobGenerator; + + if (useAssembliesBlob) { + blobGenerator = new AssemblyBlobGenerator (AssembliesPath, Log); + } else { + blobGenerator = null; + } int count = 0; - foreach (ITaskItem assembly in ResolvedUserAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); - continue; - } - if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { - Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); - } + BlobAssemblyInfo blobAssembly = null; - sourcePath = CompressAssembly (assembly); + // Add user assemblies + AddAssembliesFromCollection (ResolvedUserAssemblies); - // Add assembly - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + // Add framework assemblies + count = 0; + AddAssembliesFromCollection (ResolvedFrameworkAssemblies); - // Try to add config if exists - var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - AddAssemblyConfigEntry (apk, assemblyPath, config); + if (useAssembliesBlob) { + blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); + } - // Try to add symbols if Debug - if (debug) { - var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + void AddAssembliesFromCollection (ITaskItem[] assemblies) + { + foreach (ITaskItem assembly in assemblies) { + if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { + Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); + continue; + } - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { + Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); + } - symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + sourcePath = CompressAssembly (assembly); - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); - } - count++; - if (count >= ZipArchiveEx.ZipFlushFilesLimit) { - apk.Flush(); - count = 0; - } - } + // Add assembly + var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); + if (useAssembliesBlob) { + blobAssembly = new BlobAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); + } else { + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + } - count = 0; - // Add framework assemblies - foreach (ITaskItem assembly in ResolvedFrameworkAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); - continue; - } + // Try to add config if exists + var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); + if (useAssembliesBlob) { + blobAssembly.SetConfigPath (config); + } else { + AddAssemblyConfigEntry (apk, assemblyPath, config); + } - if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { - Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); - } + // Try to add symbols if Debug + if (debug) { + var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + string symbolsPath = null; - sourcePath = CompressAssembly (assembly); - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: true); - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); - var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - AddAssemblyConfigEntry (apk, assemblyPath, config); - // Try to add symbols if Debug - if (debug) { - var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + if (File.Exists (symbols)) { + symbolsPath = symbols; + } else { + symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + if (File.Exists (symbols)) { + symbolsPath = symbols; + } + } - symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + if (!String.IsNullOrEmpty (symbolsPath)) { + if (useAssembliesBlob) { + blobAssembly.SetDebugInfoPath (symbolsPath); + } else { + AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + } + } + } - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); - } - count++; - if (count >= ZipArchiveEx.ZipFlushFilesLimit) { - apk.Flush(); - count = 0; + if (useAssembliesBlob) { + blobGenerator.Add (blobAssembly); + } else { + count++; + if (count >= ZipArchiveEx.ZipFlushFilesLimit) { + apk.Flush(); + count = 0; + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index a90c105c1c0..a2eae12a140 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -30,6 +30,8 @@ public class GeneratePackageManagerJava : AndroidTask public ITaskItem[] SatelliteAssemblies { get; set; } + public bool UseAssembliesBlob { get; set; } + [Required] public string OutputDirectory { get; set; } @@ -269,11 +271,15 @@ void AddEnvironment () throw new InvalidOperationException ($"Unsupported BoundExceptionType value '{BoundExceptionType}'"); } + UseAssembliesBlob = true; // TODO: remove after testing! int assemblyNameWidth = 0; - int assemblyCount = ResolvedAssemblies.Length; Encoding assemblyNameEncoding = Encoding.UTF8; Action updateNameWidth = (ITaskItem assembly) => { + if (UseAssembliesBlob) { + return; + } + string assemblyName = Path.GetFileName (assembly.ItemSpec); int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; if (nameBytes > assemblyNameWidth) { @@ -281,29 +287,64 @@ void AddEnvironment () } }; + int assemblyCount = 0; + int blobCommonAssemblyCount = 0; + int blobArchAssemblyCount = 0; + HashSet archAssemblyNames = null; + + Action updateBlobCounts = (ITaskItem assembly) => { + if (!UseAssembliesBlob) { + assemblyCount++; + return; + } + + string abi = assembly.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + blobCommonAssemblyCount++; + } else { + if (archAssemblyNames == null) { + archAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); + } + + string assemblyName = Path.GetFileName (assembly.ItemSpec); + if (!archAssemblyNames.Contains (assemblyName)) { + blobArchAssemblyCount++; + archAssemblyNames.Add (assemblyName); + } + } + }; + if (SatelliteAssemblies != null) { assemblyCount += SatelliteAssemblies.Length; foreach (ITaskItem assembly in SatelliteAssemblies) { updateNameWidth (assembly); + updateBlobCounts (assembly); } } foreach (var assembly in ResolvedAssemblies) { updateNameWidth (assembly); + updateBlobCounts (assembly); } - int abiNameLength = 0; - foreach (string abi in SupportedAbis) { - if (abi.Length <= abiNameLength) { - continue; + if (!UseAssembliesBlob) { + int abiNameLength = 0; + foreach (string abi in SupportedAbis) { + if (abi.Length <= abiNameLength) { + continue; + } + abiNameLength = abi.Length; } - abiNameLength = abi.Length; + assemblyNameWidth += abiNameLength + 2; // room for '/' and the terminating NUL } - assemblyNameWidth += abiNameLength + 1; // room for '/' bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); + if (appConfState != null) { + appConfState.UseAssembliesBlob = UseAssembliesBlob; + }; + foreach (string abi in SupportedAbis) { NativeAssemblerTargetProvider asmTargetProvider = GetAssemblyTargetProvider (abi); string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}"); @@ -323,7 +364,10 @@ void AddEnvironment () JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false, HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, - BundledAssemblyNameWidth = assemblyNameWidth + 1, + BundledAssemblyNameWidth = assemblyNameWidth, + HaveAssembliesBlob = UseAssembliesBlob, + NumberOfCommonBlobAssemblies = blobCommonAssemblyCount, + NumberOfArchBlobAssemblies = blobArchAssemblyCount, }; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 379fe318c2c..ccdec6cab35 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -180,10 +180,12 @@ void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); assembly.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); assembly.SetMetadata ("DestinationSubPath", Path.Combine (destination, fileName)); + assembly.SetMetadata ("Abi", abi); if (symbol != null) { destination = Path.Combine (symbol.GetMetadata ("DestinationSubDirectory"), abi); symbol.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); symbol.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (symbol.ItemSpec))); + symbol.SetMetadata ("Abi", abi); } } else { Log.LogDebugMessage ($"Android ABI not found for: {assembly.ItemSpec}"); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 272951341b7..6f044c1f221 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -26,15 +26,18 @@ public sealed class ApplicationConfig public bool instant_run_enabled; public bool jni_add_native_method_registration_attribute_present; public bool have_runtime_config_blob; + public bool have_assemblies_blob; public byte bound_stream_io_exception_type; public uint package_naming_policy; public uint environment_variable_count; public uint system_property_count; public uint number_of_assemblies_in_apk; + public uint number_of_common_blob_assemblies; + public uint number_of_arch_blob_assemblies; public uint bundled_assembly_name_width; public string android_package_name; }; - const uint ApplicationConfigFieldCount = 15; + const uint ApplicationConfigFieldCount = 18; static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; @@ -158,37 +161,52 @@ static ApplicationConfig ReadApplicationConfig (string envFile) ret.have_runtime_config_blob = ConvertFieldToBool ("have_runtime_config_blob", envFile, i, field [1]); break; - case 8: // bound_stream_io_exception_type: byte / .byte + case 8: // have_assemblies_blob: bool / .byte + AssertFieldType (envFile, ".byte", field [0], i); + ret.have_assemblies_blob = ConvertFieldToBool ("have_assemblies_blob", envFile, i, field [1]); + break; + + case 9: // bound_stream_io_exception_type: byte / .byte AssertFieldType (envFile, ".byte", field [0], i); ret.bound_stream_io_exception_type = ConvertFieldToByte ("bound_stream_io_exception_type", envFile, i, field [1]); break; - case 9: // package_naming_policy: uint32_t / .word | .long + case 10: // package_naming_policy: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.package_naming_policy = ConvertFieldToUInt32 ("package_naming_policy", envFile, i, field [1]); break; - case 10: // environment_variable_count: uint32_t / .word | .long + case 11: // environment_variable_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.environment_variable_count = ConvertFieldToUInt32 ("environment_variable_count", envFile, i, field [1]); break; - case 11: // system_property_count: uint32_t / .word | .long + case 12: // system_property_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.system_property_count = ConvertFieldToUInt32 ("system_property_count", envFile, i, field [1]); break; - case 12: // number_of_assemblies_in_apk: uint32_t / .word | .long + case 13: // number_of_assemblies_in_apk: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.number_of_assemblies_in_apk = ConvertFieldToUInt32 ("number_of_assemblies_in_apk", envFile, i, field [1]); break; - case 13: // bundled_assembly_name_width: uint32_t / .word | .long + case 14: // number_of_common_blob_assemblies: uint32_t / .word | .long + Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); + ret.number_of_common_blob_assemblies = ConvertFieldToUInt32 ("number_of_common_blob_assemblies", envFile, i, field [1]); + break; + + case 15: // number_of_arch_blob_assemblies: uint32_t / .word | .long + Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); + ret.number_of_arch_blob_assemblies = ConvertFieldToUInt32 ("number_of_arch_blob_assemblies", envFile, i, field [1]); + break; + + case 16: // bundled_assembly_name_width: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile, i, field [1]); break; - case 14: // android_package_name: string / [pointer type] + case 17: // android_package_name: string / [pointer type] Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}"); pointers.Add (field [1].Trim ()); break; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 8f7f69aad12..f67568ba3a0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -24,7 +24,10 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public bool InstantRunEnabled { get; set; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } public bool HaveRuntimeConfigBlob { get; set; } + public bool HaveAssembliesBlob { get; set; } public int NumberOfAssembliesInApk { get; set; } + public int NumberOfCommonBlobAssemblies { get; set; } + public int NumberOfArchBlobAssemblies { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public PackageNamingPolicy PackageNamingPolicy { get; set; } @@ -52,7 +55,7 @@ protected override void WriteSymbols (StreamWriter output) WriteDataSection (output, "application_config"); WriteSymbol (output, "application_config", TargetProvider.GetStructureAlignment (true), packed: false, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.h ApplicationConfig structure + // src/monodroid/jni/xamarin-app.hh ApplicationConfig structure WriteCommentLine (output, "uses_mono_llvm"); uint size = WriteData (output, UsesMonoLLVM); @@ -77,6 +80,9 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "have_runtime_config_blob"); size += WriteData (output, HaveRuntimeConfigBlob); + WriteCommentLine (output, "have_assemblies_blob"); + size += WriteData (output, HaveAssembliesBlob); + WriteCommentLine (output, "bound_exception_type"); size += WriteData (output, (byte)BoundExceptionType); @@ -92,6 +98,12 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "number_of_assemblies_in_apk"); size += WriteData (output, NumberOfAssembliesInApk); + WriteCommentLine (output, "number_of_common_blob_assemblies"); + size += WriteData (output, NumberOfCommonBlobAssemblies); + + WriteCommentLine (output, "number_of_arch_blob_assemblies"); + size += WriteData (output, NumberOfArchBlobAssemblies); + WriteCommentLine (output, "bundled_assembly_name_width"); size += WriteData (output, BundledAssemblyNameWidth); @@ -110,6 +122,36 @@ protected override void WriteSymbols (StreamWriter output) WriteNameValueStringArray (output, "app_system_properties", systemProperties); WriteBundledAssemblies (output); + WriteBlobAssemblies (output); + } + + void WriteBlobAssemblies (StreamWriter output) + { + WriteBlobs ("Blob assemblies data, architecture specific", "blob_bundled_assemblies_arch", NumberOfArchBlobAssemblies); + WriteBlobs ("Blob assemblies data, archiecture agnostic", "blob_bundled_assemblies_common", NumberOfCommonBlobAssemblies); + + void WriteBlobs (string comment, string label, int count) + { + WriteCommentLine (output, comment); + WriteDataSection (output, label); + WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + + uint size = 0; + for (int i = 0; i < count; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteBlobAssembly (output)); + } + WriteStructureSize (output, label, size); + } + } + + uint WriteBlobAssembly (StreamWriter output) + { + WriteCommentLine (output, "mapped assembly pointer"); + uint size = WritePointer (output); + + output.WriteLine (); + + return size; } void WriteBundledAssemblies (StreamWriter output) @@ -138,6 +180,9 @@ void WriteBundledAssemblies (StreamWriter output) uint WriteBundledAssembly (StreamWriter output, string nameLabel) { + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh XamarinAndroidBundledAssembly structure + WriteCommentLine (output, "apk_fd"); uint size = WriteData (output, (int)-1); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs index 5cdd3a6e6a2..0212933b3ab 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs @@ -5,5 +5,6 @@ class ApplicationConfigTaskState public const string RegisterTaskObjectKey = "Xamarin.Android.Tasks.ApplicationConfigTaskState"; public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } = false; + public bool UseAssembliesBlob { get; set; } = false; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs new file mode 100644 index 00000000000..3d613d2af15 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs @@ -0,0 +1,193 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class AssemblyBlobGenerator + { + const uint BlobMagic = 0x41424158; // 'XABA', little-endian + + sealed class BlobIndexEntry + { + public int Index; + + public ulong NameHash64; + public uint NameHash32; + + public uint DataOffset; + public uint DataSize; + + public uint DebugDataOffset; + public uint DebugDataSize; + + public uint ConfigDataOffset; + public uint ConfigDataSize; + }; + + const string BlobPrefix = "assemblies"; + const string BlobExtension = ".blob"; + + readonly string archiveAssembliesPrefix; + readonly TaskLoggingHelper log; + + readonly List commonAssemblies; + readonly Dictionary> archAssemblies; + + public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) + { + if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); + } + + this.archiveAssembliesPrefix = archiveAssembliesPrefix; + this.log = log; + + commonAssemblies = new List (); + archAssemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); + } + + public void Add (BlobAssemblyInfo blobAssembly) + { + if (String.IsNullOrEmpty (blobAssembly.Abi)) { + log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding common assembly {blobAssembly.FilesystemAssemblyPath}"); + commonAssemblies.Add (blobAssembly); + return; + } + + if (!archAssemblies.ContainsKey (blobAssembly.Abi)) { + archAssemblies.Add (blobAssembly.Abi, new List ()); + } + + log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding arch '{blobAssembly.Abi}' assembly {blobAssembly.FilesystemAssemblyPath}"); + archAssemblies[blobAssembly.Abi].Add (blobAssembly); + } + + public void Generate (string outputDirectory) + { + if (commonAssemblies.Count > 0) { + Generate (Path.Combine (outputDirectory, $"{BlobPrefix}{BlobExtension}"), commonAssemblies); + } + + if (archAssemblies.Count == 0) { + return; + } + + foreach (var kvp in archAssemblies) { + string abi = kvp.Key; + List assemblies = kvp.Value; + + if (assemblies.Count == 0) { + continue; + } + + Generate (Path.Combine (outputDirectory, $"{BlobPrefix}_{abi}{BlobExtension}"), assemblies); + } + } + + void Generate (string outputFilePath, List assemblies) + { + log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); + // TODO: test with satellite assemblies, their name must include the culture prefix + + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { + Generate (writer, assemblies); + writer.Flush (); + } + } + } + + void Generate (BinaryWriter writer, List assemblies) + { + Encoding assemblyNameEncoding = Encoding.UTF8; + int assemblyNameWidth = 0; + var names = new List (); + + Action updateNameWidth = (string assemblyName) => { + int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; + if (nameBytes > assemblyNameWidth) { + assemblyNameWidth = nameBytes; + } + }; + + var index = new List (); + foreach (BlobAssemblyInfo assembly in assemblies) { + log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: assembly fs path == '{assembly.FilesystemAssemblyPath}'; assembly archive path == '{assembly.ArchiveAssemblyPath}'"); + string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + assemblyName = Path.GetFileNameWithoutExtension (assemblyName); + } + + string archivePath = assembly.ArchiveAssemblyPath; + if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { + archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); + } + + if (!String.IsNullOrEmpty (assembly.Abi)) { + string abiPath = $"{assembly.Abi}/"; + if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { + archivePath = archivePath.Substring (abiPath.Length); + } + } + + if (!String.IsNullOrEmpty (archivePath)) { + assemblyName = $"{archivePath}/{assemblyName}"; + } + + log.LogMessage (MessageImportance.Low, $" => assemblyName == '{assemblyName}'; archivePath == '{archivePath}'"); + updateNameWidth (assemblyName); + + names.Add (assemblyName); + assembly.NameIndex = names.Count - 1; + + BlobIndexEntry entry = WriteAssembly (writer, assembly, assemblyName); + index.Add (entry); + entry.Index = index.Count - 1; + } + + writer.Flush (); + writer.Seek (0, SeekOrigin.Begin); + + // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh + writer.Write (BlobMagic); // magic + writer.Write ((uint)index.Count); // entry_count + writer.Write ((uint)assemblyNameWidth); // name_width + + var sortedIndex = new List (index); + sortedIndex.Sort ((BlobIndexEntry a, BlobIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); + foreach (BlobIndexEntry entry in sortedIndex) { + writer.Write (entry.NameHash32); + writer.Write (entry.Index); + } + + sortedIndex.Sort ((BlobIndexEntry a, BlobIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); + foreach (BlobIndexEntry entry in sortedIndex) { + writer.Write (entry.NameHash64); + writer.Write (entry.Index); + } + + foreach (BlobIndexEntry entry in index) { + writer.Write (entry.DataOffset); + writer.Write (entry.DataSize); + writer.Write (entry.DebugDataOffset); + writer.Write (entry.DebugDataSize); + writer.Write (entry.ConfigDataOffset); + writer.Write (entry.ConfigDataSize); + } + + // TODO: write the names array + } + + BlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName) + { + var ret = new BlobIndexEntry (); + + return ret; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs new file mode 100644 index 00000000000..7b990a2939a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs @@ -0,0 +1,49 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Tasks +{ + class BlobAssemblyInfo + { + public string FilesystemAssemblyPath { get; } + public string ArchiveAssemblyPath { get; } + public string DebugInfoPath { get; private set; } + public string ConfigPath { get; private set; } + public string Abi { get; } + public int NameIndex { get; set; } + + public BlobAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) + { + if (String.IsNullOrEmpty (filesystemAssemblyPath)) { + throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); + } + + if (String.IsNullOrEmpty (archiveAssemblyPath)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); + } + + FilesystemAssemblyPath = filesystemAssemblyPath; + ArchiveAssemblyPath = archiveAssemblyPath; + Abi = abi; + } + + public void SetDebugInfoPath (string path) + { + DebugInfoPath = GetExistingPath (path); + } + + public void SetConfigPath (string path) + { + ConfigPath = GetExistingPath (path); + } + + string GetExistingPath (string path) + { + if (String.IsNullOrEmpty (path) || !File.Exists (path)) { + return String.Empty; + } + + return path; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 8e8d4fdacd7..02f03841016 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -58,6 +58,7 @@ + diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index d88d57c8807..b3ef43b3325 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -47,8 +47,9 @@ ApplicationConfig application_config = { .package_naming_policy = 0, .environment_variable_count = 0, .system_property_count = 0, - .number_of_assemblies_in_apk = 0, - .bundled_assembly_name_width = 0, + .number_of_assemblies_in_apk = 2, + .number_of_common_blob_assemblies = 2, + .number_of_arch_blob_assemblies = 2, .android_package_name = "com.xamarin.test", }; @@ -80,3 +81,13 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { .name = second_assembly_name, }, }; + +uint8_t* blob_bundled_assemblies_arch[] = { + nullptr, + nullptr, +}; + +uint8_t* blob_bundled_assemblies_common[] = { + nullptr, + nullptr, +}; diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index 7a4856aac61..331878ab45a 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -1,6 +1,7 @@ #ifndef __BASIC_UTILITIES_HH #define __BASIC_UTILITIES_HH +#include #include #include #include @@ -122,6 +123,19 @@ namespace xamarin::android return memcmp (str.get () + len - end_length, end, end_length) == 0; } + template + bool ends_with (internal::string_base const& str, std::array const& end) const noexcept + { + constexpr size_t end_length = N - 1; + + size_t len = str.length (); + if (XA_UNLIKELY (len < end_length)) { + return false; + } + + return memcmp (str.get () + len - end_length, end.data (), end_length) == 0; + } + template const TChar* find_last (internal::string_base const& str, const char ch) const noexcept { diff --git a/src/monodroid/jni/cpp-util.hh b/src/monodroid/jni/cpp-util.hh index 2ca70bbe605..8cb9fe6e3ff 100644 --- a/src/monodroid/jni/cpp-util.hh +++ b/src/monodroid/jni/cpp-util.hh @@ -1,6 +1,7 @@ #ifndef __CPP_UTIL_HH #define __CPP_UTIL_HH +#include #include #include #include @@ -54,5 +55,26 @@ namespace xamarin::android template using c_unique_ptr = std::unique_ptr>; + + template + constexpr auto concat_const (const char (&...parts)[Length]) + { + // `parts` being constant string arrays, Length for each of them includes the trailing NUL byte, thus the + // `sizeof... (Length)` part which subtracts the number of template parameters - the amount of NUL bytes so that + // we don't waste space. + constexpr size_t total_length = (... + Length) - sizeof... (Length); + std::array ret; + ret[total_length] = 0; + + size_t i = 0; + for (char const* from : {parts...}) { + for (; *from != '\0'; i++) { + ret[i] = *from++; + } + } + + return ret; + }; + } #endif // !def __CPP_UTIL_HH diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 4c58d4ad545..3efe8c314d1 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -34,48 +34,60 @@ EmbeddedAssemblies::is_debug_file (dynamic_local_string const ; } -void -EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) +force_inline bool +EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept { - uint32_t cd_offset; - uint32_t cd_size; - uint16_t cd_entries; + entry_name.clear (); + + bool result = zip_read_entry_info (buf, entry_name, state); - if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) { - log_fatal (LOG_ASSEMBLY, "Failed to read the EOCD record from APK file %s", apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } #ifdef DEBUG - log_info (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); - log_info (LOG_ASSEMBLY, "Central directory size: %u", cd_size); - log_info (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); + log_info (LOG_ASSEMBLY, "%s entry: %s", state.apk_name, entry_name.get () == nullptr ? "unknown" : entry_name.get ()); #endif - off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); - if (retval < 0) { - log_fatal (LOG_ASSEMBLY, "Failed to seek to central directory position in the APK file %s. %s (result: %d; errno: %d)", apk_name, std::strerror (errno), retval, errno); + if (!result || entry_name.empty ()) { + log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory info for entry %u in APK file %s", entry_index, state.apk_name); exit (FATAL_EXIT_NO_ASSEMBLIES); } - std::vector buf (cd_size); - const char *prefix = get_assemblies_prefix (); - uint32_t prefix_len = get_assemblies_prefix_length (); - size_t buf_offset = 0; - uint16_t compression_method; - uint32_t local_header_offset; - uint32_t data_offset; - uint32_t file_size; - - ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); - if (static_cast(nread) != cd_size) { - log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory from the APK archive %s. %s (nread: %d; errno: %d)", apk_name, std::strerror (errno), nread, errno); + if (!zip_adjust_data_offset (state.apk_fd, state)) { + log_fatal (LOG_ASSEMBLY, "Failed to adjust data start offset for entry %u in APK file %s", entry_index, state.apk_name); exit (FATAL_EXIT_NO_ASSEMBLIES); } +#ifdef DEBUG + log_info (LOG_ASSEMBLY, " ZIP: local header offset: %u; data offset: %u; file size: %u", state.local_header_offset, state.data_offset, state.file_size); +#endif + if (state.compression_method != 0) { + return false; + } + + if (entry_name.get ()[0] != state.prefix[0] || strncmp (state.prefix, entry_name.get (), state.prefix_len) != 0) { + return false; + } - dynamic_local_string entry_name; #if defined (NET6) - bool runtime_config_blob_found = false; + if (application_config.have_runtime_config_blob && !state.runtime_config_blob_found) { + if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { + state.runtime_config_blob_found = true; + runtime_config_blob_mmap = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); + return false; + } + } #endif // def NET6 + // assemblies must be 4-byte aligned, or Bad Things happen + if ((state.data_offset & 0x3) != 0) { + log_fatal (LOG_ASSEMBLY, "Assembly '%s' is located at bad offset %lu within the .apk\n", entry_name.get (), state.data_offset); + log_fatal (LOG_ASSEMBLY, "You MUST run `zipalign` on %s\n", strrchr (state.apk_name, '/') + 1); + exit (FATAL_EXIT_MISSING_ZIPALIGN); + } + + return true; +} + +force_inline void +EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept +{ + dynamic_local_string entry_name; bool bundled_assemblies_slow_path = bundled_assembly_index >= application_config.number_of_assemblies_in_apk; uint32_t max_assembly_name_size = application_config.bundled_assembly_name_width - 1; @@ -87,47 +99,10 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus // However, clang-tidy can't know that the value is owned by Mono and we must not free it, thus the suppression. // // NOLINTNEXTLINE(clang-analyzer-unix.Malloc) - for (size_t i = 0; i < cd_entries; i++) { - entry_name.clear (); - - bool result = zip_read_entry_info (buf, buf_offset, compression_method, local_header_offset, file_size, entry_name); - -#ifdef DEBUG - log_info (LOG_ASSEMBLY, "%s entry: %s", apk_name, entry_name.get () == nullptr ? "unknown" : entry_name.get ()); -#endif - if (!result || entry_name.empty ()) { - log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory info for entry %u in APK file %s", i, apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } - - if (!zip_adjust_data_offset (fd, local_header_offset, data_offset)) { - log_fatal (LOG_ASSEMBLY, "Failed to adjust data start offset for entry %u in APK file %s", i, apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } -#ifdef DEBUG - log_info (LOG_ASSEMBLY, " ZIP: local header offset: %u; data offset: %u; file size: %u", local_header_offset, data_offset, file_size); -#endif - if (compression_method != 0) + for (size_t i = 0; i < num_entries; i++) { + bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); + if (!interesting_entry) { continue; - - if (entry_name.get ()[0] != prefix[0] || strncmp (prefix, entry_name.get (), prefix_len) != 0) - continue; - -#if defined (NET6) - if (application_config.have_runtime_config_blob && !runtime_config_blob_found) { - if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { - runtime_config_blob_found = true; - runtime_config_blob_mmap = md_mmap_apk_file (fd, data_offset, file_size, entry_name.get ()); - continue; - } - } -#endif // def NET6 - - // assemblies must be 4-byte aligned, or Bad Things happen - if ((data_offset & 0x3) != 0) { - log_fatal (LOG_ASSEMBLY, "Assembly '%s' is located at bad offset %lu within the .apk\n", entry_name.get (), data_offset); - log_fatal (LOG_ASSEMBLY, "You MUST run `zipalign` on %s\n", strrchr (apk_name, '/') + 1); - exit (FATAL_EXIT_MISSING_ZIPALIGN); } #if defined (DEBUG) @@ -144,7 +119,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } bundled_debug_data->emplace_back (); - set_debug_entry_data (bundled_debug_data->back (), fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_debug_entry_data (bundled_debug_data->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); continue; } @@ -154,7 +129,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus // Remove '.config' suffix *strrchr (assembly_name, '.') = '\0'; - md_mmap_info map_info = md_mmap_apk_file (fd, data_offset, file_size, entry_name.get ()); + md_mmap_info map_info = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); mono_register_config_for_assembly (assembly_name, (const char*)map_info.area); continue; @@ -181,17 +156,98 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus extra_bundled_assemblies->emplace_back (); // means we need to allocate memory to store the entry name, only the entries pre-allocated during // build have valid pointer to the name storage area - set_entry_data (extra_bundled_assemblies->back (), fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_entry_data (extra_bundled_assemblies->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); continue; } - set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); bundled_assembly_index++; } have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } +force_inline void +EmbeddedAssemblies::zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +{ + dynamic_local_string entry_name; + bool common_blob_found = false; + bool arch_blob_found = false; + + for (size_t i = 0; i < num_entries; i++) { + bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); + if (!interesting_entry) { + continue; + } + + if (!common_blob_found && utils.ends_with (entry_name, bundled_assemblies_common_blob_name)) { + common_blob_found = true; + // TODO: mmap + } + + if (!arch_blob_found && utils.ends_with (entry_name, bundled_assemblies_arch_blob_name)) { + arch_blob_found = true; + // TODO: mmap + } + + if (common_blob_found && arch_blob_found) { +#if NET6 + if ((application_config.have_runtime_config_blob && state.runtime_config_blob_found) || !application_config.have_runtime_config_blob) +#endif + { + break; + } + } + } +} + +void +EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) +{ + uint32_t cd_offset; + uint32_t cd_size; + uint16_t cd_entries; + + if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) { + log_fatal (LOG_ASSEMBLY, "Failed to read the EOCD record from APK file %s", apk_name); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } +#ifdef DEBUG + log_info (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); + log_info (LOG_ASSEMBLY, "Central directory size: %u", cd_size); + log_info (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); +#endif + off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); + if (retval < 0) { + log_fatal (LOG_ASSEMBLY, "Failed to seek to central directory position in the APK file %s. %s (result: %d; errno: %d)", apk_name, std::strerror (errno), retval, errno); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } + + std::vector buf (cd_size); + ZipEntryLoadState state { + .apk_fd = fd, + .apk_name = apk_name, + .prefix = get_assemblies_prefix (), + .prefix_len = get_assemblies_prefix_length (), + .buf_offset = 0, +#if defined (NET6) + .runtime_config_blob_found = false, +#endif // def NET6 + }; + + ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); + if (static_cast(nread) != cd_size) { + log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory from the APK archive %s. %s (nread: %d; errno: %d)", apk_name, std::strerror (errno), nread, errno); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } + + if (application_config.have_assemblies_blob) { + zip_load_blob_assembly_entries (buf, cd_size, state); + } else { + zip_load_individual_assembly_entries (buf, cd_size, should_register, state); + } +} + template force_inline void EmbeddedAssemblies::set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept @@ -288,14 +344,14 @@ EmbeddedAssemblies::zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_ } bool -EmbeddedAssemblies::zip_adjust_data_offset (int fd, size_t local_header_offset, uint32_t &data_start_offset) +EmbeddedAssemblies::zip_adjust_data_offset (int fd, ZipEntryLoadState &state) { static constexpr size_t LH_FILE_NAME_LENGTH_OFFSET = 26; static constexpr size_t LH_EXTRA_LENGTH_OFFSET = 28; - off_t result = ::lseek (fd, static_cast(local_header_offset), SEEK_SET); + off_t result = ::lseek (fd, static_cast(state.local_header_offset), SEEK_SET); if (result < 0) { - log_error (LOG_ASSEMBLY, "Failed to seek to archive entry local header at offset %u. %s (result: %d; errno: %d)", local_header_offset, result, errno); + log_error (LOG_ASSEMBLY, "Failed to seek to archive entry local header at offset %u. %s (result: %d; errno: %d)", state.local_header_offset, result, errno); return false; } @@ -304,36 +360,36 @@ EmbeddedAssemblies::zip_adjust_data_offset (int fd, size_t local_header_offset, ssize_t nread = ::read (fd, local_header.data (), local_header.size ()); if (nread < 0 || nread != ZIP_LOCAL_LEN) { - log_error (LOG_ASSEMBLY, "Failed to read local header at offset %u: %s (nread: %d; errno: %d)", local_header_offset, std::strerror (errno), nread, errno); + log_error (LOG_ASSEMBLY, "Failed to read local header at offset %u: %s (nread: %d; errno: %d)", state.local_header_offset, std::strerror (errno), nread, errno); return false; } size_t index = 0; if (!zip_read_field (local_header, index, signature)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header entry signature at offset %u", local_header_offset); + log_error (LOG_ASSEMBLY, "Failed to read Local Header entry signature at offset %u", state.local_header_offset); return false; } if (memcmp (signature.data (), ZIP_LOCAL_MAGIC, signature.size ()) != 0) { - log_error (LOG_ASSEMBLY, "Invalid Local Header entry signature at offset %u", local_header_offset); + log_error (LOG_ASSEMBLY, "Invalid Local Header entry signature at offset %u", state.local_header_offset); return false; } uint16_t file_name_length; index = LH_FILE_NAME_LENGTH_OFFSET; if (!zip_read_field (local_header, index, file_name_length)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header 'file name length' field at offset %u", (local_header_offset + index)); + log_error (LOG_ASSEMBLY, "Failed to read Local Header 'file name length' field at offset %u", (state.local_header_offset + index)); return false; } uint16_t extra_field_length; index = LH_EXTRA_LENGTH_OFFSET; if (!zip_read_field (local_header, index, extra_field_length)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header 'extra field length' field at offset %u", (local_header_offset + index)); + log_error (LOG_ASSEMBLY, "Failed to read Local Header 'extra field length' field at offset %u", (state.local_header_offset + index)); return false; } - data_start_offset = static_cast(local_header_offset) + file_name_length + extra_field_length + local_header.size (); + state.data_offset = static_cast(state.local_header_offset) + file_name_length + extra_field_length + local_header.size (); return true; } @@ -433,7 +489,7 @@ EmbeddedAssemblies::zip_read_field (T const& buf, size_t index, size_t count, dy } bool -EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t& buf_offset, uint16_t& compression_method, uint32_t& local_header_offset, uint32_t& file_size, dynamic_local_string& file_name) +EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state) { constexpr size_t CD_COMPRESSION_METHOD_OFFSET = 10; constexpr size_t CD_UNCOMPRESSED_SIZE_OFFSET = 24; @@ -442,7 +498,7 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t constexpr size_t CD_LOCAL_HEADER_POS_OFFSET = 42; constexpr size_t CD_COMMENT_LENGTH_OFFSET = 32; - size_t index = buf_offset; + size_t index = state.buf_offset; zip_ensure_valid_params (buf, index, ZIP_CENTRAL_LEN); std::array signature; @@ -456,45 +512,45 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t return false; } - index = buf_offset + CD_COMPRESSION_METHOD_OFFSET; - if (!zip_read_field (buf, index, compression_method)) { + index = state.buf_offset + CD_COMPRESSION_METHOD_OFFSET; + if (!zip_read_field (buf, index, state.compression_method)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'compression method' field"); return false; } - index = buf_offset + CD_UNCOMPRESSED_SIZE_OFFSET;; - if (!zip_read_field (buf, index, file_size)) { + index = state.buf_offset + CD_UNCOMPRESSED_SIZE_OFFSET;; + if (!zip_read_field (buf, index, state.file_size)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'uncompressed size' field"); return false; } uint16_t file_name_length; - index = buf_offset + CD_FILENAME_LENGTH_OFFSET; + index = state.buf_offset + CD_FILENAME_LENGTH_OFFSET; if (!zip_read_field (buf, index, file_name_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'file name length' field"); return false; } uint16_t extra_field_length; - index = buf_offset + CD_EXTRA_LENGTH_OFFSET; + index = state.buf_offset + CD_EXTRA_LENGTH_OFFSET; if (!zip_read_field (buf, index, extra_field_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'extra field length' field"); return false; } uint16_t comment_length; - index = buf_offset + CD_COMMENT_LENGTH_OFFSET; + index = state.buf_offset + CD_COMMENT_LENGTH_OFFSET; if (!zip_read_field (buf, index, comment_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'file comment length' field"); return false; } - index = buf_offset + CD_LOCAL_HEADER_POS_OFFSET; - if (!zip_read_field (buf, index, local_header_offset)) { + index = state.buf_offset + CD_LOCAL_HEADER_POS_OFFSET; + if (!zip_read_field (buf, index, state.local_header_offset)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'relative offset of local header' field"); return false; } - index += sizeof(local_header_offset); + index += sizeof(state.local_header_offset); if (file_name_length == 0) { file_name.clear (); @@ -503,6 +559,6 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t return false; } - buf_offset += ZIP_CENTRAL_LEN + file_name_length + extra_field_length + comment_length; + state.buf_offset += ZIP_CENTRAL_LEN + file_name_length + extra_field_length + comment_length; return true; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 97f37aef34b..3bee6df7f41 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -55,6 +55,22 @@ namespace xamarin::android::internal { size_t size; }; + struct ZipEntryLoadState + { + int apk_fd; + const char * const apk_name; + const char * const prefix; + uint32_t prefix_len; + size_t buf_offset; + uint16_t compression_method; + uint32_t local_header_offset; + uint32_t data_offset; + uint32_t file_size; +#if defined (NET6) + bool runtime_config_blob_found; +#endif + }; + private: static constexpr char ZIP_CENTRAL_MAGIC[] = "PK\1\2"; static constexpr char ZIP_LOCAL_MAGIC[] = "PK\3\4"; @@ -63,6 +79,22 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; static constexpr char assemblies_prefix[] = "assemblies/"; + + static constexpr char bundled_assemblies_blob_prefix[] = "assemblies"; + static constexpr char bundled_assemblies_blob_ext[] = ".blob"; +#if __arm__ + static constexpr char bundled_assemblies_blob_arch[] = "armeabi-v7a"; +#elif __aarch64__ + static constexpr char bundled_assemblies_blob_arch[] = "arm64-v8a"; +#elif __x86_64__ + static constexpr char bundled_assemblies_blob_arch[] = "x86_64"; +#elif __i386__ + static constexpr char bundled_assemblies_blob_arch[] = "x86"; +#endif + static constexpr auto bundled_assemblies_common_blob_name = concat_const ("/", bundled_assemblies_blob_prefix, bundled_assemblies_blob_ext); + static constexpr auto bundled_assemblies_arch_blob_name = concat_const ("/", bundled_assemblies_blob_prefix, "_", bundled_assemblies_blob_arch, bundled_assemblies_blob_ext); + + #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; #endif @@ -161,8 +193,11 @@ namespace xamarin::android::internal { static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size); void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); + void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; + void zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; + bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); - bool zip_adjust_data_offset (int fd, size_t local_header_offset, uint32_t &data_start_offset); + bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); template bool zip_extract_cd_info (std::array const& buf, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); @@ -193,7 +228,7 @@ namespace xamarin::android::internal { template bool zip_read_field (T const& buf, size_t index, size_t count, dynamic_local_string& characters) const noexcept; - bool zip_read_entry_info (std::vector const& buf, size_t& buf_offset, uint16_t& compression_method, uint32_t& local_header_offset, uint32_t& file_size, dynamic_local_string& file_name); + bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); const char* get_assemblies_prefix () const { @@ -229,6 +264,8 @@ namespace xamarin::android::internal { bool register_debug_symbols; bool have_and_want_debug_symbols; size_t bundled_assembly_index = 0; + size_t bundled_assembly_blob_common_count = 0; + size_t bundled_assembly_blob_arch_count = 0; #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; TypeMappingInfo *managed_to_java_maps; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 06f7668da62..4f8e74029bf 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -10,6 +10,7 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t BUNDLED_ASSEMBLIES_INDEX_MAGIC = 0x41424158; // 'XABA', little-endian static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -106,6 +107,54 @@ struct XamarinAndroidBundledAssembly final char *name; }; +struct BlobHashEntry final +{ + union { + uint64_t hash64; + uint32_t hash32; + }; + + uint32_t index; +}; + +struct BlobBundledAssembly final +{ + uint32_t name_index; + + uint32_t data_offset; + uint32_t data_size; + + uint32_t debug_data_offset; + uint32_t debug_data_size; + + uint32_t config_data_offset; + uint32_t config_data_size; +}; + +// +// Blob format +// +// The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. +// The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the +// blob but also into the `uint8_t*` `blob_bundled_assemblies*` arrays. +// +// This way the `assemblies` array in the blob can remain read only, because we write the "mapped" assembly +// pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. +// +// BundledAssemblyBlobHeader header; +// BlobHashEntry hashes32[header.entry_count]; +// BlobHashEntry hashes64[header.entry_count]; +// XamarinAndroidBundledAssembly assemblies[header.entry_count]; +// char assembly_names[header.entry_count][header.name_width]; +// [DATA] +// +struct BundledAssemblyBlobHeader final +{ + uint32_t magic; + uint32_t entry_count; + uint32_t name_width; +}; + struct ApplicationConfig { bool uses_mono_llvm; @@ -116,11 +165,14 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; + bool have_assemblies_blob; uint8_t bound_exception_type; uint32_t package_naming_policy; uint32_t environment_variable_count; uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; + uint32_t number_of_common_blob_assemblies; + uint32_t number_of_arch_blob_assemblies; uint32_t bundled_assembly_name_width; const char *android_package_name; }; @@ -145,5 +197,7 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; +MONO_API uint8_t* blob_bundled_assemblies_arch[]; +MONO_API uint8_t* blob_bundled_assemblies_common[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From 1cee121f17f0becf58b13b97f6cbe0873f4dc4f2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 13 Sep 2021 23:25:20 +0200 Subject: [PATCH 02/54] [WIP] Blob generation works --- .../installers/create-installers.targets | 1 + .../Tasks/BuildApk.cs | 11 +- .../Utilities/ArchAssemblyBlob.cs | 73 ++++ .../Utilities/AssemblyBlob.cs | 334 ++++++++++++++++++ .../Utilities/AssemblyBlobGenerator.cs | 192 +++------- .../Utilities/AssemblyBlobIndexEntry.cs | 42 +++ .../Utilities/BlobAssemblyInfo.cs | 1 - .../Utilities/CommonAssemblyBlob.cs | 41 +++ src/monodroid/jni/xamarin-app.hh | 26 +- 9 files changed, 574 insertions(+), 147 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index c74d6387bfc..cf289da4f98 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -290,6 +290,7 @@ <_MSBuildFiles Include="$(MSBuildSrcDir)\Xamarin.SourceWriter.dll" /> <_MSBuildFiles Include="$(MSBuildSrcDir)\Xamarin.SourceWriter.pdb" /> <_MSBuildFiles Include="$(MSBuildSrcDir)\K4os.Compression.LZ4.dll" /> + <_MSBuildFiles Include="$(MSBuildSrcDir)\K4os.Hash.xxHash.dll" /> <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index bf64bafa6aa..ecff8d87849 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -340,6 +340,13 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); } void AddAssembliesFromCollection (ITaskItem[] assemblies) @@ -655,7 +662,7 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log); if (ndk == null) { - return; // NdkTools.Create will log appropriate error + return; // NdkTools.Create will Log appropriate error } string clangDir = ndk.GetClangDeviceLibraryPath (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs new file mode 100644 index 00000000000..09eb9342a13 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class ArchAssemblyBlob : AssemblyBlob + { + readonly Dictionary> assemblies; + HashSet seenArchAssemblyNames; + + public ArchAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) + : base (apkName, archiveAssembliesPrefix, log) + { + assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); + } + + public override void WriteIndex (List globalIndex) + { + throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); + } + + public override void Add (BlobAssemblyInfo blobAssembly) + { + if (String.IsNullOrEmpty (blobAssembly.Abi)) { + throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); + } + + if (!assemblies.ContainsKey (blobAssembly.Abi)) { + assemblies.Add (blobAssembly.Abi, new List ()); + } + + Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Arch '{blobAssembly.Abi}' assembly {blobAssembly.FilesystemAssemblyPath}"); + List blobAssemblies = assemblies[blobAssembly.Abi]; + blobAssemblies.Add (blobAssembly); + + if (seenArchAssemblyNames == null) { + seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); + } + + string assemblyName = GetAssemblyName (blobAssembly); + if (seenArchAssemblyNames.Contains (assemblyName)) { + return; + } + + seenArchAssemblyNames.Add (assemblyName); + AssemblyIndex.Add (new AssemblyBlobIndexEntry (assemblyName, ID, (uint)blobAssemblies.Count - 1)); + } + + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + { + if (assemblies.Count == 0) { + return; + } + + // TODO: make sure all the lists are identical + foreach (var kvp in assemblies) { + string abi = kvp.Key; + List archAssemblies = kvp.Value; + + if (archAssemblies.Count == 0) { + continue; + } + + Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}_{abi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths); + } + + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs new file mode 100644 index 00000000000..17ef8e3871c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -0,0 +1,334 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + abstract class AssemblyBlob + { + + const uint BlobMagic = 0x41424158; // 'XABA', little-endian + + // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh + const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); + + // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh + const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (2 * sizeof (uint)); + + // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh + const uint BlobHeaderNativeStructSize = sizeof (uint) * 4; + + protected const string BlobPrefix = "assemblies"; + protected const string BlobExtension = ".blob"; + + static readonly ArrayPool bytePool = ArrayPool.Shared; + static uint id = 0; + + string archiveAssembliesPrefix; + string indexBlobPath; + + protected string ApkName { get; } + protected TaskLoggingHelper Log { get; } + + public uint ID { get; } + public bool IsIndexBlob => ID == 0; + + public List AssemblyIndex { get; } + + protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) + { + if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); + } + + if (String.IsNullOrEmpty (apkName)) { + throw new ArgumentException ("must not be null or empty", nameof (apkName)); + } + + // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock + ID = id++; + + this.archiveAssembliesPrefix = archiveAssembliesPrefix; + ApkName = apkName; + Log = log; + + AssemblyIndex = new List (); + } + + public abstract void Add (BlobAssemblyInfo blobAssembly); + public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); + + public virtual void WriteIndex (List globalIndex) + { + if (!IsIndexBlob) { + throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); + } + + if (String.IsNullOrEmpty (indexBlobPath)) { + throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); + } + + if (globalIndex == null) { + throw new ArgumentNullException (nameof (globalIndex)); + } + + string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; + using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { + using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { + WriteIndex (writer, globalIndex); + writer.Flush (); + } + + using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + ifs.CopyTo (hfs); + hfs.Flush (); + } + } + + File.Delete (indexBlobPath); + File.Move (indexBlobHeaderPath, indexBlobPath); + } + + void WriteIndex (BinaryWriter writer, List globalIndex) + { + uint localEntryCount = 0; + var localAssemblies = new List (); + + // TODO: check if there are no duplicates here + foreach (AssemblyBlobIndexEntry assembly in globalIndex) { + if (assembly.BlobID == ID) { + localEntryCount++; + localAssemblies.Add (assembly); + } + } + + uint globalAssemblyCount = (uint)globalIndex.Count; + + Log.LogMessage (MessageImportance.Low, $"Index blob, writing header (local assemblies: {localAssemblies.Count}; global assemblies: {globalIndex.Count})"); + writer.Seek (0, SeekOrigin.Begin); + WriteBlobHeader (writer, localEntryCount, globalAssemblyCount); + + // Header and two tables of the same size, each for 32 and 64-bit hashes + uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); + + Log.LogMessage (MessageImportance.Low, "Index blob, writing assembly descriptors"); + WriteAssemblyDescriptors (writer, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); + + Log.LogMessage (MessageImportance.Low, $"Index blob, writing hash tables ({globalAssemblyCount * 2} entries, {offsetFixup} bytes"); + var sortedIndex = new List (globalIndex); + sortedIndex.Sort ((AssemblyBlobIndexEntry a, AssemblyBlobIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); + foreach (AssemblyBlobIndexEntry entry in sortedIndex) { + WriteHash (entry, entry.NameHash32); + } + + sortedIndex.Sort ((AssemblyBlobIndexEntry a, AssemblyBlobIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); + foreach (AssemblyBlobIndexEntry entry in sortedIndex) { + WriteHash (entry, entry.NameHash64); + } + + void WriteHash (AssemblyBlobIndexEntry entry, ulong hash) + { + writer.Write (hash); + writer.Write (entry.BlobID); + writer.Write (entry.Index); + } + } + + protected string GetAssemblyName (BlobAssemblyInfo assembly) + { + string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + assemblyName = Path.GetFileNameWithoutExtension (assemblyName); + } + + return assemblyName; + } + + protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths) + { + if (globalIndex == null) { + throw new ArgumentNullException (nameof (globalIndex)); + } + + if (blobPaths == null) { + throw new ArgumentNullException (nameof (blobPaths)); + } + + if (IsIndexBlob) { + indexBlobPath = outputFilePath; + } + + blobPaths.Add (outputFilePath); + Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); + // TODO: test with satellite assemblies, their name must include the culture prefix + + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { + Generate (writer, assemblies, globalIndex); + writer.Flush (); + } + } + } + + void Generate (BinaryWriter writer, List assemblies, List globalIndex) + { + var localAssemblies = new List (); + + if (!IsIndexBlob) { + // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different + // way. + uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); + var zeros = bytePool.Rent ((int)nbytes); + writer.Write (zeros, 0, (int)nbytes); + bytePool.Return (zeros); + } + + foreach (BlobAssemblyInfo assembly in assemblies) { + Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: assembly hfs path == '{assembly.FilesystemAssemblyPath}'; assembly archive path == '{assembly.ArchiveAssemblyPath}'"); + string assemblyName = GetAssemblyName (assembly); + string archivePath = assembly.ArchiveAssemblyPath; + if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { + archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); + } + + if (!String.IsNullOrEmpty (assembly.Abi)) { + string abiPath = $"{assembly.Abi}/"; + if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { + archivePath = archivePath.Substring (abiPath.Length); + } + } + + if (!String.IsNullOrEmpty (archivePath)) { + assemblyName = $"{archivePath}/{assemblyName}"; + } + + AssemblyBlobIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); + Log.LogMessage (MessageImportance.Low, $" => assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); + globalIndex.Add (entry); + localAssemblies.Add (entry); + } + + writer.Flush (); + + if (IsIndexBlob) { + return; + } + + Log.LogMessage (MessageImportance.Low, $"Not an index blob, writing header (local assemblies: {localAssemblies.Count})"); + writer.Seek (0, SeekOrigin.Begin); + WriteBlobHeader (writer, (uint)localAssemblies.Count); + + Log.LogMessage (MessageImportance.Low, "Not an index blob, writing assembly descriptors"); + WriteAssemblyDescriptors (writer, localAssemblies); + } + + uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) + { + return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; + } + + void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) + { + // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh + writer.Write (BlobMagic); // magic + writer.Write (localEntryCount); // local_entry_count + writer.Write (globalEntryCount); // global_entry_count + writer.Write ((uint)ID); // blob_id + } + + void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) + { + // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh + + foreach (AssemblyBlobIndexEntry assembly in assemblies) { + Log.LogMessage (MessageImportance.Low, $" => {assembly.Name} before adjustment: data offset == {assembly.DataOffset}"); + AdjustOffsets (assembly, offsetFixup); + Log.LogMessage (MessageImportance.Low, $" => {assembly.Name} after adjustment: data offset == {assembly.DataOffset}"); + + writer.Write (assembly.DataOffset); + writer.Write (assembly.DataSize); + + writer.Write (assembly.DebugDataOffset); + writer.Write (assembly.DebugDataSize); + + writer.Write (assembly.ConfigDataOffset); + writer.Write (assembly.ConfigDataSize); + } + } + + void AdjustOffsets (AssemblyBlobIndexEntry assembly, uint offsetFixup) + { + if (offsetFixup == 0) { + return; + } + + assembly.DataOffset += offsetFixup; + + if (assembly.DebugDataOffset > 0) { + assembly.DebugDataOffset += offsetFixup; + } + + if (assembly.ConfigDataOffset > 0) { + assembly.ConfigDataOffset += offsetFixup; + } + } + + AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName, uint index) + { + uint offset; + uint size; + + (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); + var ret = new AssemblyBlobIndexEntry (assemblyName, ID, index) { + DataOffset = offset, + DataSize = size, + }; + + (offset, size) = WriteFile (assembly.DebugInfoPath, false); + if (offset != 0 && size != 0) { + ret.DebugDataOffset = offset; + ret.DebugDataSize = size; + } + + (offset, size) = WriteFile (assembly.ConfigPath, false); + if (offset != 0 && size != 0) { + ret.ConfigDataOffset = offset; + ret.ConfigDataSize = size; + } + + return ret; + + (uint offset, uint size) WriteFile (string filePath, bool required) + { + if (!File.Exists (filePath)) { + if (required) { + throw new InvalidOperationException ($"Required file '{filePath}' not found"); + } + + return (0, 0); + } + + var fi = new FileInfo (filePath); + if (fi.Length == 0) { + return (0, 0); + } + + if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { + throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); + } + + uint offset = (uint)writer.BaseStream.Position; + using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + fs.CopyTo (writer.BaseStream); + } + + return (offset, (uint)fi.Length); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs index 3d613d2af15..cb45d2123c4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs @@ -10,33 +10,19 @@ namespace Xamarin.Android.Tasks { class AssemblyBlobGenerator { - const uint BlobMagic = 0x41424158; // 'XABA', little-endian - - sealed class BlobIndexEntry + sealed class Blob { - public int Index; - - public ulong NameHash64; - public uint NameHash32; - - public uint DataOffset; - public uint DataSize; - - public uint DebugDataOffset; - public uint DebugDataSize; - - public uint ConfigDataOffset; - public uint ConfigDataSize; - }; - - const string BlobPrefix = "assemblies"; - const string BlobExtension = ".blob"; + public AssemblyBlob Common; + public AssemblyBlob Arch; + } readonly string archiveAssembliesPrefix; readonly TaskLoggingHelper log; - readonly List commonAssemblies; - readonly Dictionary> archAssemblies; + // NOTE: when/if we have parallel BuildApk this should become a ConcurrentDictionary + readonly Dictionary blobs; + + AssemblyBlob indexBlob; public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) { @@ -47,147 +33,85 @@ public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper this.archiveAssembliesPrefix = archiveAssembliesPrefix; this.log = log; - commonAssemblies = new List (); - archAssemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); + blobs = new Dictionary (StringComparer.Ordinal); } public void Add (BlobAssemblyInfo blobAssembly) { - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding common assembly {blobAssembly.FilesystemAssemblyPath}"); - commonAssemblies.Add (blobAssembly); - return; - } + Add ("base", blobAssembly); + } - if (!archAssemblies.ContainsKey (blobAssembly.Abi)) { - archAssemblies.Add (blobAssembly.Abi, new List ()); + public void Add (string apkName, BlobAssemblyInfo blobAssembly) + { + if (String.IsNullOrEmpty (apkName)) { + throw new ArgumentException ("must not be null or empty", nameof (apkName)); } - log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding arch '{blobAssembly.Abi}' assembly {blobAssembly.FilesystemAssemblyPath}"); - archAssemblies[blobAssembly.Abi].Add (blobAssembly); - } + Blob blob; + if (!blobs.ContainsKey (apkName)) { + blob = new Blob { + Common = new CommonAssemblyBlob (apkName, archiveAssembliesPrefix, log), + Arch = new ArchAssemblyBlob (apkName, archiveAssembliesPrefix, log) + }; - public void Generate (string outputDirectory) - { - if (commonAssemblies.Count > 0) { - Generate (Path.Combine (outputDirectory, $"{BlobPrefix}{BlobExtension}"), commonAssemblies); + blobs.Add (apkName, blob); + SetIndexBlob (blob.Common); + SetIndexBlob (blob.Arch); } - if (archAssemblies.Count == 0) { - return; + blob = blobs[apkName]; + if (String.IsNullOrEmpty (blobAssembly.Abi)) { + blob.Common.Add (blobAssembly); + } else { + blob.Arch.Add (blobAssembly); } - foreach (var kvp in archAssemblies) { - string abi = kvp.Key; - List assemblies = kvp.Value; + void SetIndexBlob (AssemblyBlob b) + { + if (!b.IsIndexBlob) { + return; + } - if (assemblies.Count == 0) { - continue; + if (indexBlob != null) { + throw new InvalidOperationException ("Index blob already set!"); } - Generate (Path.Combine (outputDirectory, $"{BlobPrefix}_{abi}{BlobExtension}"), assemblies); + indexBlob = b; } } - void Generate (string outputFilePath, List assemblies) + public Dictionary> Generate (string outputDirectory) { - log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - // TODO: test with satellite assemblies, their name must include the culture prefix - - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies); - writer.Flush (); - } + if (blobs.Count == 0) { + return null; } - } - - void Generate (BinaryWriter writer, List assemblies) - { - Encoding assemblyNameEncoding = Encoding.UTF8; - int assemblyNameWidth = 0; - var names = new List (); - - Action updateNameWidth = (string assemblyName) => { - int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; - if (nameBytes > assemblyNameWidth) { - assemblyNameWidth = nameBytes; - } - }; - - var index = new List (); - foreach (BlobAssemblyInfo assembly in assemblies) { - log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: assembly fs path == '{assembly.FilesystemAssemblyPath}'; assembly archive path == '{assembly.ArchiveAssemblyPath}'"); - string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - string archivePath = assembly.ArchiveAssemblyPath; - if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { - archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); - } - - if (!String.IsNullOrEmpty (assembly.Abi)) { - string abiPath = $"{assembly.Abi}/"; - if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { - archivePath = archivePath.Substring (abiPath.Length); - } - } - - if (!String.IsNullOrEmpty (archivePath)) { - assemblyName = $"{archivePath}/{assemblyName}"; - } - - log.LogMessage (MessageImportance.Low, $" => assemblyName == '{assemblyName}'; archivePath == '{archivePath}'"); - updateNameWidth (assemblyName); - names.Add (assemblyName); - assembly.NameIndex = names.Count - 1; - - BlobIndexEntry entry = WriteAssembly (writer, assembly, assemblyName); - index.Add (entry); - entry.Index = index.Count - 1; + if (indexBlob == null) { + throw new InvalidOperationException ("Index blob not found"); } - writer.Flush (); - writer.Seek (0, SeekOrigin.Begin); + var globalIndex = new List (); + var ret = new Dictionary> (StringComparer.Ordinal); + foreach (var kvp in blobs) { + string apkName = kvp.Key; + Blob blob = kvp.Value; - // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh - writer.Write (BlobMagic); // magic - writer.Write ((uint)index.Count); // entry_count - writer.Write ((uint)assemblyNameWidth); // name_width + if (!ret.ContainsKey (apkName)) { + ret.Add (apkName, new List ()); + } - var sortedIndex = new List (index); - sortedIndex.Sort ((BlobIndexEntry a, BlobIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (BlobIndexEntry entry in sortedIndex) { - writer.Write (entry.NameHash32); - writer.Write (entry.Index); + List blobPaths = ret[apkName]; + GenerateBlob (blob.Common, blobPaths); + GenerateBlob (blob.Arch, blobPaths); } - sortedIndex.Sort ((BlobIndexEntry a, BlobIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (BlobIndexEntry entry in sortedIndex) { - writer.Write (entry.NameHash64); - writer.Write (entry.Index); - } + indexBlob.WriteIndex (globalIndex); + return ret; - foreach (BlobIndexEntry entry in index) { - writer.Write (entry.DataOffset); - writer.Write (entry.DataSize); - writer.Write (entry.DebugDataOffset); - writer.Write (entry.DebugDataSize); - writer.Write (entry.ConfigDataOffset); - writer.Write (entry.ConfigDataSize); + void GenerateBlob (AssemblyBlob blob, List blobPaths) + { + blob.Generate (outputDirectory, globalIndex, blobPaths); } - - // TODO: write the names array - } - - BlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName) - { - var ret = new BlobIndexEntry (); - - return ret; } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs new file mode 100644 index 00000000000..1c4768b68a9 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs @@ -0,0 +1,42 @@ +using System; +using System.Text; + +using K4os.Hash.xxHash; + +namespace Xamarin.Android.Tasks +{ + class AssemblyBlobIndexEntry + { + public string Name { get; } + public uint BlobID { get; } + public uint Index { get; } + + // Hash values must have the same type as they are inside a union in the native code + public ulong NameHash64 { get; } + public ulong NameHash32 { get; } + + public uint DataOffset { get; set; } + public uint DataSize { get; set; } + + public uint DebugDataOffset { get; set; } + public uint DebugDataSize { get; set; } + + public uint ConfigDataOffset { get; set; } + public uint ConfigDataSize { get; set; } + + public AssemblyBlobIndexEntry (string name, uint blobID, uint index) + { + if (String.IsNullOrEmpty (name)) { + throw new ArgumentException ("must not be null or empty", nameof (name)); + } + + Name = name; + BlobID = blobID; + Index = index; + + byte[] nameBytes = Encoding.UTF8.GetBytes (name); + NameHash32 = XXH32.DigestOf (nameBytes, 0, nameBytes.Length); + NameHash64 = XXH64.DigestOf (nameBytes, 0, nameBytes.Length); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs index 7b990a2939a..7e635462c1a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs @@ -10,7 +10,6 @@ class BlobAssemblyInfo public string DebugInfoPath { get; private set; } public string ConfigPath { get; private set; } public string Abi { get; } - public int NameIndex { get; set; } public BlobAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs new file mode 100644 index 00000000000..afd4559f3f3 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class CommonAssemblyBlob : AssemblyBlob + { + readonly List assemblies; + + public CommonAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) + : base (apkName, archiveAssembliesPrefix, log) + { + assemblies = new List (); + } + + public override void Add (BlobAssemblyInfo blobAssembly) + { + if (!String.IsNullOrEmpty (blobAssembly.Abi)) { + throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); + } + + Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Common assembly {blobAssembly.FilesystemAssemblyPath}"); + assemblies.Add (blobAssembly); + AssemblyIndex.Add (new AssemblyBlobIndexEntry (GetAssemblyName (blobAssembly), ID, (uint)assemblies.Count - 1)); + } + + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + { + if (assemblies.Count == 0) { + return; + } + + Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); + } + } +} diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 4f8e74029bf..11de0ff64d3 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -114,13 +114,15 @@ struct BlobHashEntry final uint32_t hash32; }; + // ID/index into the array with pointers to assemblies from a single blob + uint32_t blob_id; + + // Index into the array with pointers to actual assembly data uint32_t index; }; struct BlobBundledAssembly final { - uint32_t name_index; - uint32_t data_offset; uint32_t data_size; @@ -141,18 +143,23 @@ struct BlobBundledAssembly final // This way the `assemblies` array in the blob can remain read only, because we write the "mapped" assembly // pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. // +// Each blob has a unique ID assigned, which is an index into an array of pointers to arrays which store +// individual assembly addresses. Only blob with ID 0 comes with the hashes32 and hashes64 arrays. This is +// done to make it possible to use a single sorted array to find assemblies insted of each blob having its +// own sorted array of hashes, which would require several binary searches instead of just one. +// // BundledAssemblyBlobHeader header; -// BlobHashEntry hashes32[header.entry_count]; -// BlobHashEntry hashes64[header.entry_count]; -// XamarinAndroidBundledAssembly assemblies[header.entry_count]; -// char assembly_names[header.entry_count][header.name_width]; +// BlobBundledAssembly assemblies[header.local_entry_count]; +// BlobHashEntry hashes32[header.global_entry_count]; // only in blob with ID 0 +// BlobHashEntry hashes64[header.global_entry_count]; // only in blob with ID 0 // [DATA] // struct BundledAssemblyBlobHeader final { uint32_t magic; - uint32_t entry_count; - uint32_t name_width; + uint32_t local_entry_count; + uint32_t global_entry_count; + uint32_t blob_id; }; struct ApplicationConfig @@ -197,7 +204,6 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API uint8_t* blob_bundled_assemblies_arch[]; -MONO_API uint8_t* blob_bundled_assemblies_common[]; +MONO_API uint8_t** blob_bundled_assemblies[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From 8044c9ff8a57a2e919efa262549b691863935052 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 14 Sep 2021 16:27:37 +0200 Subject: [PATCH 03/54] [WIP] Simplify runtime structures + packaging --- .../Tasks/BuildApk.cs | 29 ++++++++- .../Tasks/GeneratePackageManagerJava.cs | 14 ++--- .../Utilities/EnvironmentHelper.cs | 18 +----- ...pplicationConfigNativeAssemblyGenerator.cs | 60 +++++++++---------- .../Utilities/ArchAssemblyBlob.cs | 2 +- .../Utilities/AssemblyBlob.cs | 8 +-- .../Utilities/AssemblyBlobGenerator.cs | 14 ++--- .../Utilities/AssemblyBlobIndexEntry.cs | 8 ++- .../Utilities/CommonAssemblyBlob.cs | 2 +- src/monodroid/jni/application_dso_stub.cc | 9 +-- src/monodroid/jni/xamarin-app.hh | 10 +--- 11 files changed, 82 insertions(+), 92 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index ecff8d87849..f5009e086b0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -354,8 +354,31 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); + if (!useAssembliesBlob) { + return; + } + + // Currently, all the assembly blobs end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we + // partition assemblies into "feature" APKs + const string BaseApkName = "base"; + + Dictionary> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); + if (blobPaths == null) { + throw new InvalidOperationException ("Blob generator did not generate any blobs"); + } + + if (!blobPaths.TryGetValue (BaseApkName, out List baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) { + throw new InvalidOperationException ("Blob generator didn't generate the required base blobs"); + } + + string blobPrefix = $"{BaseApkName}_"; + foreach (string blobPath in baseBlobs) { + string inArchiveName = Path.GetFileName (blobPath); + + if (inArchiveName.StartsWith (blobPrefix, StringComparison.Ordinal)) { + inArchiveName = inArchiveName.Substring (blobPrefix.Length); + } + AddFileToArchiveIfNewer (apk, blobPath, AssembliesPath + inArchiveName, compressionMethod: UncompressedMethod); } void AddAssembliesFromCollection (ITaskItem[] assemblies) @@ -413,7 +436,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) } if (useAssembliesBlob) { - blobGenerator.Add (blobAssembly); + blobGenerator.Add (BaseApkName, blobAssembly); } else { count++; if (count >= ZipArchiveEx.ZipFlushFilesLimit) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index a2eae12a140..194628997a8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -292,7 +292,7 @@ void AddEnvironment () int blobArchAssemblyCount = 0; HashSet archAssemblyNames = null; - Action updateBlobCounts = (ITaskItem assembly) => { + Action updateAssemblyCount = (ITaskItem assembly) => { if (!UseAssembliesBlob) { assemblyCount++; return; @@ -300,7 +300,7 @@ void AddEnvironment () string abi = assembly.GetMetadata ("Abi"); if (String.IsNullOrEmpty (abi)) { - blobCommonAssemblyCount++; + assemblyCount++; } else { if (archAssemblyNames == null) { archAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); @@ -308,24 +308,22 @@ void AddEnvironment () string assemblyName = Path.GetFileName (assembly.ItemSpec); if (!archAssemblyNames.Contains (assemblyName)) { - blobArchAssemblyCount++; + assemblyCount++; archAssemblyNames.Add (assemblyName); } } }; if (SatelliteAssemblies != null) { - assemblyCount += SatelliteAssemblies.Length; - foreach (ITaskItem assembly in SatelliteAssemblies) { updateNameWidth (assembly); - updateBlobCounts (assembly); + updateAssemblyCount (assembly); } } foreach (var assembly in ResolvedAssemblies) { updateNameWidth (assembly); - updateBlobCounts (assembly); + updateAssemblyCount (assembly); } if (!UseAssembliesBlob) { @@ -366,8 +364,6 @@ void AddEnvironment () NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, HaveAssembliesBlob = UseAssembliesBlob, - NumberOfCommonBlobAssemblies = blobCommonAssemblyCount, - NumberOfArchBlobAssemblies = blobArchAssemblyCount, }; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 6f044c1f221..40829a973e9 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -32,12 +32,10 @@ public sealed class ApplicationConfig public uint environment_variable_count; public uint system_property_count; public uint number_of_assemblies_in_apk; - public uint number_of_common_blob_assemblies; - public uint number_of_arch_blob_assemblies; public uint bundled_assembly_name_width; public string android_package_name; }; - const uint ApplicationConfigFieldCount = 18; + const uint ApplicationConfigFieldCount = 16; static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; @@ -191,22 +189,12 @@ static ApplicationConfig ReadApplicationConfig (string envFile) ret.number_of_assemblies_in_apk = ConvertFieldToUInt32 ("number_of_assemblies_in_apk", envFile, i, field [1]); break; - case 14: // number_of_common_blob_assemblies: uint32_t / .word | .long - Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); - ret.number_of_common_blob_assemblies = ConvertFieldToUInt32 ("number_of_common_blob_assemblies", envFile, i, field [1]); - break; - - case 15: // number_of_arch_blob_assemblies: uint32_t / .word | .long - Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); - ret.number_of_arch_blob_assemblies = ConvertFieldToUInt32 ("number_of_arch_blob_assemblies", envFile, i, field [1]); - break; - - case 16: // bundled_assembly_name_width: uint32_t / .word | .long + case 14: // bundled_assembly_name_width: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile, i, field [1]); break; - case 17: // android_package_name: string / [pointer type] + case 15: // android_package_name: string / [pointer type] Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}"); pointers.Add (field [1].Trim ()); break; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index f67568ba3a0..c6890b1b134 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -26,8 +26,6 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public bool HaveRuntimeConfigBlob { get; set; } public bool HaveAssembliesBlob { get; set; } public int NumberOfAssembliesInApk { get; set; } - public int NumberOfCommonBlobAssemblies { get; set; } - public int NumberOfArchBlobAssemblies { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public PackageNamingPolicy PackageNamingPolicy { get; set; } @@ -98,12 +96,6 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "number_of_assemblies_in_apk"); size += WriteData (output, NumberOfAssembliesInApk); - WriteCommentLine (output, "number_of_common_blob_assemblies"); - size += WriteData (output, NumberOfCommonBlobAssemblies); - - WriteCommentLine (output, "number_of_arch_blob_assemblies"); - size += WriteData (output, NumberOfArchBlobAssemblies); - WriteCommentLine (output, "bundled_assembly_name_width"); size += WriteData (output, BundledAssemblyNameWidth); @@ -127,57 +119,63 @@ protected override void WriteSymbols (StreamWriter output) void WriteBlobAssemblies (StreamWriter output) { - WriteBlobs ("Blob assemblies data, architecture specific", "blob_bundled_assemblies_arch", NumberOfArchBlobAssemblies); - WriteBlobs ("Blob assemblies data, archiecture agnostic", "blob_bundled_assemblies_common", NumberOfCommonBlobAssemblies); + output.WriteLine (); - void WriteBlobs (string comment, string label, int count) - { - WriteCommentLine (output, comment); - WriteDataSection (output, label); - WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + string label = "blob_bundled_assemblies"; + WriteCommentLine (output, "Blob assemblies data"); + WriteDataSection (output, label); + WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); - uint size = 0; - for (int i = 0; i < count; i++) { + uint size = 0; + if (HaveAssembliesBlob) { + for (int i = 0; i < NumberOfAssembliesInApk; i++) { size += WriteStructure (output, packed: false, structureWriter: () => WriteBlobAssembly (output)); } - WriteStructureSize (output, label, size); } + WriteStructureSize (output, label, size); } uint WriteBlobAssembly (StreamWriter output) { - WriteCommentLine (output, "mapped assembly pointer"); - uint size = WritePointer (output); - - output.WriteLine (); - - return size; + return WritePointer (output); } void WriteBundledAssemblies (StreamWriter output) { + output.WriteLine (); + WriteCommentLine (output, $"Bundled assembly name buffers, all {BundledAssemblyNameWidth} bytes long"); WriteSection (output, ".bss.bundled_assembly_names", hasStrings: false, writable: true, nobits: true); - var name_labels = new List (); - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - string bufferLabel = GetBufferLabel (); - WriteBufferAllocation (output, bufferLabel, (uint)BundledAssemblyNameWidth); - name_labels.Add (bufferLabel); + List name_labels = null; + if (!HaveAssembliesBlob) { + name_labels = new List (); + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + string bufferLabel = GetBufferLabel (); + WriteBufferAllocation (output, bufferLabel, (uint)BundledAssemblyNameWidth); + name_labels.Add (bufferLabel); + } } + output.WriteLine (); + string label = "bundled_assemblies"; WriteCommentLine (output, "Bundled assemblies data"); WriteDataSection (output, label); WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); uint size = 0; - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - size += WriteStructure (output, packed: false, structureWriter: () => WriteBundledAssembly (output, MakeLocalLabel (name_labels[i]))); + if (!HaveAssembliesBlob) { + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteBundledAssembly (output, MakeLocalLabel (name_labels[i]))); + } } WriteStructureSize (output, label, size); + + output.WriteLine (); } + uint WriteBundledAssembly (StreamWriter output, string nameLabel) { // Order of fields and their type must correspond *exactly* to that in diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index 09eb9342a13..f9fdaf5a374 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -47,7 +47,7 @@ public override void Add (BlobAssemblyInfo blobAssembly) } seenArchAssemblyNames.Add (assemblyName); - AssemblyIndex.Add (new AssemblyBlobIndexEntry (assemblyName, ID, (uint)blobAssemblies.Count - 1)); + AssemblyIndex.Add (new AssemblyBlobIndexEntry (assemblyName, ID)); } public override void Generate (string outputDirectory, List globalIndex, List blobPaths) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 17ef8e3871c..70c7de11640 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -11,7 +11,6 @@ namespace Xamarin.Android.Tasks { abstract class AssemblyBlob { - const uint BlobMagic = 0x41424158; // 'XABA', little-endian // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh @@ -134,7 +133,6 @@ void WriteIndex (BinaryWriter writer, List globalIndex) void WriteHash (AssemblyBlobIndexEntry entry, ulong hash) { writer.Write (hash); - writer.Write (entry.BlobID); writer.Write (entry.Index); } } @@ -207,7 +205,7 @@ void Generate (BinaryWriter writer, List assemblies, List assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); globalIndex.Add (entry); localAssemblies.Add (entry); @@ -278,13 +276,13 @@ void AdjustOffsets (AssemblyBlobIndexEntry assembly, uint offsetFixup) } } - AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName, uint index) + AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName) { uint offset; uint size; (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - var ret = new AssemblyBlobIndexEntry (assemblyName, ID, index) { + var ret = new AssemblyBlobIndexEntry (assemblyName, ID) { DataOffset = offset, DataSize = size, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs index cb45d2123c4..323e2eb4dca 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs @@ -36,11 +36,6 @@ public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper blobs = new Dictionary (StringComparer.Ordinal); } - public void Add (BlobAssemblyInfo blobAssembly) - { - Add ("base", blobAssembly); - } - public void Add (string apkName, BlobAssemblyInfo blobAssembly) { if (String.IsNullOrEmpty (apkName)) { @@ -100,17 +95,16 @@ public Dictionary> Generate (string outputDirectory) ret.Add (apkName, new List ()); } - List blobPaths = ret[apkName]; - GenerateBlob (blob.Common, blobPaths); - GenerateBlob (blob.Arch, blobPaths); + GenerateBlob (blob.Common, apkName); + GenerateBlob (blob.Arch, apkName); } indexBlob.WriteIndex (globalIndex); return ret; - void GenerateBlob (AssemblyBlob blob, List blobPaths) + void GenerateBlob (AssemblyBlob blob, string apkName) { - blob.Generate (outputDirectory, globalIndex, blobPaths); + blob.Generate (outputDirectory, globalIndex, ret[apkName]); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs index 1c4768b68a9..8b15c3430e5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs @@ -7,6 +7,8 @@ namespace Xamarin.Android.Tasks { class AssemblyBlobIndexEntry { + static uint globalAssemblyIndex = 0; + public string Name { get; } public uint BlobID { get; } public uint Index { get; } @@ -24,7 +26,7 @@ class AssemblyBlobIndexEntry public uint ConfigDataOffset { get; set; } public uint ConfigDataSize { get; set; } - public AssemblyBlobIndexEntry (string name, uint blobID, uint index) + public AssemblyBlobIndexEntry (string name, uint blobID) { if (String.IsNullOrEmpty (name)) { throw new ArgumentException ("must not be null or empty", nameof (name)); @@ -32,7 +34,9 @@ public AssemblyBlobIndexEntry (string name, uint blobID, uint index) Name = name; BlobID = blobID; - Index = index; + + // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock + Index = globalAssemblyIndex++; byte[] nameBytes = Encoding.UTF8.GetBytes (name); NameHash32 = XXH32.DigestOf (nameBytes, 0, nameBytes.Length); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs index afd4559f3f3..fe6f9662acf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs @@ -26,7 +26,7 @@ public override void Add (BlobAssemblyInfo blobAssembly) Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Common assembly {blobAssembly.FilesystemAssemblyPath}"); assemblies.Add (blobAssembly); - AssemblyIndex.Add (new AssemblyBlobIndexEntry (GetAssemblyName (blobAssembly), ID, (uint)assemblies.Count - 1)); + AssemblyIndex.Add (new AssemblyBlobIndexEntry (GetAssemblyName (blobAssembly), ID)); } public override void Generate (string outputDirectory, List globalIndex, List blobPaths) diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index b3ef43b3325..81832303417 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -48,8 +48,6 @@ ApplicationConfig application_config = { .environment_variable_count = 0, .system_property_count = 0, .number_of_assemblies_in_apk = 2, - .number_of_common_blob_assemblies = 2, - .number_of_arch_blob_assemblies = 2, .android_package_name = "com.xamarin.test", }; @@ -82,12 +80,7 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { }, }; -uint8_t* blob_bundled_assemblies_arch[] = { - nullptr, - nullptr, -}; - -uint8_t* blob_bundled_assemblies_common[] = { +uint8_t* blob_bundled_assemblies[] = { nullptr, nullptr, }; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 11de0ff64d3..d32d86c2174 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -114,10 +114,8 @@ struct BlobHashEntry final uint32_t hash32; }; - // ID/index into the array with pointers to assemblies from a single blob - uint32_t blob_id; - - // Index into the array with pointers to actual assembly data + // Index into the array with pointers to assembly data. + // It **must** be unique across all the blobs from all the apks uint32_t index; }; @@ -178,8 +176,6 @@ struct ApplicationConfig uint32_t environment_variable_count; uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; - uint32_t number_of_common_blob_assemblies; - uint32_t number_of_arch_blob_assemblies; uint32_t bundled_assembly_name_width; const char *android_package_name; }; @@ -204,6 +200,6 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API uint8_t** blob_bundled_assemblies[]; +MONO_API uint8_t* blob_bundled_assemblies[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From e9b2ba2729e47ec5c9db48602f0ab3854c33ef39 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 14 Sep 2021 22:10:33 +0200 Subject: [PATCH 04/54] [WIP] Check arch-specific assembly list identity and add a manifest --- .../Utilities/ArchAssemblyBlob.cs | 44 ++++++++++++++-- .../Utilities/AssemblyBlob.cs | 50 +++++++++++++------ .../Utilities/AssemblyBlobIndexEntry.cs | 8 +-- .../Utilities/CommonAssemblyBlob.cs | 1 - 4 files changed, 77 insertions(+), 26 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index f9fdaf5a374..a1aafcaca64 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -47,7 +47,6 @@ public override void Add (BlobAssemblyInfo blobAssembly) } seenArchAssemblyNames.Add (assemblyName); - AssemblyIndex.Add (new AssemblyBlobIndexEntry (assemblyName, ID)); } public override void Generate (string outputDirectory, List globalIndex, List blobPaths) @@ -56,7 +55,36 @@ public override void Generate (string outputDirectory, List (); + foreach (var kvp in assemblies) { + string abi = kvp.Key; + List archAssemblies = kvp.Value; + + // All the architecture blobs must have assemblies in exactly the same order + archAssemblies.Sort ((BlobAssemblyInfo a, BlobAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); + if (assemblyNames.Count == 0) { + for (int i = 0; i < archAssemblies.Count; i++) { + BlobAssemblyInfo info = archAssemblies[i]; + assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); + } + continue; + } + + if (archAssemblies.Count != assemblyNames.Count) { + throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists"); + } + + for (int i = 0; i < archAssemblies.Count; i++) { + BlobAssemblyInfo info = archAssemblies[i]; + string fileName = Path.GetFileName (info.FilesystemAssemblyPath); + + if (assemblyNames[i] != fileName) { + throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); + } + } + } + + bool addToGlobalIndex = true; foreach (var kvp in assemblies) { string abi = kvp.Key; List archAssemblies = kvp.Value; @@ -65,7 +93,17 @@ public override void Generate (string outputDirectory, List bytePool = ArrayPool.Shared; static uint id = 0; + protected static uint globalAssemblyIndex = 0; + string archiveAssembliesPrefix; string indexBlobPath; @@ -37,8 +39,6 @@ abstract class AssemblyBlob public uint ID { get; } public bool IsIndexBlob => ID == 0; - public List AssemblyIndex { get; } - protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) { if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { @@ -55,8 +55,6 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg this.archiveAssembliesPrefix = archiveAssembliesPrefix; ApkName = apkName; Log = log; - - AssemblyIndex = new List (); } public abstract void Add (BlobAssemblyInfo blobAssembly); @@ -77,9 +75,11 @@ public virtual void WriteIndex (List globalIndex) } string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; + string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); + using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { - WriteIndex (writer, globalIndex); + WriteIndex (writer, indexBlobManifestPath, globalIndex); writer.Flush (); } @@ -93,7 +93,17 @@ public virtual void WriteIndex (List globalIndex) File.Move (indexBlobHeaderPath, indexBlobPath); } - void WriteIndex (BinaryWriter writer, List globalIndex) + void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) + { + using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { + using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { + WriteIndex (blobWriter, manifestWriter, globalIndex); + manifestWriter.Flush (); + } + } + } + + void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) { uint localEntryCount = 0; var localAssemblies = new List (); @@ -104,19 +114,23 @@ void WriteIndex (BinaryWriter writer, List globalIndex) localEntryCount++; localAssemblies.Add (assembly); } + + manifestWriter.WriteLine ($"{assembly.Name}"); + manifestWriter.WriteLine ($"\thash32: 0x{assembly.NameHash32:x08}\thash64: 0x{assembly.NameHash64:x016}\tglobal index: {assembly.Index:d04}\tblob ID: {assembly.BlobID:d03}"); + manifestWriter.WriteLine (); } uint globalAssemblyCount = (uint)globalIndex.Count; Log.LogMessage (MessageImportance.Low, $"Index blob, writing header (local assemblies: {localAssemblies.Count}; global assemblies: {globalIndex.Count})"); - writer.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (writer, localEntryCount, globalAssemblyCount); + blobWriter.Seek (0, SeekOrigin.Begin); + WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); // Header and two tables of the same size, each for 32 and 64-bit hashes uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); Log.LogMessage (MessageImportance.Low, "Index blob, writing assembly descriptors"); - WriteAssemblyDescriptors (writer, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); + WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); Log.LogMessage (MessageImportance.Low, $"Index blob, writing hash tables ({globalAssemblyCount * 2} entries, {offsetFixup} bytes"); var sortedIndex = new List (globalIndex); @@ -132,8 +146,8 @@ void WriteIndex (BinaryWriter writer, List globalIndex) void WriteHash (AssemblyBlobIndexEntry entry, ulong hash) { - writer.Write (hash); - writer.Write (entry.Index); + blobWriter.Write (hash); + blobWriter.Write (entry.Index); } } @@ -147,7 +161,7 @@ protected string GetAssemblyName (BlobAssemblyInfo assembly) return assemblyName; } - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths) + protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) { if (globalIndex == null) { throw new ArgumentNullException (nameof (globalIndex)); @@ -167,13 +181,13 @@ protected void Generate (string outputFilePath, List assemblie using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies, globalIndex); + Generate (writer, assemblies, globalIndex, addToGlobalIndex); writer.Flush (); } } } - void Generate (BinaryWriter writer, List assemblies, List globalIndex) + void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) { var localAssemblies = new List (); @@ -207,7 +221,9 @@ void Generate (BinaryWriter writer, List assemblies, List assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); - globalIndex.Add (entry); + if (addToGlobalIndex) { + globalIndex.Add (entry); + } localAssemblies.Add (entry); } @@ -282,7 +298,9 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse uint size; (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - var ret = new AssemblyBlobIndexEntry (assemblyName, ID) { + + // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyBlob.Generate for a special case) + var ret = new AssemblyBlobIndexEntry (assemblyName, ID, globalAssemblyIndex++) { DataOffset = offset, DataSize = size, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs index 8b15c3430e5..1c4768b68a9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs @@ -7,8 +7,6 @@ namespace Xamarin.Android.Tasks { class AssemblyBlobIndexEntry { - static uint globalAssemblyIndex = 0; - public string Name { get; } public uint BlobID { get; } public uint Index { get; } @@ -26,7 +24,7 @@ class AssemblyBlobIndexEntry public uint ConfigDataOffset { get; set; } public uint ConfigDataSize { get; set; } - public AssemblyBlobIndexEntry (string name, uint blobID) + public AssemblyBlobIndexEntry (string name, uint blobID, uint index) { if (String.IsNullOrEmpty (name)) { throw new ArgumentException ("must not be null or empty", nameof (name)); @@ -34,9 +32,7 @@ public AssemblyBlobIndexEntry (string name, uint blobID) Name = name; BlobID = blobID; - - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - Index = globalAssemblyIndex++; + Index = index; byte[] nameBytes = Encoding.UTF8.GetBytes (name); NameHash32 = XXH32.DigestOf (nameBytes, 0, nameBytes.Length); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs index fe6f9662acf..abf16479399 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs @@ -26,7 +26,6 @@ public override void Add (BlobAssemblyInfo blobAssembly) Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Common assembly {blobAssembly.FilesystemAssemblyPath}"); assemblies.Add (blobAssembly); - AssemblyIndex.Add (new AssemblyBlobIndexEntry (GetAssemblyName (blobAssembly), ID)); } public override void Generate (string outputDirectory, List globalIndex, List blobPaths) From c270f8d7ce52586bf27b347e2a7b8f6ad4df1bf4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 16 Sep 2021 17:21:26 +0200 Subject: [PATCH 05/54] [WIP] Flush some work to GH --- .../Tasks/GeneratePackageManagerJava.cs | 1 + .../Utilities/EnvironmentHelper.cs | 10 +- ...pplicationConfigNativeAssemblyGenerator.cs | 39 +++++- .../Utilities/ArchAssemblyBlob.cs | 4 +- .../Utilities/AssemblyBlob.cs | 1 + src/monodroid/jni/application_dso_stub.cc | 17 +++ src/monodroid/jni/basic-utilities.hh | 7 ++ src/monodroid/jni/embedded-assemblies-zip.cc | 111 +++++++++++++---- src/monodroid/jni/embedded-assemblies.cc | 117 +++++++++++++++--- src/monodroid/jni/embedded-assemblies.hh | 47 ++++--- src/monodroid/jni/monodroid-glue-internal.hh | 3 + src/monodroid/jni/monodroid-glue.cc | 32 ++++- src/monodroid/jni/shared-constants.hh | 10 ++ src/monodroid/jni/xamarin-app.hh | 21 +++- 14 files changed, 355 insertions(+), 65 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 194628997a8..c44daa32349 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -363,6 +363,7 @@ void AddEnvironment () HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, + NumberOfAssemblyBlobsInApk = 2, // Until feature APKs are a thing, we're going to have just two blobs in each app HaveAssembliesBlob = UseAssembliesBlob, }; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 40829a973e9..5451c7d99e5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -33,9 +33,10 @@ public sealed class ApplicationConfig public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; + public uint number_of_assembly_blobs; public string android_package_name; }; - const uint ApplicationConfigFieldCount = 16; + const uint ApplicationConfigFieldCount = 17; static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; @@ -194,7 +195,12 @@ static ApplicationConfig ReadApplicationConfig (string envFile) ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile, i, field [1]); break; - case 15: // android_package_name: string / [pointer type] + case 15: // number_of_assembly_blobs: uint32_t / .word | .long + Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); + ret.number_of_assembly_blobs = ConvertFieldToUInt32 ("number_of_assembly_blobs", envFile, i, field [1]); + break; + + case 16: // android_package_name: string / [pointer type] Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}"); pointers.Add (field [1].Trim ()); break; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index c6890b1b134..87580641520 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -26,6 +26,7 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public bool HaveRuntimeConfigBlob { get; set; } public bool HaveAssembliesBlob { get; set; } public int NumberOfAssembliesInApk { get; set; } + public int NumberOfAssemblyBlobsInApk { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public PackageNamingPolicy PackageNamingPolicy { get; set; } @@ -99,6 +100,9 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "bundled_assembly_name_width"); size += WriteData (output, BundledAssemblyNameWidth); + WriteCommentLine (output, "number_of_assembly_blobs"); + size += WriteData (output, NumberOfAssemblyBlobsInApk); + WriteCommentLine (output, "android_package_name"); size += WritePointer (output, MakeLocalLabel (stringLabel)); @@ -122,7 +126,7 @@ void WriteBlobAssemblies (StreamWriter output) output.WriteLine (); string label = "blob_bundled_assemblies"; - WriteCommentLine (output, "Blob assemblies data"); + WriteCommentLine (output, "Individual blob assembly data"); WriteDataSection (output, label); WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); @@ -133,6 +137,21 @@ void WriteBlobAssemblies (StreamWriter output) } } WriteStructureSize (output, label, size); + + output.WriteLine (); + + label = "assembly_blobs"; + WriteCommentLine (output, "Assembly blob data"); + WriteDataSection (output, label); + WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + + size = 0; + if (HaveAssembliesBlob) { + for (int i = 0; i < NumberOfAssemblyBlobsInApk; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyBlob (output)); + } + } + WriteStructureSize (output, label, size); } uint WriteBlobAssembly (StreamWriter output) @@ -140,6 +159,24 @@ uint WriteBlobAssembly (StreamWriter output) return WritePointer (output); } + uint WriteAssemblyBlob (StreamWriter output) + { + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh AssemblyBlobRuntimeData structure + WriteCommentLine (output, "data_start"); + uint size = WritePointer (output); + + WriteCommentLine (output, "assembly_count"); + size += WriteData (output, (uint)0); + + WriteCommentLine (output, "assemblies"); + size += WritePointer (output); + + output.WriteLine (); + + return size; + } + void WriteBundledAssemblies (StreamWriter output) { output.WriteLine (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index a1aafcaca64..d43e79ac8a7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -93,7 +93,9 @@ public override void Generate (string outputDirectory, List + bool ends_with (const char *str, std::array const& end) const noexcept + { + char *p = const_cast (strstr (str, end.data ())); + return p != nullptr && p [N - 1] == '\0'; + } + template bool ends_with (internal::string_base const& str, const char (&end)[N]) const noexcept { diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 3efe8c314d1..5c84319751f 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -65,9 +66,9 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector c set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); bundled_assembly_index++; + number_of_found_assemblies = bundled_assembly_index; } have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } +force_inline void +EmbeddedAssemblies::map_assembly_blob (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept +{ + if (number_of_mapped_blobs >= application_config.number_of_assembly_blobs) { + log_fatal (LOG_ASSEMBLY, "Too many assembly blobs. Expected at most %u", application_config.number_of_assembly_blobs); + abort (); + } + + md_mmap_info blob_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); + auto header = static_cast(blob_map.area); + + if (header->magic != BUNDLED_ASSEMBLIES_BLOB_MAGIC) { + log_fatal (LOG_ASSEMBLY, "Blob '%s' is not a valid Xamarin.Android assembly blob file", entry_name.get ()); + abort (); + } + + if (header->blob_id >= application_config.number_of_assembly_blobs) { + log_fatal ( + LOG_ASSEMBLY, + "Blob '%s' index %u exceeds the number of blobs known at application build time, %u", + entry_name.get (), + header->blob_id, + application_config.number_of_assembly_blobs + ); + abort (); + } + + AssemblyBlobRuntimeData &rd = assembly_blobs[header->blob_id]; + if (rd.data_start != nullptr) { + log_fatal (LOG_ASSEMBLY, "Blob '%s' has a duplicate ID (%u)", entry_name.get (), header->blob_id); + abort (); + } + + constexpr size_t header_size = sizeof(BundledAssemblyBlobHeader); + + rd.data_start = static_cast(blob_map.area); + rd.assembly_count = header->local_entry_count; + rd.assemblies = reinterpret_cast(rd.data_start + header_size); + + number_of_found_assemblies += rd.assembly_count; + + if (header->blob_id == 0) { + log_warn (LOG_ASSEMBLY, "Found index blob ('%s')", entry_name.get ()); + + index_blob_header = header; + + constexpr size_t bundled_assembly_size = sizeof(BlobBundledAssembly); + constexpr size_t hash_entry_size = sizeof(BlobHashEntry); + + size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); + if constexpr (std::is_same_v) { + blob_assembly_hashes = static_cast(blob_map.area) + bytes_before_hashes + (hash_entry_size * header->global_entry_count); + log_warn (LOG_ASSEMBLY, "64-bit hashes at %p", blob_assembly_hashes); + } else { + blob_assembly_hashes = static_cast(blob_map.area) + bytes_before_hashes; + log_warn (LOG_ASSEMBLY, "32-bit hashes at %p", blob_assembly_hashes); + } + } + + number_of_mapped_blobs++; +} + force_inline void EmbeddedAssemblies::zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { + if (all_blobs_found ()) { + log_warn (LOG_ASSEMBLY, "All blobs already found, further scanning not needed"); + } + dynamic_local_string entry_name; - bool common_blob_found = false; - bool arch_blob_found = false; + bool common_assembly_blob_found = false; + bool arch_assembly_blob_found = false; + log_warn (LOG_ASSEMBLY, "Looking for blobs in APK (common: '%s'; arch-specific: '%s')", bundled_assemblies_common_blob_name.data (), bundled_assemblies_arch_blob_name.data ()); for (size_t i = 0; i < num_entries; i++) { + if (all_blobs_found ()) { + need_to_scan_more_apks = false; + log_warn (LOG_ASSEMBLY, "All blobs found, done scanning (at entry %u)", i); + break; + } + bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); if (!interesting_entry) { continue; } - if (!common_blob_found && utils.ends_with (entry_name, bundled_assemblies_common_blob_name)) { - common_blob_found = true; - // TODO: mmap - } - - if (!arch_blob_found && utils.ends_with (entry_name, bundled_assemblies_arch_blob_name)) { - arch_blob_found = true; - // TODO: mmap + if (!common_assembly_blob_found && utils.ends_with (entry_name, bundled_assemblies_common_blob_name)) { + common_assembly_blob_found = true; + map_assembly_blob (entry_name, state); } - if (common_blob_found && arch_blob_found) { -#if NET6 - if ((application_config.have_runtime_config_blob && state.runtime_config_blob_found) || !application_config.have_runtime_config_blob) -#endif - { - break; - } + if (!arch_assembly_blob_found && utils.ends_with (entry_name, bundled_assemblies_arch_blob_name)) { + arch_assembly_blob_found = true; + map_assembly_blob (entry_name, state); } } } @@ -230,9 +296,6 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus .prefix = get_assemblies_prefix (), .prefix_len = get_assemblies_prefix_length (), .buf_offset = 0, -#if defined (NET6) - .runtime_config_blob_found = false, -#endif // def NET6 }; ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); @@ -242,9 +305,9 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } if (application_config.have_assemblies_blob) { - zip_load_blob_assembly_entries (buf, cd_size, state); + zip_load_blob_assembly_entries (buf, cd_entries, state); } else { - zip_load_individual_assembly_entries (buf, cd_size, should_register, state); + zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index f1c21207f33..0e476d2023d 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -293,26 +294,17 @@ EmbeddedAssemblies::load_bundled_assembly ( return a; } -MonoAssembly* -EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only) -{ - const char *culture = mono_assembly_name_get_culture (aname); - const char *asmname = mono_assembly_name_get_name (aname); - - constexpr char path_separator[] = "/"; - dynamic_local_string name; - if (culture != nullptr && *culture != '\0') { - name.append_c (culture); - name.append (path_separator); - } - name.append_c (asmname); +constexpr char path_separator[] = "/"; +force_inline MonoAssembly* +EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept +{ constexpr char dll_extension[] = ".dll"; if (!utils.ends_with (name, dll_extension)) { name.append (dll_extension); } - log_debug (LOG_ASSEMBLY, "open_from_bundles: looking for bundled name: '%s'", name.get ()); + log_debug (LOG_ASSEMBLY, "individual_assemblies_open_from_bundles: looking for bundled name: '%s'", name.get ()); dynamic_local_string abi_name; abi_name @@ -338,10 +330,99 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function 0) { + ret = entries + (entry_count / 2); + if constexpr (std::is_same_vhash64; + } else { + entry_hash = ret->hash32; + } + + std::strong_ordering result = hash <=> entry_hash; + + if (result < 0) { + entry_count /= 2; + } else if (result > 0) { + entries = ret + 1; + entry_count -= entry_count / 2 + 1; + } else { + return ret; + } + } + + return nullptr; +} + +force_inline MonoAssembly* +EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept +{ + hash_t name_hash = xxhash::hash (name.get (), name.length ()); + log_warn (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); + + const BlobHashEntry *bae = find_blob_assembly_entry (name_hash, blob_assembly_hashes, application_config.number_of_assemblies_in_apk); + if (bae == nullptr) { + return nullptr; + } + + log_debug (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: found index entry (blob id: %u; index: %u)", bae->blob_id, bae->index); + if (bae->index >= application_config.number_of_assemblies_in_apk) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", bae->index, application_config.number_of_assemblies_in_apk - 1); + abort (); + } + + uint8_t *assembly_data = blob_bundled_assemblies[bae->index]; + if (assembly_data != nullptr) { + log_warn (LOG_ASSEMBLY, "Assembly already mapped"); + } else { + log_warn (LOG_ASSEMBLY, "Assembly not mapped yet"); + if (bae->blob_id >= application_config.number_of_assembly_blobs) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly blob ID %u, exceeds the maximum of %u", bae->blob_id, application_config.number_of_assembly_blobs - 1); + abort (); + } + + AssemblyBlobRuntimeData &rd = assembly_blobs[bae->blob_id]; + // TODO: add index into blob's **local** array of assemblies to BlobBundledAssembly + } + + return nullptr; +} + +MonoAssembly* +EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only) +{ + const char *culture = mono_assembly_name_get_culture (aname); + const char *asmname = mono_assembly_name_get_name (aname); + + dynamic_local_string name; + if (culture != nullptr && *culture != '\0') { + name.append_c (culture); + name.append (path_separator); + } + name.append_c (asmname); + + MonoAssembly *a; + if (application_config.have_assemblies_blob) { + a = blob_assemblies_open_from_bundles (name, loader, ref_only); + } else { + a = individual_assemblies_open_from_bundles (name, loader, ref_only); + } + + if (a == nullptr) { + log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); + } + + return a; +} + #if defined (NET6) MonoAssembly* EmbeddedAssemblies::open_from_bundles (MonoAssemblyLoadContextGCHandle alc_gchandle, MonoAssemblyName *aname, [[maybe_unused]] char **assemblies_path, [[maybe_unused]] void *user_data, MonoError *error) @@ -1067,11 +1148,11 @@ EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) size_t EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_register should_register) { - size_t prev = bundled_assembly_index; + size_t prev = number_of_found_assemblies; gather_bundled_assemblies_from_apk (apk_file, should_register); - log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, bundled_assembly_index - prev); + log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assemblies - prev); - return bundled_assembly_index; + return number_of_found_assemblies; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 3bee6df7f41..8e0dd953729 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -29,6 +29,8 @@ #include "strings.hh" #include "xamarin-app.hh" #include "cpp-util.hh" +#include "shared-constants.hh" +#include "xxhash.hh" struct TypeMapHeader; @@ -66,9 +68,6 @@ namespace xamarin::android::internal { uint32_t local_header_offset; uint32_t data_offset; uint32_t file_size; -#if defined (NET6) - bool runtime_config_blob_found; -#endif }; private: @@ -82,17 +81,8 @@ namespace xamarin::android::internal { static constexpr char bundled_assemblies_blob_prefix[] = "assemblies"; static constexpr char bundled_assemblies_blob_ext[] = ".blob"; -#if __arm__ - static constexpr char bundled_assemblies_blob_arch[] = "armeabi-v7a"; -#elif __aarch64__ - static constexpr char bundled_assemblies_blob_arch[] = "arm64-v8a"; -#elif __x86_64__ - static constexpr char bundled_assemblies_blob_arch[] = "x86_64"; -#elif __i386__ - static constexpr char bundled_assemblies_blob_arch[] = "x86"; -#endif static constexpr auto bundled_assemblies_common_blob_name = concat_const ("/", bundled_assemblies_blob_prefix, bundled_assemblies_blob_ext); - static constexpr auto bundled_assemblies_arch_blob_name = concat_const ("/", bundled_assemblies_blob_prefix, "_", bundled_assemblies_blob_arch, bundled_assemblies_blob_ext); + static constexpr auto bundled_assemblies_arch_blob_name = concat_const ("/", bundled_assemblies_blob_prefix, "_", SharedConstants::android_abi, bundled_assemblies_blob_ext); #if defined (DEBUG) || !defined (ANDROID) @@ -144,12 +134,17 @@ namespace xamarin::android::internal { size = static_cast(runtime_config_blob_mmap.size); } - bool have_runtime_config_blob () const + bool have_runtime_config_blob () const noexcept { return application_config.have_runtime_config_blob && runtime_config_blob_mmap.area != nullptr; } #endif + bool keep_scanning () const noexcept + { + return need_to_scan_more_apks; + } + private: const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid); MonoReflectionType* typemap_java_to_managed (const char *java_type_name); @@ -159,6 +154,8 @@ namespace xamarin::android::internal { MonoAssembly* open_from_bundles (MonoAssemblyName* aname, MonoAssemblyLoadContextGCHandle alc_gchandle, MonoError *error); #endif // def NET6 MonoAssembly* open_from_bundles (MonoAssemblyName* aname, bool ref_only); + MonoAssembly* individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; + MonoAssembly* blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; MonoAssembly* open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only); template @@ -240,6 +237,16 @@ namespace xamarin::android::internal { return assemblies_prefix_override != nullptr ? static_cast(strlen (assemblies_prefix_override)) : sizeof(assemblies_prefix) - 1; } + bool all_blobs_found () const noexcept + { + return + number_of_mapped_blobs == application_config.number_of_assembly_blobs +#if defined (NET6) + && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) +#endif // NET6 + ; + } + bool is_debug_file (dynamic_local_string const& name) noexcept; template @@ -256,6 +263,8 @@ namespace xamarin::android::internal { void set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; + void map_assembly_blob (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; + const BlobHashEntry* find_blob_assembly_entry (hash_t hash, const BlobHashEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -264,8 +273,8 @@ namespace xamarin::android::internal { bool register_debug_symbols; bool have_and_want_debug_symbols; size_t bundled_assembly_index = 0; - size_t bundled_assembly_blob_common_count = 0; - size_t bundled_assembly_blob_arch_count = 0; + size_t number_of_found_assemblies = 0; + #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; TypeMappingInfo *managed_to_java_maps; @@ -275,7 +284,13 @@ namespace xamarin::android::internal { const char *assemblies_prefix_override = nullptr; #if defined (NET6) md_mmap_info runtime_config_blob_mmap{}; + bool runtime_config_blob_found = false; #endif // def NET6 + uint32_t number_of_mapped_blobs = 0; + bool need_to_scan_more_apks = true; + + BundledAssemblyBlobHeader *index_blob_header = nullptr; + BlobHashEntry *blob_assembly_hashes; }; } diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index caf8d3b1cd5..f95bd804e63 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -7,6 +7,7 @@ #include "android-system.hh" #include "osbridge.hh" #include "timing.hh" +#include "cpp-util.hh" #include "xxhash.hh" #include @@ -126,6 +127,8 @@ namespace xamarin::android::internal }; private: + static constexpr auto split_config_abi_apk_name = concat_const ("/split_config.", SharedConstants::android_abi, ".apk"); + static constexpr char base_apk_name[] = "/base.apk"; static constexpr size_t SMALL_STRING_PARSE_BUFFER_LEN = 50; static constexpr bool is_running_on_desktop = #if ANDROID diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index d2eed495b74..09c24198fad 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -408,14 +408,42 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; - for (int64_t i = apk_count - 1; i >= 0; --i) { + // bool got_base_apk = false; + // bool got_split_config_abi_apk = false; + // bool scan_apk; + + for (int64_t i = 0; i < apk_count; i++) { jstring_wrapper &apk_file = runtimeApks [static_cast(i)]; + // + // Code below can be enabled only when we can reliably detect that we're running with split APKs. + // The best check appears to involve https://developer.android.com/reference/android/content/pm/ApplicationInfo#splitSourceDirs + // However, the above works only if we can compile our Java code against at least API21, but we compile against + // API19 atm + // + + // scan_apk = false; + + // if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { + // got_base_apk = scan_apk = true; + // } else if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), split_config_abi_apk_name)) { + // got_split_config_abi_apk = scan_apk = true; + // } + + // if (!scan_apk) { + // continue; + // } size_t cur_num_assemblies = embeddedAssemblies.register_from (apk_file.get_cstr ()); *out_user_assemblies_count += (cur_num_assemblies - prev_num_assemblies); prev_num_assemblies = cur_num_assemblies; + + if (!embeddedAssemblies.keep_scanning ()) { + break; + } } + + // TODO: ensure that we have all we need (blob indexes etc) } #if defined (DEBUG) && !defined (WINDOWS) @@ -888,7 +916,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks log_fatal (LOG_DEFAULT, "No assemblies found in '%s' or '%s'. Assuming this is part of Fast Deployment. Exiting...", androidSystem.get_override_dir (0), (AndroidSystem::MAX_OVERRIDES > 1 && androidSystem.get_override_dir (1) != nullptr) ? androidSystem.get_override_dir (1) : ""); - exit (FATAL_EXIT_NO_ASSEMBLIES); + abort (); } MonoDomain *domain; diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 2ce65384484..2caeb42c8d6 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -39,6 +39,16 @@ namespace xamarin::android::internal static constexpr char MONO_SGEN_SO[] = "monosgen-2.0"; static constexpr char MONO_SGEN_ARCH_SO[] = "monosgen-" __BITNESS__ "-2.0"; #endif + +#if __arm__ + static constexpr char android_abi[] = "armeabi_v7a"; +#elif __aarch64__ + static constexpr char android_abi[] = "arm64_v8a"; +#elif __x86_64__ + static constexpr char android_abi[] = "x86_64"; +#elif __i386__ + static constexpr char android_abi[] = "x86"; +#endif }; } #endif // __SHARED_CONSTANTS_HH diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index d32d86c2174..f0b39f272a3 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -10,7 +10,7 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian -static constexpr uint32_t BUNDLED_ASSEMBLIES_INDEX_MAGIC = 0x41424158; // 'XABA', little-endian +static constexpr uint32_t BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -117,6 +117,9 @@ struct BlobHashEntry final // Index into the array with pointers to assembly data. // It **must** be unique across all the blobs from all the apks uint32_t index; + + // Index into the array with blob mmap addresses + uint32_t blob_id; }; struct BlobBundledAssembly final @@ -160,6 +163,20 @@ struct BundledAssemblyBlobHeader final uint32_t blob_id; }; +struct AssemblyBlobRuntimeData final +{ + uint8_t *data_start; + uint32_t assembly_count; + BlobBundledAssembly *assemblies; +}; + +struct BlobAssemblyRuntimeData final +{ + uint8_t *image_data; + uint8_t *debug_info_data; + uint8_t *config_data; +}; + struct ApplicationConfig { bool uses_mono_llvm; @@ -177,6 +194,7 @@ struct ApplicationConfig uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; + uint32_t number_of_assembly_blobs; const char *android_package_name; }; @@ -201,5 +219,6 @@ MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; MONO_API uint8_t* blob_bundled_assemblies[]; +MONO_API AssemblyBlobRuntimeData assembly_blobs[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From 94f422e4a877ae868ee98b24b8cc3a26f85107aa Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 16 Sep 2021 17:38:21 +0200 Subject: [PATCH 06/54] [WIP] Doesn't build, saving work --- src/monodroid/jni/xamarin-app.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index f0b39f272a3..5e99456e0fc 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -218,7 +218,7 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API uint8_t* blob_bundled_assemblies[]; +MONO_API BlobAssemblyRuntimeData blob_bundled_assemblies[]; MONO_API AssemblyBlobRuntimeData assembly_blobs[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From 4cc4efcbbe054ffe2f0efcdffbe1a70bee3430ff Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 16 Sep 2021 22:07:16 +0200 Subject: [PATCH 07/54] [WIP] assembly mapping progress --- ...pplicationConfigNativeAssemblyGenerator.cs | 18 +++- .../Utilities/AssemblyBlob.cs | 13 +-- .../Utilities/AssemblyBlobIndexEntry.cs | 8 +- src/monodroid/jni/application_dso_stub.cc | 19 ++-- src/monodroid/jni/embedded-assemblies-zip.cc | 31 +++++-- src/monodroid/jni/embedded-assemblies.cc | 89 +++++++++++++------ src/monodroid/jni/embedded-assemblies.hh | 4 +- src/monodroid/jni/xamarin-app.hh | 71 ++++++++------- 8 files changed, 175 insertions(+), 78 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 87580641520..d90f40da386 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -156,7 +156,23 @@ void WriteBlobAssemblies (StreamWriter output) uint WriteBlobAssembly (StreamWriter output) { - return WritePointer (output); + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh BlobAssemblyRuntimeData structure + WriteCommentLine (output, "image_data"); + uint size = WritePointer (output); + + WriteCommentLine (output, "debug_info_data"); + size += WritePointer (output); + + WriteCommentLine (output, "config_data"); + size += WritePointer (output); + + WriteCommentLine (output, "descriptor"); + size += WritePointer (output); + + output.WriteLine (); + + return size; } uint WriteAssemblyBlob (StreamWriter output) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 2f07e4fd1be..6ef2f2c6d25 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -17,7 +17,7 @@ abstract class AssemblyBlob const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (2 * sizeof (uint)); + const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh const uint BlobHeaderNativeStructSize = sizeof (uint) * 4; @@ -116,7 +116,7 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List assemblies, List assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); if (addToGlobalIndex) { globalIndex.Add (entry); @@ -293,7 +294,7 @@ void AdjustOffsets (AssemblyBlobIndexEntry assembly, uint offsetFixup) } } - AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName) + AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName, uint localBlobIndex) { uint offset; uint size; @@ -301,7 +302,7 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyBlob.Generate for a special case) - var ret = new AssemblyBlobIndexEntry (assemblyName, ID, globalAssemblyIndex++) { + var ret = new AssemblyBlobIndexEntry (assemblyName, ID, globalAssemblyIndex++, localBlobIndex) { DataOffset = offset, DataSize = size, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs index 1c4768b68a9..41d5c3bd3d6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs @@ -9,7 +9,8 @@ class AssemblyBlobIndexEntry { public string Name { get; } public uint BlobID { get; } - public uint Index { get; } + public uint MappingIndex { get; } + public uint LocalBlobIndex { get; } // Hash values must have the same type as they are inside a union in the native code public ulong NameHash64 { get; } @@ -24,7 +25,7 @@ class AssemblyBlobIndexEntry public uint ConfigDataOffset { get; set; } public uint ConfigDataSize { get; set; } - public AssemblyBlobIndexEntry (string name, uint blobID, uint index) + public AssemblyBlobIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) { if (String.IsNullOrEmpty (name)) { throw new ArgumentException ("must not be null or empty", nameof (name)); @@ -32,7 +33,8 @@ public AssemblyBlobIndexEntry (string name, uint blobID, uint index) Name = name; BlobID = blobID; - Index = index; + MappingIndex = mappingIndex; + LocalBlobIndex = localBlobIndex; byte[] nameBytes = Encoding.UTF8.GetBytes (name); NameHash32 = XXH32.DigestOf (nameBytes, 0, nameBytes.Length); diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index fcef25f0f41..5b2e931a4f2 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -83,20 +83,29 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { }, }; -uint8_t* blob_bundled_assemblies[] = { - nullptr, - nullptr, +BlobAssemblyRuntimeData blob_bundled_assemblies[] = { + { + .image_data = nullptr, + .debug_info_data = nullptr, + .config_data = nullptr, + }, + + { + .image_data = nullptr, + .debug_info_data = nullptr, + .config_data = nullptr, + } }; AssemblyBlobRuntimeData assembly_blobs[] = { { - .image_data_start = nullptr, + .data_start = nullptr, .assembly_count = 0, .assemblies = nullptr, }, { - .image_data_start = nullptr, + .data_start = nullptr, .assembly_count = 0, .assemblies = nullptr, }, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 5c84319751f..5d552436d5c 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -211,21 +211,40 @@ EmbeddedAssemblies::map_assembly_blob (dynamic_local_string c number_of_found_assemblies += rd.assembly_count; if (header->blob_id == 0) { - log_warn (LOG_ASSEMBLY, "Found index blob ('%s')", entry_name.get ()); - - index_blob_header = header; - constexpr size_t bundled_assembly_size = sizeof(BlobBundledAssembly); constexpr size_t hash_entry_size = sizeof(BlobHashEntry); + log_warn (LOG_ASSEMBLY, "header_size == %u; bundled_assembly_size == %u; hash_entry_size == %u", header_size, bundled_assembly_size, hash_entry_size); + log_warn (LOG_ASSEMBLY, "local_entry_count == %u; global_entry_count == %u", header->local_entry_count, header->global_entry_count); + log_warn (LOG_ASSEMBLY, "Found index blob ('%s');", entry_name.get ()); + log_warn (LOG_ASSEMBLY, "Local assemblies array starts at offset %zu (%p)", reinterpret_cast(rd.assemblies) - rd.data_start, rd.assemblies); + log_warn (LOG_ASSEMBLY, "Local assemblies array is %zu bytes long", bundled_assembly_size * rd.assembly_count); + log_warn (LOG_ASSEMBLY, "Global hashes start at offset %zu", (reinterpret_cast(rd.assemblies) + (bundled_assembly_size * rd.assembly_count)) - rd.data_start); + + index_blob_header = header; + size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); if constexpr (std::is_same_v) { - blob_assembly_hashes = static_cast(blob_map.area) + bytes_before_hashes + (hash_entry_size * header->global_entry_count); + blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); log_warn (LOG_ASSEMBLY, "64-bit hashes at %p", blob_assembly_hashes); } else { - blob_assembly_hashes = static_cast(blob_map.area) + bytes_before_hashes; + blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); log_warn (LOG_ASSEMBLY, "32-bit hashes at %p", blob_assembly_hashes); } + + log_warn (LOG_ASSEMBLY, "Global hash index contents:"); + hash_t entry_hash; + for (size_t i = 0; i < header->global_entry_count; i++) { + BlobHashEntry &he = blob_assembly_hashes[i]; + + if constexpr (std::is_same_v) { + entry_hash = he.hash64; + } else { + entry_hash = he.hash32; + } + + log_warn (LOG_ASSEMBLY, " [%u]: hash == 0x%zx; mapping index == %u; local blob index == %u; blob ID == %u", i, entry_hash, he.mapping_index, he.local_blob_index, he.blob_id); + } } number_of_mapped_blobs++; diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 0e476d2023d..a0d38e013d5 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -67,10 +67,10 @@ void EmbeddedAssemblies::set_assemblies_prefix (const char *prefix) } force_inline void -EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) +EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { #if defined (ANDROID) && defined (HAVE_LZ4) && defined (RELEASE) - auto header = reinterpret_cast(e.data); + auto header = reinterpret_cast(data); if (header->magic == COMPRESSED_DATA_MAGIC) { if (XA_UNLIKELY (compressed_assemblies.descriptors == nullptr)) { log_fatal (LOG_ASSEMBLY, "Compressed assembly found but no descriptor defined"); @@ -82,7 +82,7 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u } CompressedAssemblyDescriptor &cad = compressed_assemblies.descriptors[header->descriptor_index]; - assembly_data_size = e.data_size - sizeof(CompressedAssemblyHeader); + assembly_data_size = data_size - sizeof(CompressedAssemblyHeader); if (!cad.loaded) { if (XA_UNLIKELY (cad.data == nullptr)) { log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor at %u: no data", header->descriptor_index); @@ -97,29 +97,29 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u if (header->uncompressed_length != cad.uncompressed_file_size) { if (header->uncompressed_length > cad.uncompressed_file_size) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", e.name, cad.uncompressed_file_size, header->uncompressed_length); + log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", name, cad.uncompressed_file_size, header->uncompressed_length); exit (FATAL_EXIT_MISSING_ASSEMBLY); } else { - log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", e.name); + log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", name); } cad.uncompressed_file_size = header->uncompressed_length; } - const char *data_start = reinterpret_cast(e.data + sizeof(CompressedAssemblyHeader)); + const char *data_start = reinterpret_cast(data + sizeof(CompressedAssemblyHeader)); int ret = LZ4_decompress_safe (data_start, reinterpret_cast(cad.data), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); if (XA_UNLIKELY (log_timing)) { decompress_time.mark_end (); - TIMING_LOG_INFO (decompress_time, "%s LZ4 decompression time", e.name); + TIMING_LOG_INFO (decompress_time, "%s LZ4 decompression time", name); } if (ret < 0) { - log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", e.name, ret); + log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", name, ret); exit (FATAL_EXIT_MISSING_ASSEMBLY); } if (static_cast(ret) != cad.uncompressed_file_size) { - log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", e.name, cad.uncompressed_file_size, static_cast(ret)); + log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", name, cad.uncompressed_file_size, static_cast(ret)); exit (FATAL_EXIT_MISSING_ASSEMBLY); } cad.loaded = true; @@ -129,11 +129,23 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u } else #endif { - assembly_data = e.data; - assembly_data_size = e.data_size; + assembly_data = data; + assembly_data_size = data_size; } } +force_inline void +EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +{ + get_assembly_data (e.data, e.data_size, e.name, assembly_data, assembly_data_size); +} + +force_inline void +EmbeddedAssemblies::get_assembly_data (BlobAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +{ + get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); +} + #if defined (NET6) MonoAssembly* EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, MonoAssemblyLoadContextGCHandle alc_gchandle, [[maybe_unused]] MonoError *error) @@ -341,7 +353,7 @@ EmbeddedAssemblies::find_blob_assembly_entry (hash_t hash, const BlobHashEntry * while (entry_count > 0) { ret = entries + (entry_count / 2); - if constexpr (std::is_same_v) { entry_hash = ret->hash64; } else { entry_hash = ret->hash32; @@ -368,31 +380,58 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_stringblob_id, bae->index); - if (bae->index >= application_config.number_of_assemblies_in_apk) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", bae->index, application_config.number_of_assemblies_in_apk - 1); + log_debug (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: found index entry (blob id: %u; index: %u)", hash_entry->blob_id, hash_entry->mapping_index); + if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); abort (); } - uint8_t *assembly_data = blob_bundled_assemblies[bae->index]; - if (assembly_data != nullptr) { - log_warn (LOG_ASSEMBLY, "Assembly already mapped"); - } else { + BlobAssemblyRuntimeData &assembly_data = blob_bundled_assemblies[hash_entry->mapping_index]; + if (assembly_data.image_data == nullptr) { log_warn (LOG_ASSEMBLY, "Assembly not mapped yet"); - if (bae->blob_id >= application_config.number_of_assembly_blobs) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly blob ID %u, exceeds the maximum of %u", bae->blob_id, application_config.number_of_assembly_blobs - 1); + if (hash_entry->blob_id >= application_config.number_of_assembly_blobs) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly blob ID %u, exceeds the maximum of %u", hash_entry->blob_id, application_config.number_of_assembly_blobs - 1); abort (); } - AssemblyBlobRuntimeData &rd = assembly_blobs[bae->blob_id]; - // TODO: add index into blob's **local** array of assemblies to BlobBundledAssembly + AssemblyBlobRuntimeData &rd = assembly_blobs[hash_entry->blob_id]; + if (hash_entry->local_blob_index >= rd.assembly_count) { + log_fatal (LOG_ASSEMBLY, "Invalid index %u into local blob descriptor array", hash_entry->local_blob_index); + } + + BlobBundledAssembly *bba = &rd.assemblies[hash_entry->local_blob_index]; + assembly_data.image_data = rd.data_start + bba->data_offset; + assembly_data.descriptor = bba; + + if (bba->debug_data_offset != 0) { + assembly_data.debug_info_data = rd.data_start + bba->debug_data_offset; + } +#if !defined (NET6) + if (bba->config_data_size != 0) { + assembly_data.debug_info_data = rd.data_start + bba->config_data_offset; + } +#endif // NET6 + } else { + log_warn (LOG_ASSEMBLY, "Assembly already mapped"); } + log_warn ( + LOG_ASSEMBLY, + "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u", + assembly_data.image_data, + assembly_data.debug_info_data, + assembly_data.config_data, + assembly_data.descriptor, + assembly_data.descriptor->data_size, + assembly_data.descriptor->debug_data_size, + assembly_data.descriptor->config_data_size + ); return nullptr; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 8e0dd953729..c30f90c793d 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -187,7 +187,9 @@ namespace xamarin::android::internal { #else // def NET6 static MonoAssembly* open_from_bundles_refonly (MonoAssemblyName *aname, char **assemblies_path, void *user_data); #endif // ndef NET6 - static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size); + static void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; + static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; + static void get_assembly_data (BlobAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 5e99456e0fc..7981ba380db 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -107,33 +107,6 @@ struct XamarinAndroidBundledAssembly final char *name; }; -struct BlobHashEntry final -{ - union { - uint64_t hash64; - uint32_t hash32; - }; - - // Index into the array with pointers to assembly data. - // It **must** be unique across all the blobs from all the apks - uint32_t index; - - // Index into the array with blob mmap addresses - uint32_t blob_id; -}; - -struct BlobBundledAssembly final -{ - uint32_t data_offset; - uint32_t data_size; - - uint32_t debug_data_offset; - uint32_t debug_data_size; - - uint32_t config_data_offset; - uint32_t config_data_size; -}; - // // Blob format // @@ -155,7 +128,12 @@ struct BlobBundledAssembly final // BlobHashEntry hashes64[header.global_entry_count]; // only in blob with ID 0 // [DATA] // -struct BundledAssemblyBlobHeader final + +// +// The structures which are found in the blob files must be packed, to avoid problems when calculating offsets (runtime +// size of a structure can be different than the real data size) +// +struct [[gnu::packed]] BundledAssemblyBlobHeader final { uint32_t magic; uint32_t local_entry_count; @@ -163,6 +141,36 @@ struct BundledAssemblyBlobHeader final uint32_t blob_id; }; +struct [[gnu::packed]] BlobHashEntry final +{ + union { + uint64_t hash64; + uint32_t hash32; + }; + + // Index into the array with pointers to assembly data. + // It **must** be unique across all the blobs from all the apks + uint32_t mapping_index; + + // Index into the array with assembly descriptors inside a blob + uint32_t local_blob_index; + + // Index into the array with blob mmap addresses + uint32_t blob_id; +}; + +struct [[gnu::packed]] BlobBundledAssembly final +{ + uint32_t data_offset; + uint32_t data_size; + + uint32_t debug_data_offset; + uint32_t debug_data_size; + + uint32_t config_data_offset; + uint32_t config_data_size; +}; + struct AssemblyBlobRuntimeData final { uint8_t *data_start; @@ -172,9 +180,10 @@ struct AssemblyBlobRuntimeData final struct BlobAssemblyRuntimeData final { - uint8_t *image_data; - uint8_t *debug_info_data; - uint8_t *config_data; + uint8_t *image_data; + uint8_t *debug_info_data; + uint8_t *config_data; + BlobBundledAssembly *descriptor; }; struct ApplicationConfig From 83df401b39962572c85b9954bafe2233dfb47b99 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 17 Sep 2021 16:31:47 +0200 Subject: [PATCH 08/54] [WIP] It's alive! --- .../Utilities/AssemblyBlob.cs | 1 + src/monodroid/jni/embedded-assemblies-zip.cc | 46 +++++----- src/monodroid/jni/embedded-assemblies.cc | 90 +++++++++++++------ src/monodroid/jni/shared-constants.hh | 2 + 4 files changed, 87 insertions(+), 52 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 6ef2f2c6d25..69e83cc97fb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -117,6 +117,7 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List c } #endif // ndef NET6 - if (!utils.ends_with (entry_name, ".dll")) + if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) continue; #if defined (DEBUG) @@ -214,58 +214,58 @@ EmbeddedAssemblies::map_assembly_blob (dynamic_local_string c constexpr size_t bundled_assembly_size = sizeof(BlobBundledAssembly); constexpr size_t hash_entry_size = sizeof(BlobHashEntry); - log_warn (LOG_ASSEMBLY, "header_size == %u; bundled_assembly_size == %u; hash_entry_size == %u", header_size, bundled_assembly_size, hash_entry_size); - log_warn (LOG_ASSEMBLY, "local_entry_count == %u; global_entry_count == %u", header->local_entry_count, header->global_entry_count); - log_warn (LOG_ASSEMBLY, "Found index blob ('%s');", entry_name.get ()); - log_warn (LOG_ASSEMBLY, "Local assemblies array starts at offset %zu (%p)", reinterpret_cast(rd.assemblies) - rd.data_start, rd.assemblies); - log_warn (LOG_ASSEMBLY, "Local assemblies array is %zu bytes long", bundled_assembly_size * rd.assembly_count); - log_warn (LOG_ASSEMBLY, "Global hashes start at offset %zu", (reinterpret_cast(rd.assemblies) + (bundled_assembly_size * rd.assembly_count)) - rd.data_start); + // log_warn (LOG_ASSEMBLY, "header_size == %u; bundled_assembly_size == %u; hash_entry_size == %u", header_size, bundled_assembly_size, hash_entry_size); + // log_warn (LOG_ASSEMBLY, "local_entry_count == %u; global_entry_count == %u", header->local_entry_count, header->global_entry_count); + // log_warn (LOG_ASSEMBLY, "Found index blob ('%s');", entry_name.get ()); + // log_warn (LOG_ASSEMBLY, "Local assemblies array starts at offset %zu (%p)", reinterpret_cast(rd.assemblies) - rd.data_start, rd.assemblies); + // log_warn (LOG_ASSEMBLY, "Local assemblies array is %zu bytes long", bundled_assembly_size * rd.assembly_count); + // log_warn (LOG_ASSEMBLY, "Global hashes start at offset %zu", (reinterpret_cast(rd.assemblies) + (bundled_assembly_size * rd.assembly_count)) - rd.data_start); index_blob_header = header; size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); if constexpr (std::is_same_v) { blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - log_warn (LOG_ASSEMBLY, "64-bit hashes at %p", blob_assembly_hashes); + // log_warn (LOG_ASSEMBLY, "64-bit hashes at %p", blob_assembly_hashes); } else { blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - log_warn (LOG_ASSEMBLY, "32-bit hashes at %p", blob_assembly_hashes); + // log_warn (LOG_ASSEMBLY, "32-bit hashes at %p", blob_assembly_hashes); } - log_warn (LOG_ASSEMBLY, "Global hash index contents:"); - hash_t entry_hash; - for (size_t i = 0; i < header->global_entry_count; i++) { - BlobHashEntry &he = blob_assembly_hashes[i]; + // log_warn (LOG_ASSEMBLY, "Global hash index contents:"); + // hash_t entry_hash; + // for (size_t i = 0; i < header->global_entry_count; i++) { + // BlobHashEntry &he = blob_assembly_hashes[i]; - if constexpr (std::is_same_v) { - entry_hash = he.hash64; - } else { - entry_hash = he.hash32; - } + // if constexpr (std::is_same_v) { + // entry_hash = he.hash64; + // } else { + // entry_hash = he.hash32; + // } - log_warn (LOG_ASSEMBLY, " [%u]: hash == 0x%zx; mapping index == %u; local blob index == %u; blob ID == %u", i, entry_hash, he.mapping_index, he.local_blob_index, he.blob_id); - } + // log_warn (LOG_ASSEMBLY, " [%u]: hash == 0x%zx; mapping index == %u; local blob index == %u; blob ID == %u", i, entry_hash, he.mapping_index, he.local_blob_index, he.blob_id); + // } } number_of_mapped_blobs++; + have_and_want_debug_symbols = register_debug_symbols; } force_inline void EmbeddedAssemblies::zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { if (all_blobs_found ()) { - log_warn (LOG_ASSEMBLY, "All blobs already found, further scanning not needed"); + return; } dynamic_local_string entry_name; bool common_assembly_blob_found = false; bool arch_assembly_blob_found = false; - log_warn (LOG_ASSEMBLY, "Looking for blobs in APK (common: '%s'; arch-specific: '%s')", bundled_assemblies_common_blob_name.data (), bundled_assemblies_arch_blob_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for blobs in APK (common: '%s'; arch-specific: '%s')", bundled_assemblies_common_blob_name.data (), bundled_assemblies_arch_blob_name.data ()); for (size_t i = 0; i < num_entries; i++) { if (all_blobs_found ()) { need_to_scan_more_apks = false; - log_warn (LOG_ASSEMBLY, "All blobs found, done scanning (at entry %u)", i); break; } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index a0d38e013d5..e9acacd3c5d 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -67,7 +67,7 @@ void EmbeddedAssemblies::set_assemblies_prefix (const char *prefix) } force_inline void -EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { #if defined (ANDROID) && defined (HAVE_LZ4) && defined (RELEASE) auto header = reinterpret_cast(data); @@ -311,9 +311,8 @@ constexpr char path_separator[] = "/"; force_inline MonoAssembly* EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { - constexpr char dll_extension[] = ".dll"; - if (!utils.ends_with (name, dll_extension)) { - name.append (dll_extension); + if (!utils.ends_with (name, SharedConstants::DLL_EXTENSION)) { + name.append (SharedConstants::DLL_EXTENSION); } log_debug (LOG_ASSEMBLY, "individual_assemblies_open_from_bundles: looking for bundled name: '%s'", name.get ()); @@ -377,24 +376,29 @@ EmbeddedAssemblies::find_blob_assembly_entry (hash_t hash, const BlobHashEntry * force_inline MonoAssembly* EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { - hash_t name_hash = xxhash::hash (name.get (), name.length ()); - log_warn (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); + size_t len = name.length (); + + if (utils.ends_with (name, SharedConstants::DLL_EXTENSION)) { + len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; + } + + hash_t name_hash = xxhash::hash (name.get (), len); + log_debug (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); const BlobHashEntry *hash_entry = find_blob_assembly_entry (name_hash, blob_assembly_hashes, application_config.number_of_assemblies_in_apk); if (hash_entry == nullptr) { - log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); + log_debug (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); return nullptr; } - log_debug (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: found index entry (blob id: %u; index: %u)", hash_entry->blob_id, hash_entry->mapping_index); + // log_debug (LOG_ASSEMBLY, "blob_assemblies_open_from_bundles: found index entry (blob id: %u; index: %u)", hash_entry->blob_id, hash_entry->mapping_index); if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); abort (); } - BlobAssemblyRuntimeData &assembly_data = blob_bundled_assemblies[hash_entry->mapping_index]; - if (assembly_data.image_data == nullptr) { - log_warn (LOG_ASSEMBLY, "Assembly not mapped yet"); + BlobAssemblyRuntimeData &assembly_runtime_info = blob_bundled_assemblies[hash_entry->mapping_index]; + if (assembly_runtime_info.image_data == nullptr) { if (hash_entry->blob_id >= application_config.number_of_assembly_blobs) { log_fatal (LOG_ASSEMBLY, "Invalid assembly blob ID %u, exceeds the maximum of %u", hash_entry->blob_id, application_config.number_of_assembly_blobs - 1); abort (); @@ -406,33 +410,61 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_stringlocal_blob_index]; - assembly_data.image_data = rd.data_start + bba->data_offset; - assembly_data.descriptor = bba; + + // The assignments here don't need to be atomic, the value will always be the same, so even if two threads + // arrive here at the same time, nothing bad will happen. + assembly_runtime_info.image_data = rd.data_start + bba->data_offset; + assembly_runtime_info.descriptor = bba; if (bba->debug_data_offset != 0) { - assembly_data.debug_info_data = rd.data_start + bba->debug_data_offset; + assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; } #if !defined (NET6) if (bba->config_data_size != 0) { - assembly_data.debug_info_data = rd.data_start + bba->config_data_offset; + assembly_runtime_info.config_data = rd.data_start + bba->config_data_offset; + + mono_register_config_for_assembly (name.get (), reinterpret_cast(assembly_runtime_info.config_data)); } #endif // NET6 - } else { - log_warn (LOG_ASSEMBLY, "Assembly already mapped"); + + log_debug ( + LOG_ASSEMBLY, + "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u", + assembly_runtime_info.image_data, + assembly_runtime_info.debug_info_data, + assembly_runtime_info.config_data, + assembly_runtime_info.descriptor, + assembly_runtime_info.descriptor->data_size, + assembly_runtime_info.descriptor->debug_data_size, + assembly_runtime_info.descriptor->config_data_size + ); } - log_warn ( - LOG_ASSEMBLY, - "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u", - assembly_data.image_data, - assembly_data.debug_info_data, - assembly_data.config_data, - assembly_data.descriptor, - assembly_data.descriptor->data_size, - assembly_data.descriptor->debug_data_size, - assembly_data.descriptor->config_data_size - ); - return nullptr; + uint8_t *assembly_data; + uint32_t assembly_data_size; + + get_assembly_data (assembly_runtime_info, assembly_data, assembly_data_size); + MonoImage *image = loader (assembly_data, assembly_data_size, name.get ()); + if (image == nullptr) { + log_warn (LOG_ASSEMBLY, "Failed to load MonoImage of '%s'", name.get ()); + return nullptr; + } + + if (have_and_want_debug_symbols && assembly_runtime_info.debug_info_data != nullptr) { + log_debug (LOG_ASSEMBLY, "Registering debug data for assembly '%s'", name.get ()); + mono_debug_open_image_from_memory (image, reinterpret_cast (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); + } + + MonoImageOpenStatus status; + MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); + if (a == nullptr) { + return nullptr; + } + +#if !defined (NET6) + mono_config_for_assembly (image); +#endif + return a; } MonoAssembly* diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 2caeb42c8d6..e670ef4827c 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -22,6 +22,8 @@ namespace xamarin::android::internal static constexpr char JNIENV_CLASS_NAME[] = "JNIEnv"; static constexpr char ANDROID_ENVIRONMENT_CLASS_NAME[] = "AndroidEnvironment"; + static constexpr char DLL_EXTENSION[] = ".dll"; + #if defined (NET6) static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "rc.bin"; #endif // def NET6 From 43a54cdf7e0a619f65dcb1107a3e2360439fbf20 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Sat, 18 Sep 2021 00:46:30 +0200 Subject: [PATCH 09/54] [WIP] Towards split config detection --- Configuration.props | 1 + src/java-runtime/java-runtime.targets | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Configuration.props b/Configuration.props index bf26d82ab51..75fefeeca48 100644 --- a/Configuration.props +++ b/Configuration.props @@ -25,6 +25,7 @@ v4.4 19 + 21 21 $(AndroidFirstApiLevel) diff --git a/src/java-runtime/java-runtime.targets b/src/java-runtime/java-runtime.targets index 0f17932c4d7..6b43550ea1e 100644 --- a/src/java-runtime/java-runtime.targets +++ b/src/java-runtime/java-runtime.targets @@ -51,7 +51,7 @@ /> <_Target Condition="'$(JavacSourceVersion)' != ''">-source $(JavacSourceVersion) -target $(JavacTargetVersion) - <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidFirstPlatformId)\android.jar" + <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidJavaRuntimeApiLevel)\android.jar" Date: Mon, 20 Sep 2021 14:03:17 +0200 Subject: [PATCH 10/54] [WIP] Optimize code when dealing with split APKs --- .../java/mono/android/DebugRuntime.java | 2 +- .../java/mono/android/MonoPackageManager.java | 12 +++- .../java/mono/android/Runtime.java | 2 +- src/monodroid/jni/basic-android-system.cc | 36 ++++++++---- src/monodroid/jni/basic-android-system.hh | 6 +- src/monodroid/jni/debug-app-helper.cc | 4 +- src/monodroid/jni/embedded-assemblies.cc | 6 +- src/monodroid/jni/embedded-assemblies.hh | 3 +- src/monodroid/jni/mono_android_Runtime.h | 2 +- src/monodroid/jni/monodroid-glue-designer.cc | 2 +- src/monodroid/jni/monodroid-glue-internal.hh | 11 ++-- src/monodroid/jni/monodroid-glue.cc | 57 +++++++++---------- src/monodroid/jni/shared-constants.hh | 4 ++ 13 files changed, 87 insertions(+), 60 deletions(-) diff --git a/src/java-runtime/java/mono/android/DebugRuntime.java b/src/java-runtime/java/mono/android/DebugRuntime.java index fb5d99faff1..8d42e45e263 100644 --- a/src/java-runtime/java/mono/android/DebugRuntime.java +++ b/src/java-runtime/java/mono/android/DebugRuntime.java @@ -4,5 +4,5 @@ public class DebugRuntime { private DebugRuntime () {} - public static native void init (String[] apks, String runtimeLibDir, String[] appDirs); + public static native void init (String[] apks, String runtimeLibDir, String[] appDirs, boolean haveSplitApks); } diff --git a/src/java-runtime/java/mono/android/MonoPackageManager.java b/src/java-runtime/java/mono/android/MonoPackageManager.java index 101f45c86c5..5b0f6416934 100644 --- a/src/java-runtime/java/mono/android/MonoPackageManager.java +++ b/src/java-runtime/java/mono/android/MonoPackageManager.java @@ -43,6 +43,13 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack ClassLoader loader = context.getClassLoader (); String runtimeDir = getNativeLibraryPath (runtimePackage); String[] appDirs = new String[] {filesDir, cacheDir, dataDir}; + boolean haveSplitApks = false; + + if (android.os.Build.VERSION.SDK_INT >= 21) { + if (runtimePackage.splitSourceDirs != null) { + haveSplitApks = runtimePackage.splitSourceDirs.length > 1; + } + } // // Preload DSOs libmonodroid.so depends on so that the dynamic @@ -73,7 +80,7 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack // if (BuildConfig.Debug) { System.loadLibrary ("xamarin-debug-app-helper"); - DebugRuntime.init (apks, runtimeDir, appDirs); + DebugRuntime.init (apks, runtimeDir, appDirs, haveSplitApks); } else { System.loadLibrary("monosgen-2.0"); } @@ -98,7 +105,8 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack loader, MonoPackageManager_Resources.Assemblies, Build.VERSION.SDK_INT, - isEmulator () + isEmulator (), + haveSplitApks ); mono.android.app.ApplicationRegistration.registerApplications (); diff --git a/src/java-runtime/java/mono/android/Runtime.java b/src/java-runtime/java/mono/android/Runtime.java index fb38ba728bb..62547d2b157 100644 --- a/src/java-runtime/java/mono/android/Runtime.java +++ b/src/java-runtime/java/mono/android/Runtime.java @@ -15,7 +15,7 @@ public class Runtime { } public static native void init (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] externalStorageDirs, String[] assemblies, String packageName, int apiLevel, String[] environmentVariables); - public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] assemblies, int apiLevel, boolean isEmulator); + public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] assemblies, int apiLevel, boolean isEmulator, boolean haveSplitApks); public static native void register (String managedType, java.lang.Class nativeClass, String methods); public static native void notifyTimeZoneChanged (); public static native int createNewContext (String[] runtimeApks, String[] assemblies, ClassLoader loader); diff --git a/src/monodroid/jni/basic-android-system.cc b/src/monodroid/jni/basic-android-system.cc index 12c5f7e5191..d27e7da7328 100644 --- a/src/monodroid/jni/basic-android-system.cc +++ b/src/monodroid/jni/basic-android-system.cc @@ -28,7 +28,7 @@ BasicAndroidSystem::detect_embedded_dso_mode (jstring_array_wrapper& appDirs) no } void -BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs) +BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks) { if (!is_embedded_dso_mode_enabled ()) { log_info (LOG_DEFAULT, "Setting up for DSO lookup in app data directories"); @@ -44,7 +44,7 @@ BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtim unsigned short built_for_cpu = 0, running_on_cpu = 0; unsigned char is64bit = 0; _monodroid_detect_cpu_and_architecture (&built_for_cpu, &running_on_cpu, &is64bit); - setup_apk_directories (running_on_cpu, runtimeApks); + setup_apk_directories (running_on_cpu, runtimeApks, have_split_apks); } } @@ -59,20 +59,36 @@ BasicAndroidSystem::for_each_apk (jstring_array_wrapper &runtimeApks, ForEachApk } } -void -BasicAndroidSystem::add_apk_libdir (const char *apk, size_t index, [[maybe_unused]] size_t apk_count, void *user_data) +force_inline void +BasicAndroidSystem::add_apk_libdir (const char *apk, size_t &index, const char *abi) noexcept { - abort_if_invalid_pointer_argument (user_data); abort_unless (index < app_lib_directories_size, "Index out of range"); - app_lib_directories [index] = utils.string_concat (apk, "!/lib/", static_cast(user_data)); + app_lib_directories [index] = utils.string_concat (apk, "!/lib/", abi); log_debug (LOG_ASSEMBLY, "Added APK DSO lookup location: %s", app_lib_directories[index]); + index++; } -void -BasicAndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks) +force_inline void +BasicAndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept { - // Man, the cast is ugly... - for_each_apk (runtimeApks, &BasicAndroidSystem::add_apk_libdir, const_cast (static_cast (android_abi_names [running_on_cpu]))); + const char *abi = android_abi_names [running_on_cpu]; + size_t number_of_added_directories = 0; + + for (size_t i = 0; i < runtimeApks.get_length (); ++i) { + jstring_wrapper &e = runtimeApks [i]; + const char *apk = e.get_cstr (); + + if (have_split_apks) { + if (utils.ends_with (apk, SharedConstants::split_config_abi_apk_name)) { + add_apk_libdir (apk, number_of_added_directories, abi); + break; + } + } else { + add_apk_libdir (apk, number_of_added_directories, abi); + } + } + + app_lib_directories_size = number_of_added_directories; } char* diff --git a/src/monodroid/jni/basic-android-system.hh b/src/monodroid/jni/basic-android-system.hh index 42f17181eb6..a12ba47fa1d 100644 --- a/src/monodroid/jni/basic-android-system.hh +++ b/src/monodroid/jni/basic-android-system.hh @@ -58,8 +58,7 @@ namespace xamarin::android::internal static const char* get_built_for_abi_name (); public: - void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs); - void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks); + void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks); const char* get_override_dir (size_t index) const { @@ -105,10 +104,11 @@ namespace xamarin::android::internal } protected: - void add_apk_libdir (const char *apk, size_t index, size_t apk_count, void *user_data); void for_each_apk (jstring_array_wrapper &runtimeApks, ForEachApkHandler handler, void *user_data); private: + void add_apk_libdir (const char *apk, size_t &index, const char *abi) noexcept; + void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept; char* determine_primary_override_dir (jstring_wrapper &home); void set_embedded_dso_mode_enabled (bool yesno) noexcept diff --git a/src/monodroid/jni/debug-app-helper.cc b/src/monodroid/jni/debug-app-helper.cc index b3966e59361..e47daf8bdef 100644 --- a/src/monodroid/jni/debug-app-helper.cc +++ b/src/monodroid/jni/debug-app-helper.cc @@ -69,7 +69,7 @@ JNI_OnLoad ([[maybe_unused]] JavaVM *vm, [[maybe_unused]] void *reserved) JNIEXPORT void JNICALL Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, jobjectArray runtimeApksJava, - jstring runtimeNativeLibDir, jobjectArray appDirs) + jstring runtimeNativeLibDir, jobjectArray appDirs, jboolean haveSplitApks) { jstring_array_wrapper applicationDirs (env, appDirs); jstring_array_wrapper runtimeApks (env, runtimeApksJava); @@ -77,7 +77,7 @@ Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, androidSystem.detect_embedded_dso_mode (applicationDirs); androidSystem.set_primary_override_dir (applicationDirs [0]); androidSystem.set_override_dir (0, androidSystem.get_primary_override_dir ()); - androidSystem.setup_app_library_directories (runtimeApks, applicationDirs); + androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); jstring_wrapper jstr (env); diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index e9acacd3c5d..3170a898355 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -306,8 +306,6 @@ EmbeddedAssemblies::load_bundled_assembly ( return a; } -constexpr char path_separator[] = "/"; - force_inline MonoAssembly* EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { @@ -320,7 +318,7 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin dynamic_local_string abi_name; abi_name .assign_c (BasicAndroidSystem::get_built_for_abi_name ()) - .append (path_separator) + .append (zip_path_separator) .append (name); MonoAssembly *a = nullptr; @@ -476,7 +474,7 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function name; if (culture != nullptr && *culture != '\0') { name.append_c (culture); - name.append (path_separator); + name.append (zip_path_separator); } name.append_c (asmname); diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index c30f90c793d..fa02db03190 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -77,7 +77,8 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_EOCD_LEN = 22; static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; - static constexpr char assemblies_prefix[] = "assemblies/"; + static constexpr char assemblies_prefix[] = "assemblies/"; + static constexpr char zip_path_separator[] = "/"; static constexpr char bundled_assemblies_blob_prefix[] = "assemblies"; static constexpr char bundled_assemblies_blob_ext[] = ".blob"; diff --git a/src/monodroid/jni/mono_android_Runtime.h b/src/monodroid/jni/mono_android_Runtime.h index 0e5aa37d095..d86951e0991 100644 --- a/src/monodroid/jni/mono_android_Runtime.h +++ b/src/monodroid/jni/mono_android_Runtime.h @@ -21,7 +21,7 @@ JNIEXPORT void JNICALL Java_mono_android_Runtime_init * Signature: (Ljava/lang/String;[Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;Ljava/lang/ClassLoader;[Ljava/lang/String;IZ)V */ JNIEXPORT void JNICALL Java_mono_android_Runtime_initInternal -(JNIEnv *, jclass, jstring, jobjectArray, jstring, jobjectArray, jobject, jobjectArray, jint, jboolean); +(JNIEnv *, jclass, jstring, jobjectArray, jstring, jobjectArray, jobject, jobjectArray, jint, jboolean, jboolean); /* * Class: mono_android_Runtime diff --git a/src/monodroid/jni/monodroid-glue-designer.cc b/src/monodroid/jni/monodroid-glue-designer.cc index 422e45e3c12..8e8d6ab4cba 100644 --- a/src/monodroid/jni/monodroid-glue-designer.cc +++ b/src/monodroid/jni/monodroid-glue-designer.cc @@ -42,7 +42,7 @@ MonodroidRuntime::Java_mono_android_Runtime_createNewContextWithData (JNIEnv *en jstring_array_wrapper runtimeApks (env, runtimeApksJava); jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliePaths (env, assembliesPaths); - MonoDomain *domain = create_and_initialize_domain (env, klass, runtimeApks, assemblies, assembliesBytes, assembliePaths, loader, /*is_root_domain:*/ false, force_preload_assemblies); + MonoDomain *domain = create_and_initialize_domain (env, klass, runtimeApks, assemblies, assembliesBytes, assembliePaths, loader, /*is_root_domain:*/ false, force_preload_assemblies, /* have_split_apks */ false); mono_domain_set (domain, FALSE); int domain_id = mono_domain_get_id (domain); current_context_id = domain_id; diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index f95bd804e63..ce6b573bbb4 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -127,7 +127,6 @@ namespace xamarin::android::internal }; private: - static constexpr auto split_config_abi_apk_name = concat_const ("/split_config.", SharedConstants::android_abi, ".apk"); static constexpr char base_apk_name[] = "/base.apk"; static constexpr size_t SMALL_STRING_PARSE_BUFFER_LEN = 50; static constexpr bool is_running_on_desktop = @@ -154,7 +153,8 @@ namespace xamarin::android::internal void Java_mono_android_Runtime_register (JNIEnv *env, jstring managedType, jclass nativeClass, jstring methods); void Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator); + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks); #if !defined (ANDROID) jint Java_mono_android_Runtime_createNewContextWithData (JNIEnv *env, jclass klass, jobjectArray runtimeApksJava, jobjectArray assembliesJava, jobjectArray assembliesBytes, jobjectArray assembliesPaths, jobject loader, jboolean force_preload_assemblies); @@ -297,12 +297,13 @@ namespace xamarin::android::internal #else // def NET6 MonoClass* get_android_runtime_class (MonoDomain *domain); #endif - MonoDomain* create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain); + MonoDomain* create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks); MonoDomain* create_and_initialize_domain (JNIEnv* env, jclass runtimeClass, jstring_array_wrapper &runtimeApks, jstring_array_wrapper &assemblies, jobjectArray assembliesBytes, jstring_array_wrapper &assembliesPaths, - jobject loader, bool is_root_domain, bool force_preload_assemblies); + jobject loader, bool is_root_domain, bool force_preload_assemblies, + bool have_split_apks); - void gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count); + void gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count, bool have_split_apks); static bool should_register_file (const char *filename); void set_trace_options (); void set_profile_options (); diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 09c24198fad..1eb74b5a775 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -392,7 +392,7 @@ MonodroidRuntime::should_register_file ([[maybe_unused]] const char *filename) } inline void -MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count) +MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count, bool have_split_apks) { #if defined(DEBUG) || !defined (ANDROID) if (application_config.instant_run_enabled) { @@ -408,30 +408,25 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; - // bool got_base_apk = false; - // bool got_split_config_abi_apk = false; - // bool scan_apk; + bool got_split_config_abi_apk = false; + bool got_base_apk = false; for (int64_t i = 0; i < apk_count; i++) { jstring_wrapper &apk_file = runtimeApks [static_cast(i)]; - // - // Code below can be enabled only when we can reliably detect that we're running with split APKs. - // The best check appears to involve https://developer.android.com/reference/android/content/pm/ApplicationInfo#splitSourceDirs - // However, the above works only if we can compile our Java code against at least API21, but we compile against - // API19 atm - // - // scan_apk = false; + if (have_split_apks) { + bool scan_apk = false; - // if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { - // got_base_apk = scan_apk = true; - // } else if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), split_config_abi_apk_name)) { - // got_split_config_abi_apk = scan_apk = true; - // } + if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), SharedConstants::split_config_abi_apk_name)) { + got_split_config_abi_apk = scan_apk = true; + } else if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { + got_base_apk = scan_apk = true; + } - // if (!scan_apk) { - // continue; - // } + if (!scan_apk) { + continue; + } + } size_t cur_num_assemblies = embeddedAssemblies.register_from (apk_file.get_cstr ()); @@ -884,7 +879,7 @@ MonodroidRuntime::cleanup_runtime_config (MonovmRuntimeConfigArguments *args, [[ #endif // def NET6 MonoDomain* -MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain) +MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks) { size_t user_assemblies_count = 0; #if defined (NET6) @@ -893,7 +888,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks bool have_mono_mkbundle_init = mono_mkbundle_init != nullptr; #endif // ndef NET6 - gather_bundled_assemblies (runtimeApks, &user_assemblies_count); + gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); #if defined (NET6) timing_period blob_time; @@ -1864,9 +1859,9 @@ MonoDomain* MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass, jstring_array_wrapper &runtimeApks, jstring_array_wrapper &assemblies, [[maybe_unused]] jobjectArray assembliesBytes, [[maybe_unused]] jstring_array_wrapper &assembliesPaths, jobject loader, bool is_root_domain, - bool force_preload_assemblies) + bool force_preload_assemblies, bool have_split_apks) { - MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain); + MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain, have_split_apks); // When running on desktop, the root domain is only a dummy so don't initialize it if constexpr (is_running_on_desktop) { @@ -2053,7 +2048,8 @@ MonodroidRuntime::install_logging_handlers () inline void MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator) + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks) { char *mono_log_mask = nullptr; char *mono_log_level = nullptr; @@ -2108,7 +2104,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl disable_external_signal_handlers (); jstring_array_wrapper runtimeApks (env, runtimeApksJava); - androidSystem.setup_app_library_directories (runtimeApks, applicationDirs); + androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); init_reference_logging (androidSystem.get_primary_override_dir ()); androidSystem.create_update_dir (androidSystem.get_primary_override_dir ()); @@ -2256,7 +2252,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliesPaths (env); /* the first assembly is used to initialize the AppDomain name */ - create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false); + create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false, haveSplitApks); #if defined (ANDROID) && !defined (NET6) // Mono from mono/mono has a bug which requires us to install the handlers after `mono_init_jit_version` is called @@ -2343,14 +2339,16 @@ Java_mono_android_Runtime_init (JNIEnv *env, jclass klass, jstring lang, jobject loader, assembliesJava, apiLevel, - /* isEmulator */ JNI_FALSE + /* isEmulator */ JNI_FALSE, + /* haveSplitApks */ JNI_FALSE ); } JNIEXPORT void JNICALL Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator) + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks) { monodroidRuntime.Java_mono_android_Runtime_initInternal ( env, @@ -2362,7 +2360,8 @@ Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, loader, assembliesJava, apiLevel, - isEmulator + isEmulator, + haveSplitApks ); } diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index e670ef4827c..f3778e7b632 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -1,6 +1,8 @@ #ifndef __SHARED_CONSTANTS_HH #define __SHARED_CONSTANTS_HH +#include "cpp-util.hh" + namespace xamarin::android::internal { // _WIN32 is defined with _WIN64 so _WIN64 must be checked first. @@ -51,6 +53,8 @@ namespace xamarin::android::internal #elif __i386__ static constexpr char android_abi[] = "x86"; #endif + + static constexpr auto split_config_abi_apk_name = concat_const ("/split_config.", android_abi, ".apk"); }; } #endif // __SHARED_CONSTANTS_HH From 9030cf32e8e440e14564397e5146e7000172028b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 20 Sep 2021 22:20:09 +0200 Subject: [PATCH 11/54] [WIP] Fix DebugRuntime.init signature --- src/monodroid/jni/debug-app-helper.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monodroid/jni/debug-app-helper.hh b/src/monodroid/jni/debug-app-helper.hh index 817c7636afd..209f1baba30 100644 --- a/src/monodroid/jni/debug-app-helper.hh +++ b/src/monodroid/jni/debug-app-helper.hh @@ -12,6 +12,6 @@ extern "C" { * Signature: ([Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String);[Ljava/lang/String);Ljava/lang/String;IZ)V */ JNIEXPORT void JNICALL Java_mono_android_DebugRuntime_init - (JNIEnv *, jclass, jobjectArray, jstring, jobjectArray); + (JNIEnv *, jclass, jobjectArray, jstring, jobjectArray, jboolean); } #endif // _Included_mono_android_DebugRuntime From b2d5325c7ca57665b0101d19df1903772e3cbcd1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 21 Sep 2021 23:03:37 +0200 Subject: [PATCH 12/54] [WIP] Beginnings of the blob reader --- Xamarin.Android.sln | 7 ++ .../Tasks/BuildApk.cs | 10 ++- .../Tasks/GeneratePackageManagerJava.cs | 1 - .../PackagingTest.cs | 7 +- .../Utilities/ArchAssemblyBlob.cs | 2 +- .../Utilities/AssemblyBlob.cs | 15 ++-- .../Utilities/AssemblyBlobGenerator.cs | 9 ++- .../Xamarin.Android.Common.targets | 2 + src/monodroid/jni/embedded-assemblies-zip.cc | 5 ++ src/monodroid/jni/xamarin-app.hh | 2 + tools/assembly-blob-reader/BlobAssembly.cs | 25 ++++++ tools/assembly-blob-reader/BlobHashEntry.cs | 25 ++++++ tools/assembly-blob-reader/BlobReader.cs | 79 +++++++++++++++++++ tools/assembly-blob-reader/Program.cs | 57 +++++++++++++ .../assembly-blob-reader.csproj | 24 ++++++ tools/tmt/tmt.csproj | 1 + 16 files changed, 260 insertions(+), 11 deletions(-) create mode 100644 tools/assembly-blob-reader/BlobAssembly.cs create mode 100644 tools/assembly-blob-reader/BlobHashEntry.cs create mode 100644 tools/assembly-blob-reader/BlobReader.cs create mode 100644 tools/assembly-blob-reader/Program.cs create mode 100644 tools/assembly-blob-reader/assembly-blob-reader.csproj diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 3804cbfc365..46b98910697 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -148,6 +148,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "to EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-blob-reader", "tools\assembly-blob-reader\assembly-blob-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Xamarin.Android.NamingCustomAttributes\Xamarin.Android.NamingCustomAttributes.projitems*{3f1f2f50-af1a-4a5a-bedb-193372f068d7}*SharedItemsImports = 5 @@ -408,6 +410,10 @@ Global {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Debug|AnyCPU.Build.0 = Debug|Any CPU {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Release|AnyCPU.ActiveCfg = Release|Any CPU {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Release|AnyCPU.Build.0 = Release|Any CPU + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Debug|AnyCPU.ActiveCfg = Debug|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Debug|AnyCPU.Build.0 = Debug|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Release|AnyCPU.ActiveCfg = Release|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Release|AnyCPU.Build.0 = Release|anycpu EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -474,6 +480,7 @@ Global {37FCD325-1077-4603-98E7-4509CAD648D6} = {864062D3-A415-4A6F-9324-5820237BA058} {88B746FF-8D6E-464D-9D66-FF2ECCF148E0} = {864062D3-A415-4A6F-9324-5820237BA058} {1A273ED2-AE84-48E9-9C23-E978C2D0CB34} = {864062D3-A415-4A6F-9324-5820237BA058} + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51} = {864062D3-A415-4A6F-9324-5820237BA058} {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B} = {04E3E11E-B47D-4599-8AFC-50515A95E715} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index f5009e086b0..977dbef5ae6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -378,7 +378,15 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> (StringComparer.OrdinalIgnoreCase); } - public override void WriteIndex (List globalIndex) + public override string WriteIndex (List globalIndex) { throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 69e83cc97fb..a190c561a4c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -11,7 +11,9 @@ namespace Xamarin.Android.Tasks { abstract class AssemblyBlob { - const uint BlobMagic = 0x41424158; // 'XABA', little-endian + // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh + const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant + const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); @@ -60,7 +62,7 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg public abstract void Add (BlobAssemblyInfo blobAssembly); public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - public virtual void WriteIndex (List globalIndex) + public virtual string WriteIndex (List globalIndex) { if (!IsIndexBlob) { throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); @@ -91,6 +93,8 @@ public virtual void WriteIndex (List globalIndex) File.Delete (indexBlobPath); File.Move (indexBlobHeaderPath, indexBlobPath); + + return indexBlobManifestPath; } void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) @@ -108,6 +112,7 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List (); + manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); // TODO: check if there are no duplicates here foreach (AssemblyBlobIndexEntry assembly in globalIndex) { if (assembly.BlobID == ID) { @@ -115,10 +120,7 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List> Generate (string outputDirectory) var globalIndex = new List (); var ret = new Dictionary> (StringComparer.Ordinal); + string indexBlobApkName = null; foreach (var kvp in blobs) { string apkName = kvp.Key; Blob blob = kvp.Value; @@ -95,11 +96,17 @@ public Dictionary> Generate (string outputDirectory) ret.Add (apkName, new List ()); } + if (blob.Common == indexBlob || blob.Arch == indexBlob) { + indexBlobApkName = apkName; + } + GenerateBlob (blob.Common, apkName); GenerateBlob (blob.Arch, apkName); } - indexBlob.WriteIndex (globalIndex); + string manifestPath = indexBlob.WriteIndex (globalIndex); + ret[indexBlobApkName].Add (manifestPath); + return ret; void GenerateBlob (AssemblyBlob blob, string apkName) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index ccf5537b1f5..37094f52089 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,6 +179,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False + True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> @@ -1596,6 +1597,7 @@ because xbuild doesn't support framework reference assemblies. InstantRunEnabled="$(_InstantRunEnabled)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" + UseAssembliesBlob="$(AndroidUseAssembliesBlob)" > diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index f72336d16e8..5e7a4f1af3f 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -185,6 +185,11 @@ EmbeddedAssemblies::map_assembly_blob (dynamic_local_string c abort (); } + if (header->version > BUNDLED_ASSEMBLIES_BLOB_VERSION) { + log_fatal (LOG_ASSEMBLY, "Blob '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); + abort (); + } + if (header->blob_id >= application_config.number_of_assembly_blobs) { log_fatal ( LOG_ASSEMBLY, diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 7981ba380db..64a4aae7da0 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -11,6 +11,7 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian +static constexpr uint32_t BUNDLED_ASSEMBLIES_BLOB_VERSION = 1; // Increase whenever an incompatible change is made to the blob format static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -136,6 +137,7 @@ struct XamarinAndroidBundledAssembly final struct [[gnu::packed]] BundledAssemblyBlobHeader final { uint32_t magic; + uint32_t version; uint32_t local_entry_count; uint32_t global_entry_count; uint32_t blob_id; diff --git a/tools/assembly-blob-reader/BlobAssembly.cs b/tools/assembly-blob-reader/BlobAssembly.cs new file mode 100644 index 00000000000..ed38cc52cff --- /dev/null +++ b/tools/assembly-blob-reader/BlobAssembly.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class BlobAssembly + { + public uint DataOffset { get; } + public uint DataSize { get; } + public uint DebugDataOffset { get; } + public uint DebugDataSize { get; } + public uint ConfigDataOffset { get; } + public uint ConfigDataSize { get; } + + internal BlobAssembly (BinaryReader reader) + { + DataOffset = reader.ReadUInt32 (); + DataSize = reader.ReadUInt32 (); + DebugDataOffset = reader.ReadUInt32 (); + DebugDataSize = reader.ReadUInt32 (); + ConfigDataOffset = reader.ReadUInt32 (); + ConfigDataSize = reader.ReadUInt32 (); + } + } +} diff --git a/tools/assembly-blob-reader/BlobHashEntry.cs b/tools/assembly-blob-reader/BlobHashEntry.cs new file mode 100644 index 00000000000..856466e3bc7 --- /dev/null +++ b/tools/assembly-blob-reader/BlobHashEntry.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class BlobHashEntry + { + public bool Is32Bit { get; } + + public ulong Hash { get; } + public uint MappingIndex { get; } + public uint LocalBlobIndex { get; } + public uint BlobID { get; } + + internal BlobHashEntry (BinaryReader reader, bool is32Bit) + { + Is32Bit = is32Bit; + + Hash = reader.ReadUInt64 (); + MappingIndex = reader.ReadUInt32 (); + LocalBlobIndex = reader.ReadUInt32 (); + BlobID = reader.ReadUInt32 (); + } + } +} diff --git a/tools/assembly-blob-reader/BlobReader.cs b/tools/assembly-blob-reader/BlobReader.cs new file mode 100644 index 00000000000..9be55a138c8 --- /dev/null +++ b/tools/assembly-blob-reader/BlobReader.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class BlobReader + { + // These two constants must be identical to the native ones in src/monodroid/jni/xamarin-app.hh + const uint BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian + const uint BUNDLED_ASSEMBLIES_BLOB_VERSION = 1; // The highest format version this reader understands + + public uint Version { get; private set; } + public uint LocalEntryCount { get; private set; } + public uint GlobalEntryCount { get; private set; } + public uint BlobID { get; private set; } + public List Assemblies { get; } + public List GlobalIndex32 { get; } + public List GlobalIndex64 { get; } + + public bool HasGlobalIndex => BlobID == 0; + + public BlobReader (Stream blob) + { + using (var reader = new BinaryReader (blob, Encoding.UTF8, leaveOpen: true)) { + ReadHeader (reader); + + Assemblies = new List (); + ReadLocalEntries (reader, Assemblies); + if (HasGlobalIndex) { + GlobalIndex32 = new List (); + GlobalIndex64 = new List (); + ReadGlobalIndex (reader, GlobalIndex32, GlobalIndex64); + } + } + } + + void ReadHeader (BinaryReader reader) + { + uint magic = reader.ReadUInt32 (); + if (magic != BUNDLED_ASSEMBLIES_BLOB_MAGIC) { + throw new InvalidOperationException ("Invalid header magic number"); + } + + Version = reader.ReadUInt32 (); + if (Version == 0) { + throw new InvalidOperationException ("Invalid version number: 0"); + } + + if (Version > BUNDLED_ASSEMBLIES_BLOB_VERSION) { + throw new InvalidOperationException ($"Blob format version {Version} is higher than the one understood by this reader, {BUNDLED_ASSEMBLIES_BLOB_VERSION}"); + } + + LocalEntryCount = reader.ReadUInt32 (); + GlobalEntryCount = reader.ReadUInt32 (); + BlobID = reader.ReadUInt32 (); + } + + void ReadLocalEntries (BinaryReader reader, List assemblies) + { + for (uint i = 0; i < LocalEntryCount; i++) { + assemblies.Add (new BlobAssembly (reader)); + } + } + + void ReadGlobalIndex (BinaryReader reader, List index32, List index64) + { + ReadIndex (true, index32); + ReadIndex (true, index64); + + void ReadIndex (bool is32Bit, List index) { + for (uint i = 0; i < GlobalEntryCount; i++) { + index.Add (new BlobHashEntry (reader, is32Bit)); + } + } + } + } +} diff --git a/tools/assembly-blob-reader/Program.cs b/tools/assembly-blob-reader/Program.cs new file mode 100644 index 00000000000..3d6d2bb0989 --- /dev/null +++ b/tools/assembly-blob-reader/Program.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; + +using Mono.Options; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class Program + { + static void Main (string[] args) + { + using (var fs = File.OpenRead (args[0])) { + var br = new BlobReader (fs); + Console.WriteLine ($"Blob {args[0]} information:"); + Console.WriteLine ($" Format version: {br.Version}"); + Console.WriteLine ($" Local entry count: {br.LocalEntryCount}"); + Console.WriteLine ($" Global entry count: {br.GlobalEntryCount}"); + Console.WriteLine ($" Blob ID: {br.BlobID}"); + + string yesno = br.HasGlobalIndex ? "yes" : "no"; + Console.WriteLine ($" Contains global index: {yesno}"); + + Console.WriteLine (); + Console.WriteLine ("Assemblies in the blob:"); + + for (int i = 0; i < br.Assemblies.Count; i++) { + BlobAssembly assembly = br.Assemblies[i]; + Console.Write ($" {i:d03}: "); + WriteAssemblySegment ("data", assembly.DataOffset, assembly.DataSize); + Console.Write ("; "); + WriteAssemblySegment ("debug data", assembly.DebugDataOffset, assembly.DebugDataSize); + Console.Write ("; "); + WriteAssemblySegment ("config file", assembly.ConfigDataOffset, assembly.ConfigDataSize); + Console.WriteLine (); + } + + if (br.HasGlobalIndex) { + Console.WriteLine (); + Console.WriteLine ("Global index"); + Console.WriteLine (" 32-bit hash entries:"); + + Console.WriteLine (" 64-bit hash entries:"); + } + } + } + + static void WriteAssemblySegment (string label, uint offset, uint size) + { + if (offset == 0) { + Console.Write ($"no {label}"); + return; + } + + Console.Write ($"{label} starts at {offset}, {size} bytes"); + } + } +} diff --git a/tools/assembly-blob-reader/assembly-blob-reader.csproj b/tools/assembly-blob-reader/assembly-blob-reader.csproj new file mode 100644 index 00000000000..b2098204f66 --- /dev/null +++ b/tools/assembly-blob-reader/assembly-blob-reader.csproj @@ -0,0 +1,24 @@ + + + + + Microsoft Corporation + 2020 Microsoft Corporation + 0.0.1 + false + ../../bin/$(Configuration)/bin/assembly-blob-reader + Exe + net6.0 + Xamarin.Android.AssemblyBlobReader + disable + enable + + + + + + + + + + diff --git a/tools/tmt/tmt.csproj b/tools/tmt/tmt.csproj index 8b98c4a56e6..998c541c70d 100644 --- a/tools/tmt/tmt.csproj +++ b/tools/tmt/tmt.csproj @@ -10,6 +10,7 @@ Exe true enable + Major From 28611c9ce574e246872fa155e1ccff8342d3aad9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 23 Sep 2021 19:18:39 +0200 Subject: [PATCH 13/54] [WIP] Blob reader console app --- .../Utilities/ArchAssemblyBlob.cs | 2 +- .../Utilities/AssemblyBlob.cs | 2 +- src/monodroid/jni/embedded-assemblies.hh | 2 +- tools/assembly-blob-reader/BlobAssembly.cs | 11 +- tools/assembly-blob-reader/BlobExplorer.cs | 246 ++++++++++++++++++ .../BlobExplorerLogLevel.cs | 10 + .../assembly-blob-reader/BlobManifestEntry.cs | 63 +++++ .../BlobManifestReader.cs | 46 ++++ tools/assembly-blob-reader/BlobReader.cs | 14 +- tools/assembly-blob-reader/Program.cs | 78 ++++-- 10 files changed, 438 insertions(+), 36 deletions(-) create mode 100644 tools/assembly-blob-reader/BlobExplorer.cs create mode 100644 tools/assembly-blob-reader/BlobExplorerLogLevel.cs create mode 100644 tools/assembly-blob-reader/BlobManifestEntry.cs create mode 100644 tools/assembly-blob-reader/BlobManifestReader.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index 90362afc752..a9d4bf01278 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -95,7 +95,7 @@ public override void Generate (string outputDirectory, List logger; + + public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); + public IDictionary AssembliesByHash32 { get; } = new Dictionary (); + public IDictionary AssembliesByHash64 { get; } = new Dictionary (); + public List Assemblies { get; } = new List (); + public IDictionary> Blobs { get; } = new SortedDictionary> (); + public string BlobPath { get; } + public string BlobSetName { get; } + + public bool IsCompleteSet => indexBlob != null && manifest != null; + public int NumberOfBlobs => numberOfBlobs; + public Action Logger => logger; + + // blobPath can point to: + // + // aab + // apk + // index blob (e.g. base_assemblies.blob) + // arch blob (e.g. base_assemblies.arm64_v8a.blob) + // blob manifest (e.g. base_assemblies.manifest) + // blob base name (e.g. base or base_assemblies) + // + // In each case the whole set of blobs and manifests will be read (if available). Search for the various members of the blob set (common/main blob, arch blobs, + // manifest) is based on this naming convention: + // + // {BASE_NAME}[.ARCH_NAME].{blob|manifest} + // + // Whichever file is referenced in `blobPath`, the BASE_NAME component is extracted and all the found files are read. + // If `blobPath` points to an aab or an apk, BASE_NAME will always be `assemblies` + // + public BlobExplorer (string blobPath, Action? customLogger = null) + { + logger = customLogger ?? DefaultLogger; + + if (String.IsNullOrEmpty (blobPath)) { + throw new ArgumentException ("must not be null or empty", nameof (blobPath)); + } + + if (Directory.Exists (blobPath)) { + throw new ArgumentException ($"'{blobPath}' points to a directory", nameof (blobPath)); + } + + BlobPath = blobPath; + string? extension = Path.GetExtension (blobPath); + string? baseName = null; + + if (String.IsNullOrEmpty (extension)) { + baseName = GetBaseNameNoExtension (blobPath); + } else { + baseName = GetBaseNameHaveExtension (blobPath, extension); + } + + if (String.IsNullOrEmpty (baseName)) { + throw new InvalidOperationException ($"Unable to determine base name of a blob set from path '{blobPath}'"); + } + + BlobSetName = baseName; + if (!IsAndroidArchive (extension)) { + string? directoryName = Path.GetDirectoryName (blobPath); + if (String.IsNullOrEmpty (directoryName)) { + directoryName = "."; + } + + ReadBlobSetFromFilesystem (baseName, directoryName); + } else { + ReadBlobSetFromArchive (baseName, blobPath); + } + + ProcessBlobs (); + } + + void DefaultLogger (BlobExplorerLogLevel level, string message) + { + Console.WriteLine ($"{level}: {message}"); + } + + void ProcessBlobs () + { + if (Blobs.Count == 0 || indexBlob == null) { + return; + } + + ProcessIndex (indexBlob.GlobalIndex32, "32", (BlobHashEntry he, BlobAssembly assembly) => { + assembly.Hash32 = (uint)he.Hash; + assembly.RuntimeIndex = he.MappingIndex; + + if (manifest != null && manifest.EntriesByHash32.TryGetValue (assembly.Hash32, out BlobManifestEntry? me) && me != null) { + assembly.Name = me.Name; + } + + if (!AssembliesByHash32.ContainsKey (assembly.Hash32)) { + AssembliesByHash32.Add (assembly.Hash32, assembly); + } + }); + + ProcessIndex (indexBlob.GlobalIndex64, "64", (BlobHashEntry he, BlobAssembly assembly) => { + assembly.Hash64 = he.Hash; + if (assembly.RuntimeIndex != he.MappingIndex) { + Logger (BlobExplorerLogLevel.Warning, $"assembly with hashes 0x{assembly.Hash32} and 0x{assembly.Hash64} has a different 32-bit runtime index ({assembly.RuntimeIndex}) than the 64-bit runtime index({he.MappingIndex})"); + } + + if (manifest != null && manifest.EntriesByHash64.TryGetValue (assembly.Hash64, out BlobManifestEntry? me) && me != null) { + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} did not match any assembly name in the manifest"); + assembly.Name = me.Name; + } else if (String.Compare (assembly.Name, me.Name, StringComparison.Ordinal) != 0) { + Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} maps to assembly name '{assembly.Name}', however 64-bit hash 0x{assembly.Hash64:x} for the same entry matches assembly name '{me.Name}'"); + } + } + + if (!AssembliesByHash64.ContainsKey (assembly.Hash64)) { + AssembliesByHash64.Add (assembly.Hash64, assembly); + } + }); + + // TODO: compare arch-specific blogs and warn if they differ + + void ProcessIndex (List index, string bitness, Action assemblyHandler) + { + foreach (BlobHashEntry he in index) { + if (!Blobs.TryGetValue (he.BlobID, out List? blobList) || blobList == null) { + Logger (BlobExplorerLogLevel.Warning, $"blob with id {he.BlobID} not part of the set"); + continue; + } + + foreach (BlobReader blob in blobList) { + if (he.LocalBlobIndex >= (uint)blob.Assemblies.Count) { + Logger (BlobExplorerLogLevel.Warning, $"{bitness}-bit index entry with hash 0x{he.Hash:x} has invalid blob {blob.BlobID} index {he.LocalBlobIndex} (maximum allowed is {blob.Assemblies.Count})"); + continue; + } + + BlobAssembly assembly = blob.Assemblies[(int)he.LocalBlobIndex]; + assemblyHandler (he, assembly); + + if (!AssembliesByName.ContainsKey (assembly.Name)) { + AssembliesByName.Add (assembly.Name, assembly); + } + } + } + } + } + + void ReadBlobSetFromArchive (string baseName, string archivePath) + { + } + + void ReadBlobSetFromFilesystem (string baseName, string setPath) + { + foreach (string de in Directory.EnumerateFiles (setPath, $"{baseName}.*", SearchOption.TopDirectoryOnly)) { + string? extension = Path.GetExtension (de); + if (String.IsNullOrEmpty (extension)) { + continue; + } + + if (String.Compare (".blob", extension, StringComparison.OrdinalIgnoreCase) == 0) { + BlobReader reader = ReadBlob (de); + if (reader.HasGlobalIndex) { + indexBlob = reader; + } + + List? blobList; + if (!Blobs.TryGetValue (reader.BlobID, out blobList)) { + blobList = new List (); + Blobs.Add (reader.BlobID, blobList); + } + blobList.Add (reader); + + Assemblies.AddRange (reader.Assemblies); + } else if (String.Compare (".manifest", extension, StringComparison.OrdinalIgnoreCase) == 0) { + manifest = ReadManifest (de); + } + } + + BlobReader ReadBlob (string filePath) + { + string? arch = Path.GetFileNameWithoutExtension (filePath); + if (!String.IsNullOrEmpty (arch)) { + arch = Path.GetExtension (arch); + if (!String.IsNullOrEmpty (arch)) { + arch = arch.Substring (1); + } + } + + using (var fs = File.OpenRead (filePath)) { + return CreateBlobReader (fs, arch); + } + } + + BlobManifestReader ReadManifest (string filePath) + { + using (var fs = File.OpenRead (filePath)) { + return new BlobManifestReader (fs); + } + } + } + + BlobReader CreateBlobReader (Stream input, string arch) + { + numberOfBlobs++; + return new BlobReader (input, arch); + } + + bool IsAndroidArchive (string extension) + { + return + String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0; + } + + string GetBaseNameHaveExtension (string blobPath, string extension) + { + if (IsAndroidArchive (extension)) { + return "assemblies"; + } + + string fileName = Path.GetFileNameWithoutExtension (blobPath); + int dot = fileName.IndexOf ('.'); + if (dot >= 0) { + return fileName.Substring (0, dot); + } + + return fileName; + } + + string GetBaseNameNoExtension (string blobPath) + { + string fileName = Path.GetFileName (blobPath); + if (fileName.EndsWith ("_assemblies")) { + return fileName; + } + return $"{fileName}_assemblies"; + } + } +} diff --git a/tools/assembly-blob-reader/BlobExplorerLogLevel.cs b/tools/assembly-blob-reader/BlobExplorerLogLevel.cs new file mode 100644 index 00000000000..203be0a0429 --- /dev/null +++ b/tools/assembly-blob-reader/BlobExplorerLogLevel.cs @@ -0,0 +1,10 @@ +namespace Xamarin.Android.AssemblyBlobReader +{ + enum BlobExplorerLogLevel + { + Debug, + Info, + Warning, + Error, + } +} diff --git a/tools/assembly-blob-reader/BlobManifestEntry.cs b/tools/assembly-blob-reader/BlobManifestEntry.cs new file mode 100644 index 00000000000..e77466c09da --- /dev/null +++ b/tools/assembly-blob-reader/BlobManifestEntry.cs @@ -0,0 +1,63 @@ +using System; +using System.Globalization; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class BlobManifestEntry + { + // Fields are: + // Hash 32 | Hash 64 | Blob ID | Blob idx | Name + const int NumberOfFields = 5; + const int Hash32FieldIndex = 0; + const int Hash64FieldIndex = 1; + const int BlobIDFieldIndex = 2; + const int BlobIndexFieldIndex = 3; + const int NameFieldIndex = 4; + + public uint Hash32 { get; } + public ulong Hash64 { get; } + public uint BlobID { get; } + public uint IndexInBlob { get; } + public string Name { get; } + + public BlobManifestEntry (string[] fields) + { + if (fields.Length != NumberOfFields) { + throw new ArgumentOutOfRangeException (nameof (fields), "Invalid number of fields"); + } + + Hash32 = GetUInt32 (fields[Hash32FieldIndex]); + Hash64 = GetUInt64 (fields[Hash64FieldIndex]); + BlobID = GetUInt32 (fields[BlobIDFieldIndex]); + IndexInBlob = GetUInt32 (fields[BlobIndexFieldIndex]); + Name = fields[NameFieldIndex].Trim (); + } + + uint GetUInt32 (string value) + { + if (UInt32.TryParse (PrepHexValue (value), NumberStyles.HexNumber, null, out uint hash)) { + return hash; + } + + return 0; + } + + ulong GetUInt64 (string value) + { + if (UInt64.TryParse (PrepHexValue (value), NumberStyles.HexNumber, null, out ulong hash)) { + return hash; + } + + return 0; + } + + string PrepHexValue (string value) + { + if (value.StartsWith ("0x", StringComparison.Ordinal)) { + return value.Substring (2); + } + + return value; + } + } +} diff --git a/tools/assembly-blob-reader/BlobManifestReader.cs b/tools/assembly-blob-reader/BlobManifestReader.cs new file mode 100644 index 00000000000..bf7fa1ea6b8 --- /dev/null +++ b/tools/assembly-blob-reader/BlobManifestReader.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Xamarin.Android.AssemblyBlobReader +{ + class BlobManifestReader + { + public List Entries { get; } = new List (); + public Dictionary EntriesByHash32 { get; } = new Dictionary (); + public Dictionary EntriesByHash64 { get; } = new Dictionary (); + + public BlobManifestReader (Stream manifest) + { + manifest.Seek (0, SeekOrigin.Begin); + using (var sr = new StreamReader (manifest, Encoding.UTF8, detectEncodingFromByteOrderMarks: false)) { + ReadManifest (sr); + } + } + + void ReadManifest (StreamReader reader) + { + // First line is ignored, it contains headers + reader.ReadLine (); + + // Each subsequent line consists of fields separated with any number of spaces (for the pleasure of a human being reading the manifest) + while (!reader.EndOfStream) { + string[]? fields = reader.ReadLine ()?.Split (' ', StringSplitOptions.RemoveEmptyEntries); + if (fields == null) { + continue; + } + + var entry = new BlobManifestEntry (fields); + Entries.Add (entry); + if (entry.Hash32 != 0) { + EntriesByHash32.Add (entry.Hash32, entry); + } + + if (entry.Hash64 != 0) { + EntriesByHash64.Add (entry.Hash64, entry); + } + } + } + } +} diff --git a/tools/assembly-blob-reader/BlobReader.cs b/tools/assembly-blob-reader/BlobReader.cs index 9be55a138c8..f242689b942 100644 --- a/tools/assembly-blob-reader/BlobReader.cs +++ b/tools/assembly-blob-reader/BlobReader.cs @@ -16,21 +16,23 @@ class BlobReader public uint GlobalEntryCount { get; private set; } public uint BlobID { get; private set; } public List Assemblies { get; } - public List GlobalIndex32 { get; } - public List GlobalIndex64 { get; } + public List GlobalIndex32 { get; } = new List (); + public List GlobalIndex64 { get; } = new List (); + public string Arch { get; } public bool HasGlobalIndex => BlobID == 0; - public BlobReader (Stream blob) + public BlobReader (Stream blob, string? arch = null) { + Arch = arch ?? String.Empty; + + blob.Seek (0, SeekOrigin.Begin); using (var reader = new BinaryReader (blob, Encoding.UTF8, leaveOpen: true)) { ReadHeader (reader); Assemblies = new List (); ReadLocalEntries (reader, Assemblies); if (HasGlobalIndex) { - GlobalIndex32 = new List (); - GlobalIndex64 = new List (); ReadGlobalIndex (reader, GlobalIndex32, GlobalIndex64); } } @@ -60,7 +62,7 @@ void ReadHeader (BinaryReader reader) void ReadLocalEntries (BinaryReader reader, List assemblies) { for (uint i = 0; i < LocalEntryCount; i++) { - assemblies.Add (new BlobAssembly (reader)); + assemblies.Add (new BlobAssembly (reader, this)); } } diff --git a/tools/assembly-blob-reader/Program.cs b/tools/assembly-blob-reader/Program.cs index 3d6d2bb0989..82f10371acd 100644 --- a/tools/assembly-blob-reader/Program.cs +++ b/tools/assembly-blob-reader/Program.cs @@ -9,37 +9,63 @@ class Program { static void Main (string[] args) { - using (var fs = File.OpenRead (args[0])) { - var br = new BlobReader (fs); - Console.WriteLine ($"Blob {args[0]} information:"); - Console.WriteLine ($" Format version: {br.Version}"); - Console.WriteLine ($" Local entry count: {br.LocalEntryCount}"); - Console.WriteLine ($" Global entry count: {br.GlobalEntryCount}"); - Console.WriteLine ($" Blob ID: {br.BlobID}"); + var explorer = new BlobExplorer (args[0]); - string yesno = br.HasGlobalIndex ? "yes" : "no"; - Console.WriteLine ($" Contains global index: {yesno}"); + string yesno = explorer.IsCompleteSet ? "yes" : "no"; + Console.WriteLine ($"Blob set '{explorer.BlobSetName}':"); + Console.WriteLine ($" Is complete set? {yesno}"); + Console.WriteLine ($" Number of blobs in the set: {explorer.NumberOfBlobs}"); + Console.WriteLine (); + Console.WriteLine ("Assemblies:"); - Console.WriteLine (); - Console.WriteLine ("Assemblies in the blob:"); - - for (int i = 0; i < br.Assemblies.Count; i++) { - BlobAssembly assembly = br.Assemblies[i]; - Console.Write ($" {i:d03}: "); - WriteAssemblySegment ("data", assembly.DataOffset, assembly.DataSize); - Console.Write ("; "); - WriteAssemblySegment ("debug data", assembly.DebugDataOffset, assembly.DebugDataSize); - Console.Write ("; "); - WriteAssemblySegment ("config file", assembly.ConfigDataOffset, assembly.ConfigDataSize); - Console.WriteLine (); + string infoIndent = " "; + foreach (BlobAssembly assembly in explorer.Assemblies) { + Console.WriteLine ($" {assembly.RuntimeIndex}:"); + Console.Write ($"{infoIndent}Name: "); + if (String.IsNullOrEmpty (assembly.Name)) { + Console.WriteLine ("unknown"); + } else { + Console.WriteLine (assembly.Name); + } + + Console.Write ($"{infoIndent}Blob ID: {assembly.Blob.BlobID} ("); + if (String.IsNullOrEmpty (assembly.Blob.Arch)) { + Console.Write ("shared"); + } else { + Console.Write (assembly.Blob.Arch); } + Console.WriteLine (")"); + + Console.Write ($"{infoIndent}Hashes: 32-bit == "); + WriteHashValue (assembly.Hash32); + + Console.Write ("; 64-bit == "); + WriteHashValue (assembly.Hash64); + Console.WriteLine (); - if (br.HasGlobalIndex) { - Console.WriteLine (); - Console.WriteLine ("Global index"); - Console.WriteLine (" 32-bit hash entries:"); + Console.WriteLine ($"{infoIndent}Assembly image: offset == {assembly.DataOffset}; size == {assembly.DataSize}"); + WriteOptionalDataLine ("Debug data", assembly.DebugDataOffset, assembly.DebugDataOffset); + WriteOptionalDataLine ("Config file", assembly.ConfigDataOffset, assembly.ConfigDataSize); + + Console.WriteLine (); + } + + void WriteOptionalDataLine (string label, uint offset, uint size) + { + Console.Write ($"{infoIndent}{label}: "); + if (offset == 0) { + Console.WriteLine ("absent"); + } else { + Console.WriteLine ("offset == {offset}; size == {size}"); + } + } - Console.WriteLine (" 64-bit hash entries:"); + void WriteHashValue (ulong hash) + { + if (hash == 0) { + Console.Write ("unknown"); + } else { + Console.Write ($"0x{hash:x}"); } } } From 4e7ac66b23400c63bd9a89a00b4e483d25b12937 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 24 Sep 2021 14:24:28 +0200 Subject: [PATCH 14/54] [WIP] Support reading from apk/aab archives --- tools/assembly-blob-reader/BlobExplorer.cs | 129 ++++++++++++++++----- tools/assembly-blob-reader/BlobReader.cs | 12 ++ 2 files changed, 115 insertions(+), 26 deletions(-) diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index baa508cf30e..24f53ffb9b4 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; +using Xamarin.Tools.Zip; + namespace Xamarin.Android.AssemblyBlobReader { class BlobExplorer @@ -18,10 +20,11 @@ class BlobExplorer public IDictionary> Blobs { get; } = new SortedDictionary> (); public string BlobPath { get; } public string BlobSetName { get; } + public bool HasErrors { get; private set; } + public bool HasWarnings { get; private set; } public bool IsCompleteSet => indexBlob != null && manifest != null; public int NumberOfBlobs => numberOfBlobs; - public Action Logger => logger; // blobPath can point to: // @@ -75,12 +78,27 @@ public BlobExplorer (string blobPath, Action? cust ReadBlobSetFromFilesystem (baseName, directoryName); } else { - ReadBlobSetFromArchive (baseName, blobPath); + ReadBlobSetFromArchive (baseName, blobPath, extension); } ProcessBlobs (); } + void Logger (BlobExplorerLogLevel level, string message) + { + if (level == BlobExplorerLogLevel.Error) { + HasErrors = true; + } else if (level == BlobExplorerLogLevel.Warning) { + HasWarnings = true; + } + + if (logger != null) { + logger (level, message); + } else { + DefaultLogger (level, message); + } + } + void DefaultLogger (BlobExplorerLogLevel level, string message) { Console.WriteLine ($"{level}: {message}"); @@ -115,6 +133,9 @@ void ProcessBlobs () if (String.IsNullOrEmpty (assembly.Name)) { Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} did not match any assembly name in the manifest"); assembly.Name = me.Name; + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (BlobExplorerLogLevel.Warning, $"64-bit hash 0x{assembly.Hash64:x} did not match any assembly name in the manifest"); + } } else if (String.Compare (assembly.Name, me.Name, StringComparison.Ordinal) != 0) { Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} maps to assembly name '{assembly.Name}', however 64-bit hash 0x{assembly.Hash64:x} for the same entry matches assembly name '{me.Name}'"); } @@ -125,7 +146,20 @@ void ProcessBlobs () } }); - // TODO: compare arch-specific blogs and warn if they differ + foreach (var kvp in Blobs) { + List list = kvp.Value; + if (list.Count < 2) { + continue; + } + + BlobReader template = list[0]; + for (int i = 1; i < list.Count; i++) { + BlobReader other = list[i]; + if (!template.HasIdenticalContent (other)) { + Logger (BlobExplorerLogLevel.Error, $"Blob ID {template.BlobID} for architecture {other.Arch} is not identical to other blobs with the same ID"); + } + } + } void ProcessIndex (List index, string bitness, Action assemblyHandler) { @@ -152,8 +186,70 @@ void ProcessIndex (List index, string bitness, Action? blobList; + if (!Blobs.TryGetValue (reader.BlobID, out blobList)) { + blobList = new List (); + Blobs.Add (reader.BlobID, blobList); + } + blobList.Add (reader); + + Assemblies.AddRange (reader.Assemblies); + } + + string? GetBlobArch (string path) { + string? arch = Path.GetFileNameWithoutExtension (path); + if (!String.IsNullOrEmpty (arch)) { + arch = Path.GetExtension (arch); + if (!String.IsNullOrEmpty (arch)) { + arch = arch.Substring (1); + } + } + + return arch; } void ReadBlobSetFromFilesystem (string baseName, string setPath) @@ -165,19 +261,7 @@ void ReadBlobSetFromFilesystem (string baseName, string setPath) } if (String.Compare (".blob", extension, StringComparison.OrdinalIgnoreCase) == 0) { - BlobReader reader = ReadBlob (de); - if (reader.HasGlobalIndex) { - indexBlob = reader; - } - - List? blobList; - if (!Blobs.TryGetValue (reader.BlobID, out blobList)) { - blobList = new List (); - Blobs.Add (reader.BlobID, blobList); - } - blobList.Add (reader); - - Assemblies.AddRange (reader.Assemblies); + AddBlob (ReadBlob (de)); } else if (String.Compare (".manifest", extension, StringComparison.OrdinalIgnoreCase) == 0) { manifest = ReadManifest (de); } @@ -185,14 +269,7 @@ void ReadBlobSetFromFilesystem (string baseName, string setPath) BlobReader ReadBlob (string filePath) { - string? arch = Path.GetFileNameWithoutExtension (filePath); - if (!String.IsNullOrEmpty (arch)) { - arch = Path.GetExtension (arch); - if (!String.IsNullOrEmpty (arch)) { - arch = arch.Substring (1); - } - } - + string? arch = GetBlobArch (filePath); using (var fs = File.OpenRead (filePath)) { return CreateBlobReader (fs, arch); } @@ -206,7 +283,7 @@ BlobManifestReader ReadManifest (string filePath) } } - BlobReader CreateBlobReader (Stream input, string arch) + BlobReader CreateBlobReader (Stream input, string? arch) { numberOfBlobs++; return new BlobReader (input, arch); diff --git a/tools/assembly-blob-reader/BlobReader.cs b/tools/assembly-blob-reader/BlobReader.cs index f242689b942..86fc8da714c 100644 --- a/tools/assembly-blob-reader/BlobReader.cs +++ b/tools/assembly-blob-reader/BlobReader.cs @@ -38,6 +38,18 @@ public BlobReader (Stream blob, string? arch = null) } } + public bool HasIdenticalContent (BlobReader other) + { + return + other.Version == Version && + other.LocalEntryCount == LocalEntryCount && + other.GlobalEntryCount == GlobalEntryCount && + other.BlobID == BlobID && + other.Assemblies.Count == Assemblies.Count && + other.GlobalIndex32.Count == GlobalIndex32.Count && + other.GlobalIndex64.Count == GlobalIndex64.Count; + } + void ReadHeader (BinaryReader reader) { uint magic = reader.ReadUInt32 (); From bd5ea4cafe7547af1c02398ff69c77f11fc705f9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 27 Sep 2021 16:23:51 +0200 Subject: [PATCH 15/54] [WIP] Test helper progress --- .../PackagingTest.cs | 24 +++--- .../Utilities/ArchiveAssemblyHelper.cs | 84 +++++++++++++++++++ .../Xamarin.Android.Build.Tests.csproj | 1 + .../MSBuildDeviceIntegration.csproj | 1 + tools/assembly-blob-reader/BlobExplorer.cs | 4 +- .../BlobManifestReader.cs | 4 +- 6 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 889d405204f..04943207477 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -112,17 +112,19 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssembliesB Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - using (var zip = ZipHelper.OpenZip (apk)) { - var existingFiles = zip.Where (a => a.FullName.StartsWith ("assemblies/", StringComparison.InvariantCultureIgnoreCase)); - var missingFiles = expectedFiles.Where (x => !zip.ContainsEntry ("assemblies/" + Path.GetFileName (x))); - Assert.IsFalse (missingFiles.Any (), - string.Format ("The following Expected files are missing. {0}", - string.Join (Environment.NewLine, missingFiles))); - var additionalFiles = existingFiles.Where (x => !expectedFiles.Contains (Path.GetFileName (x.FullName))); - Assert.IsTrue (!additionalFiles.Any (), - string.Format ("Unexpected Files found! {0}", - string.Join (Environment.NewLine, additionalFiles.Select (x => x.FullName)))); - } + var helper = new ArchiveAssemblyHelper (apk, usesAssembliesBlob); + IEnumerable existingFiles; + IEnumerable missingFiles; + IEnumerable additionalFiles; + + (existingFiles, missingFiles, additionalFiles) = helper.Contains (expectedFiles); + Assert.IsFalse (missingFiles.Any (), + string.Format ("The following Expected files are missing. {0}", + string.Join (Environment.NewLine, missingFiles))); + + Assert.IsTrue (!additionalFiles.Any (), + string.Format ("Unexpected Files found! {0}", + string.Join (Environment.NewLine, additionalFiles))); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs new file mode 100644 index 00000000000..17c5abb3d5d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Xamarin.Android.AssemblyBlobReader; +using Xamarin.ProjectTools; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.Build.Tests +{ + class ArchiveAssemblyHelper + { + readonly string archivePath; + readonly string assembliesRootDir; + bool useAssemblyBlobs; + + public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) + { + if (String.IsNullOrEmpty (archivePath)) { + throw new ArgumentException ("must not be null or empty", nameof (archivePath)); + } + + this.archivePath = archivePath; + this.useAssemblyBlobs = useAssemblyBlobs; + + string extension = Path.GetExtension (archivePath) ?? String.Empty; + if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "assemblies/"; + } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "base/root/assemblies"; + } else { + throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); + } + } + + public (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) Contains (string[] assemblyNames) + { + if (assemblyNames == null) { + throw new ArgumentNullException (nameof (assemblyNames)); + } + + if (assemblyNames.Length == 0) { + throw new ArgumentException ("must not be empty", nameof (assemblyNames)); + } + + if (useAssemblyBlobs) { + return BlobContains (assemblyNames); + } + + return ArchiveContains (assemblyNames); + } + + (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) ArchiveContains (string[] assemblyNames) + { + using (var zip = ZipHelper.OpenZip (archivePath)) { + IEnumerable existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)); + IEnumerable missingFiles = assemblyNames.Where (x => !zip.ContainsEntry (assembliesRootDir + Path.GetFileName (x))); + IEnumerable additionalFiles = existingFiles.Where (x => !assemblyNames.Contains (Path.GetFileName (x.FullName))).Select (x => x.FullName); + + return (existingFiles.Select (x => x.FullName), missingFiles, additionalFiles); + } + } + + (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) BlobContains (string[] assemblyNames) + { + // TODO: support files not related to assemblies (e.g. rc.bin) + // TODO: support .config and .pdb/.mdb files + // Blobs don't store the assembly extension + IEnumerable expectedNames = assemblyNames.Select (x => Path.GetFileNameWithoutExtension (x)); + + var explorer = new BlobExplorer (archivePath); + if (explorer.AssembliesByName.Count == 0) { + return (new string[0], new string[0], new string[0]); + } + + IEnumerable existingFiles = explorer.AssembliesByName.Keys; + IEnumerable missingFiles = expectedNames.Where (x => !explorer.AssembliesByName.ContainsKey (x)); + IEnumerable additionalFiles = existingFiles.Where (x => !expectedNames.Contains (x)); + + return (existingFiles, missingFiles, additionalFiles); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index 34662f9ca3a..a57da49b8fb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -32,6 +32,7 @@ + ..\Expected\GenerateDesignerFileExpected.cs PreserveNewest diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index 9af5f8da79e..ab5c0b9fe88 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -22,6 +22,7 @@ + diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index 24f53ffb9b4..c88a143bcb0 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -11,7 +11,7 @@ class BlobExplorer BlobReader? indexBlob; BlobManifestReader? manifest; int numberOfBlobs = 0; - Action logger; + Action? logger; public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); public IDictionary AssembliesByHash32 { get; } = new Dictionary (); @@ -45,8 +45,6 @@ class BlobExplorer // public BlobExplorer (string blobPath, Action? customLogger = null) { - logger = customLogger ?? DefaultLogger; - if (String.IsNullOrEmpty (blobPath)) { throw new ArgumentException ("must not be null or empty", nameof (blobPath)); } diff --git a/tools/assembly-blob-reader/BlobManifestReader.cs b/tools/assembly-blob-reader/BlobManifestReader.cs index bf7fa1ea6b8..9c438b6103c 100644 --- a/tools/assembly-blob-reader/BlobManifestReader.cs +++ b/tools/assembly-blob-reader/BlobManifestReader.cs @@ -7,6 +7,8 @@ namespace Xamarin.Android.AssemblyBlobReader { class BlobManifestReader { + static readonly char[] fieldSplit = new char[] { ' ' }; + public List Entries { get; } = new List (); public Dictionary EntriesByHash32 { get; } = new Dictionary (); public Dictionary EntriesByHash64 { get; } = new Dictionary (); @@ -26,7 +28,7 @@ void ReadManifest (StreamReader reader) // Each subsequent line consists of fields separated with any number of spaces (for the pleasure of a human being reading the manifest) while (!reader.EndOfStream) { - string[]? fields = reader.ReadLine ()?.Split (' ', StringSplitOptions.RemoveEmptyEntries); + string[]? fields = reader.ReadLine ()?.Split (fieldSplit, StringSplitOptions.RemoveEmptyEntries); if (fields == null) { continue; } From 117c8f752623bcd99570c054b2cff816e966c757 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 28 Sep 2021 18:55:19 +0200 Subject: [PATCH 16/54] [WIP] Fix a handful of tests --- .../Tasks/BuildApk.cs | 1 + .../PackagingTest.cs | 17 +- .../Utilities/ArchiveAssemblyHelper.cs | 158 +++++++++++++++--- .../Utilities/AssemblyBlob.cs | 10 +- .../Xamarin.Android.Common.targets | 1 + .../Tests/BundleToolTests.cs | 93 ++++++++--- tools/assembly-blob-reader/BlobExplorer.cs | 5 +- 7 files changed, 221 insertions(+), 64 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 977dbef5ae6..72e85b1425a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -406,6 +406,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) // Add assembly var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); if (useAssembliesBlob) { + Log.LogDebugMessage ($"Creating BlobAssemblyInfo: sourcePath == {sourcePath}; assemblyPath == {assemblyPath}"); blobAssembly = new BlobAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); } else { AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 04943207477..9a9ad7e8c0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -69,10 +69,6 @@ public void CheckProguardMappingFileExists () [Test] public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssembliesBlob) { - if (usesAssembliesBlob) { - Assert.Ignore ("Assembly blob not implemented yet"); - } - var proj = new XamarinAndroidApplicationProject { IsRelease = true }; @@ -113,16 +109,17 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssembliesB var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssembliesBlob); - IEnumerable existingFiles; - IEnumerable missingFiles; - IEnumerable additionalFiles; + List existingFiles; + List missingFiles; + List additionalFiles; + + helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles); - (existingFiles, missingFiles, additionalFiles) = helper.Contains (expectedFiles); - Assert.IsFalse (missingFiles.Any (), + Assert.IsTrue (missingFiles == null || missingFiles.Count == 0, string.Format ("The following Expected files are missing. {0}", string.Join (Environment.NewLine, missingFiles))); - Assert.IsTrue (!additionalFiles.Any (), + Assert.IsTrue (additionalFiles == null || additionalFiles.Count == 0, string.Format ("Unexpected Files found! {0}", string.Join (Environment.NewLine, additionalFiles))); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 17c5abb3d5d..26c2e369c5f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -11,6 +11,15 @@ namespace Xamarin.Android.Build.Tests { class ArchiveAssemblyHelper { + public const string DefaultBlobEntryPrefix = "{blob}"; + + static readonly HashSet SpecialExtensions = new HashSet (StringComparer.OrdinalIgnoreCase) { + ".dll", + ".config", + ".pdb", + ".mdb", + }; + readonly string archivePath; readonly string assembliesRootDir; bool useAssemblyBlobs; @@ -26,59 +35,154 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) string extension = Path.GetExtension (archivePath) ?? String.Empty; if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "assemblies/"; - } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { assembliesRootDir = "base/root/assemblies"; + } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "assemblies/"; + } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "root/assemblies/"; } else { throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); } } - public (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) Contains (string[] assemblyNames) + public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix) { - if (assemblyNames == null) { - throw new ArgumentNullException (nameof (assemblyNames)); + if (String.IsNullOrEmpty (blobEntryPrefix)) { + throw new ArgumentException (nameof (blobEntryPrefix), "must not be null or empty"); } - if (assemblyNames.Length == 0) { - throw new ArgumentException ("must not be empty", nameof (assemblyNames)); + var entries = new List (); + using (var zip = ZipArchive.Open (archivePath, FileMode.Open)) { + foreach (var entry in zip) { + entries.Add (entry.FullName); + } } - if (useAssemblyBlobs) { - return BlobContains (assemblyNames); + if (!useAssemblyBlobs) { + Console.WriteLine ("Not using assembly blobs"); + return entries; } - return ArchiveContains (assemblyNames); + var explorer = new BlobExplorer (archivePath); + Console.WriteLine ($"explorer.AssembliesByName count: {explorer.AssembliesByName.Count}"); + foreach (var kvp in explorer.AssembliesByName) { + string name = kvp.Key; + BlobAssembly asm = kvp.Value; + + entries.Add ($"{blobEntryPrefix}{name}.dll"); + if (asm.DebugDataOffset > 0) { + entries.Add ($"{blobEntryPrefix}{name}.pdb"); + } + + if (asm.ConfigDataOffset > 0) { + entries.Add ($"{blobEntryPrefix}{name}.dll.config"); + } + } + + return entries; } - (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) ArchiveContains (string[] assemblyNames) + public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) { - using (var zip = ZipHelper.OpenZip (archivePath)) { - IEnumerable existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)); - IEnumerable missingFiles = assemblyNames.Where (x => !zip.ContainsEntry (assembliesRootDir + Path.GetFileName (x))); - IEnumerable additionalFiles = existingFiles.Where (x => !assemblyNames.Contains (Path.GetFileName (x.FullName))).Select (x => x.FullName); + if (fileNames == null) { + throw new ArgumentNullException (nameof (fileNames)); + } - return (existingFiles.Select (x => x.FullName), missingFiles, additionalFiles); + if (fileNames.Length == 0) { + throw new ArgumentException ("must not be empty", nameof (fileNames)); + } + + if (useAssemblyBlobs) { + BlobContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + } else { + ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); } } - (IEnumerable existingFiles, IEnumerable missingFiles, IEnumerable additionalFiles) BlobContains (string[] assemblyNames) + void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) { - // TODO: support files not related to assemblies (e.g. rc.bin) - // TODO: support .config and .pdb/.mdb files - // Blobs don't store the assembly extension - IEnumerable expectedNames = assemblyNames.Select (x => Path.GetFileNameWithoutExtension (x)); + using (var zip = ZipHelper.OpenZip (archivePath)) { + existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); + missingFiles = fileNames.Where (x => !zip.ContainsEntry (assembliesRootDir + Path.GetFileName (x))).ToList (); + additionalFiles = existingFiles.Where (x => !fileNames.Contains (Path.GetFileName (x))).ToList (); + } + } + + void BlobContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + { + var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); + var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); + var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase) || x.EndsWith (".mdb", StringComparison.OrdinalIgnoreCase)).ToList (); + var otherFiles = fileNames.Where (x => !SpecialExtensions.Contains (Path.GetExtension (x))).ToList (); + + existingFiles = new List (); + missingFiles = new List (); + additionalFiles = new List (); + + if (otherFiles.Count > 0) { + using (var zip = ZipHelper.OpenZip (archivePath)) { + foreach (string file in otherFiles) { + string fullPath = assembliesRootDir + Path.GetFileName (file); + if (zip.ContainsEntry (fullPath)) { + existingFiles.Add (file); + } + } + } + } var explorer = new BlobExplorer (archivePath); - if (explorer.AssembliesByName.Count == 0) { - return (new string[0], new string[0], new string[0]); + + // Blobs don't store the assembly extension + var blobAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); + if (explorer.AssembliesByName.Count != 0) { + existingFiles.AddRange (blobAssemblies); + + // We need to fake config and debug files since they have no named entries in the blob + foreach (string file in configFiles) { + BlobAssembly asm = GetBlobAssembly (file); + if (asm == null) { + continue; + } + + if (asm.ConfigDataOffset > 0) { + existingFiles.Add (file); + } + } + + foreach (string file in debugFiles) { + BlobAssembly asm = GetBlobAssembly (file); + if (asm == null) { + continue; + } + + if (asm.DebugDataOffset > 0) { + existingFiles.Add (file); + } + } } - IEnumerable existingFiles = explorer.AssembliesByName.Keys; - IEnumerable missingFiles = expectedNames.Where (x => !explorer.AssembliesByName.ContainsKey (x)); - IEnumerable additionalFiles = existingFiles.Where (x => !expectedNames.Contains (x)); + foreach (string file in fileNames) { + if (existingFiles.Contains (Path.GetFileName (file))) { + continue; + } + missingFiles.Add (file); + } - return (existingFiles, missingFiles, additionalFiles); + additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); + + BlobAssembly GetBlobAssembly (string file) + { + string assemblyName = Path.GetFileNameWithoutExtension (file); + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + assemblyName = Path.GetFileNameWithoutExtension (assemblyName); + } + + if (!explorer.AssembliesByName.TryGetValue (assemblyName, out BlobAssembly asm) || asm == null) { + return null; + } + + return asm; + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 224954e3851..5055eec1c5a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -208,6 +208,9 @@ void Generate (BinaryWriter writer, List assemblies, List assemblies, List .so;$(AndroidStoreUncompressedFileExtensions) .dex;$(AndroidStoreUncompressedFileExtensions) + .blob;$(AndroidStoreUncompressedFileExtensions) diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index eac880827ed..f049ba91ace 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -10,15 +10,22 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] + [TestFixtureSource(nameof(FixtureArgs))] [Category ("Node-1")] public class BundleToolTests : DeviceTest { + static readonly object[] FixtureArgs = { + new object[] { false }, + new object[] { true }, + }; + static readonly string [] Abis = new [] { "armeabi-v7a", "arm64-v8a", "x86" }; XamarinAndroidLibraryProject lib; XamarinAndroidApplicationProject app; ProjectBuilder libBuilder, appBuilder; string intermediate; string bin; + bool usesAssemblyBlobs; // Disable split by language const string BuildConfig = @"{ @@ -34,6 +41,11 @@ public class BundleToolTests : DeviceTest } }"; + public BundleToolTests (bool usesAssemblyBlobs) + { + this.usesAssemblyBlobs = usesAssemblyBlobs; + } + [OneTimeSetUp] public void OneTimeSetUp () { @@ -51,6 +63,8 @@ public void OneTimeSetUp () } }; + lib.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + var bytes = new byte [1024]; app = new XamarinFormsMapsApplicationProject { IsRelease = true, @@ -103,14 +117,10 @@ public void OneTimeTearDown () appBuilder?.Dispose (); } - string [] ListArchiveContents (string archive) + string [] ListArchiveContents (string archive, bool usesAssembliesBlob) { - var entries = new List (); - using (var zip = ZipArchive.Open (archive, FileMode.Open)) { - foreach (var entry in zip) { - entries.Add (entry.FullName); - } - } + var helper = new ArchiveAssemblyHelper (archive, usesAssembliesBlob); + List entries = helper.ListArchiveContents (); entries.Sort (); return entries.ToArray (); } @@ -119,7 +129,7 @@ string [] ListArchiveContents (string archive) public void BaseZip () { var baseZip = Path.Combine (intermediate, "android", "bin", "base.zip"); - var contents = ListArchiveContents (baseZip); + var contents = ListArchiveContents (baseZip, usesAssemblyBlobs); var expectedFiles = new List { "dex/classes.dex", "manifest/AndroidManifest.xml", @@ -130,16 +140,33 @@ public void BaseZip () "res/drawable-xxxhdpi-v4/icon.png", "res/layout/main.xml", "resources.pb", - "root/assemblies/Java.Interop.dll", - "root/assemblies/Mono.Android.dll", - "root/assemblies/Localization.dll", - "root/assemblies/es/Localization.resources.dll", - "root/assemblies/UnnamedProject.dll", }; + + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultBlobEntryPrefix; + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); + } else { + expectedFiles.Add ("root/assemblies/Java.Interop.dll"); + expectedFiles.Add ("root/assemblies/Mono.Android.dll"); + expectedFiles.Add ("root/assemblies/Localization.dll"); + expectedFiles.Add ("root/assemblies/es/Localization.resources.dll"); + expectedFiles.Add ("root/assemblies/UnnamedProject.dll"); + } + if (Builder.UseDotNet) { - expectedFiles.Add ("root/assemblies/System.Console.dll"); - expectedFiles.Add ("root/assemblies/System.Linq.dll"); - expectedFiles.Add ("root/assemblies/System.Net.Http.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}System.Console.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Linq.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Net.Http.dll"); + } else { + expectedFiles.Add ("root/assemblies/System.Console.dll"); + expectedFiles.Add ("root/assemblies/System.Linq.dll"); + expectedFiles.Add ("root/assemblies/System.Net.Http.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/play-services-base.properties"); @@ -147,10 +174,17 @@ public void BaseZip () expectedFiles.Add ("root/play-services-maps.properties"); expectedFiles.Add ("root/play-services-tasks.properties"); } else { - expectedFiles.Add ("root/assemblies/mscorlib.dll"); - expectedFiles.Add ("root/assemblies/System.Core.dll"); - expectedFiles.Add ("root/assemblies/System.dll"); - expectedFiles.Add ("root/assemblies/System.Runtime.Serialization.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}mscorlib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Core.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Runtime.Serialization.dll"); + } else { + expectedFiles.Add ("root/assemblies/mscorlib.dll"); + expectedFiles.Add ("root/assemblies/System.Core.dll"); + expectedFiles.Add ("root/assemblies/System.dll"); + expectedFiles.Add ("root/assemblies/System.Runtime.Serialization.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/build-data.properties"); @@ -180,7 +214,7 @@ public void AppBundle () { var aab = Path.Combine (intermediate, "android", "bin", $"{app.PackageName}.aab"); FileAssert.Exists (aab); - var contents = ListArchiveContents (aab); + var contents = ListArchiveContents (aab, usesAssemblyBlobs); var expectedFiles = new List { "base/dex/classes.dex", "base/manifest/AndroidManifest.xml", @@ -243,7 +277,7 @@ public void AppBundleSigned () { var aab = Path.Combine (bin, $"{app.PackageName}-Signed.aab"); FileAssert.Exists (aab); - var contents = ListArchiveContents (aab); + var contents = ListArchiveContents (aab, usesAssembliesBlob: false); Assert.IsTrue (StringAssertEx.ContainsText (contents, "META-INF/MANIFEST.MF"), $"{aab} is not signed!"); } @@ -259,11 +293,11 @@ public void ApkSet () FileAssert.Exists (aab); // Expecting: splits/base-arm64_v8a.apk, splits/base-master.apk, splits/base-xxxhdpi.apk // This are split up based on: abi, base, and dpi - var contents = ListArchiveContents (aab).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); + var contents = ListArchiveContents (aab, usesAssemblyBlobs).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); Assert.AreEqual (3, contents.Length, "Expecting three APKs!"); // Language split has been removed by the bundle configuration file, and therefore shouldn't be present - var languageSplitContent = ListArchiveContents (aab).Where (a => a.EndsWith ("-en.apk", StringComparison.OrdinalIgnoreCase)).ToArray (); + var languageSplitContent = ListArchiveContents (aab, usesAssemblyBlobs).Where (a => a.EndsWith ("-en.apk", StringComparison.OrdinalIgnoreCase)).ToArray (); Assert.AreEqual (0, languageSplitContent.Length, "Found language split apk in bundle, but disabled by bundle configuration file!"); using (var stream = new MemoryStream ()) @@ -273,7 +307,16 @@ public void ApkSet () baseMaster.Extract (stream); stream.Position = 0; - var uncompressed = new [] { ".dll", ".bar", ".wav" }; + var uncompressed = new List { + ".bar", + ".wav", + }; + + if (usesAssemblyBlobs) { + uncompressed.Add (".blob"); + } else { + uncompressed.Add (".dll"); + } using (var baseApk = ZipArchive.Open (stream)) { foreach (var file in baseApk) { foreach (var ext in uncompressed) { diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index c88a143bcb0..0e9c1c4311a 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -192,6 +192,8 @@ void ReadBlobSetFromArchive (string baseName, string archivePath, string extensi basePathInArchive = "base/root/assemblies"; } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { basePathInArchive = "assemblies"; + } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "root/assemblies"; } else { throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); } @@ -291,7 +293,8 @@ bool IsAndroidArchive (string extension) { return String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0 || - String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0; + String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0; } string GetBaseNameHaveExtension (string blobPath, string extension) From 88f95bf308dbb375fcceffd00dd875e6aa64e275 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 28 Sep 2021 23:45:47 +0200 Subject: [PATCH 17/54] [WIP] More test fixes + some debug logging --- .../Utilities/ArchiveAssemblyHelper.cs | 3 +- .../Utilities/AssemblyBlobGenerator.cs | 4 ++ .../Tests/BundleToolTests.cs | 51 ++++++++++++++----- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 26c2e369c5f..c99b9ecedef 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -41,7 +41,7 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { assembliesRootDir = "root/assemblies/"; } else { - throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); + assembliesRootDir = String.Empty; } } @@ -64,7 +64,6 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } var explorer = new BlobExplorer (archivePath); - Console.WriteLine ($"explorer.AssembliesByName count: {explorer.AssembliesByName.Count}"); foreach (var kvp in explorer.AssembliesByName) { string name = kvp.Key; BlobAssembly asm = kvp.Value; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs index 12edb6fbff2..37997e07676 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs @@ -38,6 +38,7 @@ public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper public void Add (string apkName, BlobAssemblyInfo blobAssembly) { + log.LogMessage (MessageImportance.Low, $"Add: apkName == '{apkName}'"); if (String.IsNullOrEmpty (apkName)) { throw new ArgumentException ("must not be null or empty", nameof (apkName)); } @@ -63,10 +64,13 @@ public void Add (string apkName, BlobAssemblyInfo blobAssembly) void SetIndexBlob (AssemblyBlob b) { + log.LogMessage (MessageImportance.Low, $"Checking if {b} is an index blob"); if (!b.IsIndexBlob) { + log.LogMessage (MessageImportance.Low, $" it is not (ID: {b.ID})"); return; } + log.LogMessage (MessageImportance.Low, $" it is"); if (indexBlob != null) { throw new InvalidOperationException ("Index blob already set!"); } diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index f049ba91ace..e0eff168a85 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -87,6 +87,7 @@ public void OneTimeSetUp () app.SetProperty (app.ReleaseProperties, "AndroidPackageFormat", "aab"); app.SetAndroidSupportedAbis (Abis); app.SetProperty ("AndroidBundleConfigurationFile", "buildConfig.json"); + app.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); libBuilder = CreateDllBuilder (Path.Combine (path, lib.ProjectName), cleanupOnDispose: true); Assert.IsTrue (libBuilder.Build (lib), "Library build should have succeeded."); @@ -226,17 +227,34 @@ public void AppBundle () "base/res/drawable-xxxhdpi-v4/icon.png", "base/res/layout/main.xml", "base/resources.pb", - "base/root/assemblies/Java.Interop.dll", - "base/root/assemblies/Mono.Android.dll", - "base/root/assemblies/Localization.dll", - "base/root/assemblies/es/Localization.resources.dll", - "base/root/assemblies/UnnamedProject.dll", "BundleConfig.pb", }; + + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultBlobEntryPrefix; + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/Java.Interop.dll"); + expectedFiles.Add ("base/root/assemblies/Mono.Android.dll"); + expectedFiles.Add ("base/root/assemblies/Localization.dll"); + expectedFiles.Add ("base/root/assemblies/es/Localization.resources.dll"); + expectedFiles.Add ("base/root/assemblies/UnnamedProject.dll"); + } + if (Builder.UseDotNet) { - expectedFiles.Add ("base/root/assemblies/System.Console.dll"); - expectedFiles.Add ("base/root/assemblies/System.Linq.dll"); - expectedFiles.Add ("base/root/assemblies/System.Net.Http.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}System.Console.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Linq.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Net.Http.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/System.Console.dll"); + expectedFiles.Add ("base/root/assemblies/System.Linq.dll"); + expectedFiles.Add ("base/root/assemblies/System.Net.Http.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/play-services-base.properties"); @@ -244,10 +262,17 @@ public void AppBundle () expectedFiles.Add ("base/root/play-services-maps.properties"); expectedFiles.Add ("base/root/play-services-tasks.properties"); } else { - expectedFiles.Add ("base/root/assemblies/mscorlib.dll"); - expectedFiles.Add ("base/root/assemblies/System.Core.dll"); - expectedFiles.Add ("base/root/assemblies/System.dll"); - expectedFiles.Add ("base/root/assemblies/System.Runtime.Serialization.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}mscorlib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Core.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Runtime.Serialization.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/mscorlib.dll"); + expectedFiles.Add ("base/root/assemblies/System.Core.dll"); + expectedFiles.Add ("base/root/assemblies/System.dll"); + expectedFiles.Add ("base/root/assemblies/System.Runtime.Serialization.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/build-data.properties"); @@ -293,7 +318,7 @@ public void ApkSet () FileAssert.Exists (aab); // Expecting: splits/base-arm64_v8a.apk, splits/base-master.apk, splits/base-xxxhdpi.apk // This are split up based on: abi, base, and dpi - var contents = ListArchiveContents (aab, usesAssemblyBlobs).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); + var contents = ListArchiveContents (aab, usesAssembliesBlob: false).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); Assert.AreEqual (3, contents.Length, "Expecting three APKs!"); // Language split has been removed by the bundle configuration file, and therefore shouldn't be present From f81a467a67dd85e40071adedbf7fde9e5e469d48 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 29 Sep 2021 15:44:29 +0200 Subject: [PATCH 18/54] [WIP] More test fixes --- .../Tasks/BuildApk.cs | 28 ++++---- .../BuildTest.TestCaseSource.cs | 49 ++++++++++++- .../Xamarin.Android.Build.Tests/BuildTest.cs | 72 +++++++++---------- .../Utilities/ArchiveAssemblyHelper.cs | 18 ++++- .../Utilities/AssemblyBlob.cs | 7 +- 5 files changed, 121 insertions(+), 53 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 72e85b1425a..a793a86a650 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -120,7 +120,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List existingEntries = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string blobApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -180,7 +180,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } if (EmbedAssemblies && !BundleAssemblies) - AddAssemblies (apk, debug, compress, compressedAssembliesInfo); + AddAssemblies (apk, debug, compress, compressedAssembliesInfo, blobApkName); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); @@ -301,7 +301,7 @@ public override bool RunTask () throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); } - ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo); + ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, blobApkName: null); outputFiles.Add (ApkOutputPath); if (CreatePackagePerAbi && SupportedAbis.Length > 1) { foreach (var abi in SupportedAbis) { @@ -310,7 +310,7 @@ public override bool RunTask () var apk = Path.GetFileNameWithoutExtension (ApkOutputPath); ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi), Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)), - debug, compress, compressedAssembliesInfo); + debug, compress, compressedAssembliesInfo, blobApkName: abi); outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi))); } } @@ -322,7 +322,7 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo) + void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string blobApkName) { var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); bool useAssembliesBlob = appConfState != null ? appConfState.UseAssembliesBlob : false; @@ -332,6 +332,7 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); if (blobPaths == null) { throw new InvalidOperationException ("Blob generator did not generate any blobs"); } - if (!blobPaths.TryGetValue (BaseApkName, out List baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) { + if (!blobPaths.TryGetValue (blobApkName, out List baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) { throw new InvalidOperationException ("Blob generator didn't generate the required base blobs"); } - string blobPrefix = $"{BaseApkName}_"; + string blobPrefix = $"{blobApkName}_"; foreach (string blobPath in baseBlobs) { string inArchiveName = Path.GetFileName (blobPath); @@ -445,7 +449,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) } if (useAssembliesBlob) { - blobGenerator.Add (BaseApkName, blobAssembly); + blobGenerator.Add (blobApkName, blobAssembly); } else { count++; if (count >= ZipArchiveEx.ZipFlushFilesLimit) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs index ec519683685..e12617fa18a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs @@ -96,6 +96,17 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo /* expectedRuntime */ "debug", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ false, + /* monoSymbolArchive */ false , + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Full", + /* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo + /* expectedRuntime */ "debug", + /* usesAssemblyBlobs */ true, }, new object[] { /* isRelease */ true, @@ -105,6 +116,17 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ false, + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Full", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, }, new object[] { /* isRelease */ true, @@ -114,6 +136,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -123,6 +146,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -132,6 +156,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -141,15 +166,37 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, /* monoSymbolArchive */ false , + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Portable", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ false , + /* aotAssemblies */ true, + /* debugSymbols */ false, + /* debugType */ "", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ true , /* aotAssemblies */ true, /* debugSymbols */ false, /* debugType */ "", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -159,9 +206,9 @@ public partial class BuildTest : BaseTest /* debugType */ "", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, }, }; #pragma warning restore 414 } } - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 5e2c6c0d600..de0f4732f19 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -98,7 +98,7 @@ public void CheckWhichRuntimeIsIncluded (string supportedAbi, bool debugSymbols, [Category ("AOT"), Category ("MonoSymbolicate")] [TestCaseSource (nameof (SequencePointChecks))] public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive, bool aotAssemblies, - bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime) + bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime, bool usesAssemblyBlobs) { var proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, @@ -109,6 +109,7 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive proj.SetProperty (proj.ActiveConfigurationProperties, "MonoSymbolArchive", monoSymbolArchive); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugSymbols", debugSymbols); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugType", debugType); + proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder ()) { if (aotAssemblies && !b.CrossCompilerAvailable (string.Join (";", abis))) Assert.Ignore ("Cross compiler was not available"); @@ -116,45 +117,42 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var msymarchive = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, proj.PackageName + ".apk.mSYM"); - using (var zipFile = ZipHelper.OpenZip (apk)) { - var mdbExits = ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.dll.mdb") != null || - ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.pdb") != null; - Assert.AreEqual (embedMdb, mdbExits, - $"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not "); - if (aotAssemblies) { - foreach (var abi in abis) { - var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, - "aot", abi, "libaot-UnnamedProject.dll.so"); - var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable"); - var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, - "aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault (); - if (shouldExist) - Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist"); - else - Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile); - if (shouldExist) { - var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any (); - Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive); - } - Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), - $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); - } - } - var runtimeInfo = b.GetSupportedRuntimes (); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + var mdbExits = helper.Exists ("assemblies/UnnamedProject.dll.mdb") || helper.Exists ("assemblies/UnnamedProject.pdb"); + Assert.AreEqual (embedMdb, mdbExits, + $"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not "); + if (aotAssemblies) { foreach (var abi in abis) { - var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime); - Assert.IsNotNull (runtime, "Could not find the expected runtime."); - var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name)); - var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length); - Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used."); - Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime); + var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, + "aot", abi, "libaot-UnnamedProject.dll.so"); + var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable"); + var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, + "aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault (); + if (shouldExist) + Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist"); + else + Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile); + if (shouldExist) { + var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any (); + Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive); + } + Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); + Assert.IsTrue (helper.Exists ($"lib/{abi}/libaot-UnnamedProject.dll.so"), + $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); + Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), + $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); } } + var runtimeInfo = b.GetSupportedRuntimes (); + foreach (var abi in abis) { + var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime); + Assert.IsNotNull (runtime, "Could not find the expected runtime."); + var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name)); + var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length); + Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used."); + Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime); + } + b.Clean (proj); Assert.IsTrue (!Directory.Exists (msymarchive), "{0} should have been deleted on Clean", msymarchive); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index c99b9ecedef..40838f70c47 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -23,6 +23,7 @@ class ArchiveAssemblyHelper readonly string archivePath; readonly string assembliesRootDir; bool useAssemblyBlobs; + List archiveContents; public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) { @@ -45,8 +46,12 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) } } - public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix) + public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix, bool forceRefresh = false) { + if (!forceRefresh && archiveContents != null) { + return archiveContents; + } + if (String.IsNullOrEmpty (blobEntryPrefix)) { throw new ArgumentException (nameof (blobEntryPrefix), "must not be null or empty"); } @@ -58,6 +63,7 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } } + archiveContents = entries; if (!useAssemblyBlobs) { Console.WriteLine ("Not using assembly blobs"); return entries; @@ -81,6 +87,16 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt return entries; } + public bool Exists (string entryPath) + { + List contents = ListArchiveContents (assembliesRootDir); + if (contents.Count == 0) { + return false; + } + + return contents.Contains (entryPath); + } + public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) { if (fileNames == null) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 5055eec1c5a..dcc8532e4cb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -28,7 +28,7 @@ abstract class AssemblyBlob protected const string BlobExtension = ".blob"; static readonly ArrayPool bytePool = ArrayPool.Shared; - static uint id = 0; + static readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); protected static uint globalAssemblyIndex = 0; @@ -52,7 +52,10 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg } // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - ID = id++; + if (!apkIds.ContainsKey (apkName)) { + apkIds.Add (apkName, 0); + } + ID = apkIds[apkName]++; this.archiveAssembliesPrefix = archiveAssembliesPrefix; ApkName = apkName; From bdcfc8958d16798d67ef24e9c73dd4dae6939acb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 29 Sep 2021 22:54:59 +0200 Subject: [PATCH 19/54] [WIP] Another set of test fixes --- .../Tasks/LinkerTests.cs | 7 +- .../Utilities/ArchiveAssemblyHelper.cs | 36 +++++++--- .../Utilities/AssertionExtensions.cs | 22 +++++++ .../Xamarin.Android.Build.Tests/XASdkTests.cs | 65 +++++++++++++++---- 4 files changed, 108 insertions(+), 22 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 217c5d2ebaf..a57e4d430bd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -211,20 +211,23 @@ public void WarnAboutAppDomains ([Values (true, false)] bool isRelease) } [Test] - public void RemoveDesigner () + public void RemoveDesigner ([Values (true, false)] bool usesAssemblyBlobs) { var proj = new XamarinAndroidApplicationProject { IsRelease = true, }; proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); proj.SetProperty ("AndroidLinkResources", "True"); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); string assemblyName = proj.ProjectName; using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); + // TODO: helper must be able to extract assembly from the blob using (var zip = ZipHelper.OpenZip (apk)) { - Assert.IsTrue (zip.ContainsEntry ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); var entry = zip.ReadEntry ($"assemblies/{assemblyName}.dll"); using (var stream = new MemoryStream ()) { entry.Extract (stream); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 40838f70c47..5bc9e160ac8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -9,7 +9,7 @@ namespace Xamarin.Android.Build.Tests { - class ArchiveAssemblyHelper + public class ArchiveAssemblyHelper { public const string DefaultBlobEntryPrefix = "{blob}"; @@ -20,11 +20,20 @@ class ArchiveAssemblyHelper ".mdb", }; + static readonly Dictionary ArchToAbi = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"x86", "x86"}, + {"x86_64", "x86_64"}, + {"armeabi_v7a", "armeabi-v7a"}, + {"arm64_v8a", "arm64-v8a"}, + }; + readonly string archivePath; readonly string assembliesRootDir; bool useAssemblyBlobs; List archiveContents; + public string ArchivePath => archivePath; + public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) { if (String.IsNullOrEmpty (archivePath)) { @@ -70,26 +79,35 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } var explorer = new BlobExplorer (archivePath); - foreach (var kvp in explorer.AssembliesByName) { - string name = kvp.Key; - BlobAssembly asm = kvp.Value; + foreach (var asm in explorer.Assemblies) { + string prefix = blobEntryPrefix; - entries.Add ($"{blobEntryPrefix}{name}.dll"); + if (!String.IsNullOrEmpty (asm.Blob.Arch)) { + string arch = ArchToAbi[asm.Blob.Arch]; + prefix = $"{prefix}{arch}/"; + } + + entries.Add ($"{prefix}{asm.Name}.dll"); if (asm.DebugDataOffset > 0) { - entries.Add ($"{blobEntryPrefix}{name}.pdb"); + entries.Add ($"{prefix}{asm.Name}.pdb"); } if (asm.ConfigDataOffset > 0) { - entries.Add ($"{blobEntryPrefix}{name}.dll.config"); + entries.Add ($"{prefix}{asm.Name}.dll.config"); } } + Console.WriteLine ("Archive entries with synthetised assembly blob entries:"); + foreach (string e in entries) { + Console.WriteLine ($" {e}"); + } + return entries; } - public bool Exists (string entryPath) + public bool Exists (string entryPath, bool forceRefresh = false) { - List contents = ListArchiveContents (assembliesRootDir); + List contents = ListArchiveContents (assembliesRootDir, forceRefresh); if (contents.Count == 0) { return false; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs index 740bb880c4e..8552a37a22b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs @@ -60,12 +60,24 @@ public static void AssertContainsEntry (this ZipArchive zip, string zipPath, str Assert.IsTrue (zip.ContainsEntry (archivePath), $"{zipPath} should contain {archivePath}"); } + [DebuggerHidden] + public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, string archivePath) + { + Assert.IsTrue (helper.Exists (archivePath), $"{helper.ArchivePath} should contain {archivePath}"); + } + [DebuggerHidden] public static void AssertDoesNotContainEntry (this ZipArchive zip, string zipPath, string archivePath) { Assert.IsFalse (zip.ContainsEntry (archivePath), $"{zipPath} should *not* contain {archivePath}"); } + [DebuggerHidden] + public static void AssertDoesNotContainEntry (this ArchiveAssemblyHelper helper, string archivePath) + { + Assert.IsFalse (helper.Exists (archivePath), $"{helper.ArchivePath} should *not* contain {archivePath}"); + } + [DebuggerHidden] public static void AssertContainsEntry (this ZipArchive zip, string zipPath, string archivePath, bool shouldContainEntry) { @@ -76,6 +88,16 @@ public static void AssertContainsEntry (this ZipArchive zip, string zipPath, str } } + [DebuggerHidden] + public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, string archivePath, bool shouldContainEntry) + { + if (shouldContainEntry) { + helper.AssertContainsEntry (archivePath); + } else { + helper.AssertDoesNotContainEntry (archivePath); + } + } + [DebuggerHidden] public static void AssertEntryContents (this ZipArchive zip, string zipPath, string archivePath, string contents) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index e8317d89e96..49465ffc193 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -398,63 +398,104 @@ public void DotNetBuildBinding () /* runtimeIdentifiers */ "android-arm", /* isRelease */ false, /* aot */ false, + /* usesAssemblyBlobs */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ false, + /* aot */ false, + /* usesAssemblyBlobs */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyBlobs */ false, }, new object [] { /* runtimeIdentifiers */ "android-x86", /* isRelease */ false, /* aot */ false, + /* usesAssemblyBlobs */ false, }, new object [] { /* runtimeIdentifiers */ "android-x64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyBlobs */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ true, + /* aot */ false, + /* usesAssemblyBlobs */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, + /* usesAssemblyBlobs */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, + /* usesAssemblyBlobs */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ true, + /* aot */ true, + /* usesAssemblyBlobs */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ true, /* aot */ false, + /* usesAssemblyBlobs */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyBlobs */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", + /* isRelease */ false, + /* aot */ false, + /* usesAssemblyBlobs */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86", /* isRelease */ true, /* aot */ false, + /* usesAssemblyBlobs */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, + /* usesAssemblyBlobs */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", + /* isRelease */ true, + /* aot */ false, + /* usesAssemblyBlobs */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ true, + /* usesAssemblyBlobs */ false, }, }; [Test] [Category ("SmokeTests")] [TestCaseSource (nameof (DotNetBuildSource))] - public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) + public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyBlobs) { var proj = new XASdkProject { IsRelease = isRelease, @@ -489,6 +530,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) } }; proj.MainActivity = proj.DefaultMainActivity.Replace (": Activity", ": AndroidX.AppCompat.App.AppCompatActivity"); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); if (aot) { proj.SetProperty ("RunAOTCompilation", "true"); } @@ -577,21 +619,22 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) bool expectEmbeddedAssembies = !(CommercialBuildAvailable && !isRelease); var apkPath = Path.Combine (outputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apkPath); - using (var apk = ZipHelper.OpenZip (apkPath)) { - apk.AssertContainsEntry (apkPath, $"assemblies/{proj.ProjectName}.dll", shouldContainEntry: expectEmbeddedAssembies); - apk.AssertContainsEntry (apkPath, $"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); - apk.AssertContainsEntry (apkPath, $"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); - apk.AssertContainsEntry (apkPath, $"assemblies/es/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); + var helper = new ArchiveAssemblyHelper (apkPath, usesAssemblyBlobs); + helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); + helper.AssertContainsEntry ($"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ($"assemblies/es/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); +// using (var apk = ZipHelper.OpenZip (apkPath)) { foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { - apk.AssertContainsEntry (apkPath, $"lib/{abi}/libmonodroid.so"); - apk.AssertContainsEntry (apkPath, $"lib/{abi}/libmonosgen-2.0.so"); + helper.AssertContainsEntry ($"lib/{abi}/libmonodroid.so"); + helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); if (rids.Length > 1) { - apk.AssertContainsEntry (apkPath, $"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ($"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); } else { - apk.AssertContainsEntry (apkPath, "assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ("assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); } } - } +// } } From 15b937705a1a751ade7814a6f256f37c6b15a37a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Sep 2021 10:34:52 +0200 Subject: [PATCH 20/54] [WIP] More test fixes --- .../Xamarin.Android.Build.Tests/AotTests.cs | 58 ++++++++----- .../Tasks/LinkerTests.cs | 15 ++-- .../Utilities/ArchiveAssemblyHelper.cs | 84 ++++++++++++++++++- 3 files changed, 123 insertions(+), 34 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index bdc9d6faa81..9ff86704647 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -115,50 +115,64 @@ public void BuildBasicApplicationReleaseProfiledAotWithoutDefaultProfile () /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* supportedAbis */ "armeabi-v7a", + /* enableLLVM */ false, + /* expectedResult */ true, + /* usesAssemblyBlobs */ true, }, new object[] { /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, }; [Test] [TestCaseSource (nameof (AotChecks))] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = false, @@ -168,6 +182,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); if (checkMinLlvmPath) { // Set //uses-sdk/@android:minSdkVersion so that LLVM uses the right libc.so @@ -211,13 +226,13 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -234,9 +249,9 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL [TestCaseSource (nameof (AotChecks))] [Category ("Minor"), Category ("MkBundle")] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = true, @@ -246,6 +261,7 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { if (!b.CrossCompilerAvailable (supportedAbis)) Assert.Ignore ("Cross compiler was not available"); @@ -264,13 +280,12 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsFalse (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -381,6 +396,8 @@ public static void Foo () { [Category ("HybridAOT")] public void HybridAOT ([Values ("armeabi-v7a;arm64-v8a", "armeabi-v7a", "arm64-v8a")] string abis) { + // There's no point in testing all of the ABIs with and without assembly blobs, let's test just one of them this way + bool usesAssemblyBlobs = String.Compare ("arm64-v8a", abis, StringComparison.Ordinal) == 0; var proj = new XamarinAndroidApplicationProject () { IsRelease = true, AotAssemblies = true, @@ -388,6 +405,7 @@ public static void Foo () { proj.SetProperty ("AndroidAotMode", "Hybrid"); // So we can use Mono.Cecil to open assemblies directly proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); proj.SetAndroidSupportedAbis (abis); using (var b = CreateApkBuilder ()) { @@ -412,17 +430,15 @@ public static void Foo () { var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{proj.ProjectName}.dll"); - Assert.IsNotNull (entry, $"{proj.ProjectName}.dll should exist in apk!"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); - var method = type.Methods.First (m => m.Name == "OnCreate"); - Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); - } + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ($"assemblies/{proj.ProjectName}.dll"), $"{proj.ProjectName}.dll should exist in apk!"); + + using (var stream = helper.ReadEntry ($"assemblies/{proj.ProjectName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); + var method = type.Methods.First (m => m.Name == "OnCreate"); + Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index a57e4d430bd..e85fdeac118 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -226,16 +226,11 @@ public void RemoveDesigner ([Values (true, false)] bool usesAssemblyBlobs) FileAssert.Exists (apk); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); - // TODO: helper must be able to extract assembly from the blob - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{assemblyName}.dll"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); - Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); - } + using (var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); + Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 5bc9e160ac8..3e5de3a0744 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; @@ -11,7 +12,8 @@ namespace Xamarin.Android.Build.Tests { public class ArchiveAssemblyHelper { - public const string DefaultBlobEntryPrefix = "{blob}"; + public const string DefaultBlobEntryPrefix = "{blobReader}"; + const int BlobReadBufferSize = 8192; static readonly HashSet SpecialExtensions = new HashSet (StringComparer.OrdinalIgnoreCase) { ".dll", @@ -27,6 +29,8 @@ public class ArchiveAssemblyHelper {"arm64_v8a", "arm64-v8a"}, }; + static readonly ArrayPool buffers = ArrayPool.Create (); + readonly string archivePath; readonly string assembliesRootDir; bool useAssemblyBlobs; @@ -55,6 +59,80 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) } } + public Stream ReadEntry (string path) + { + if (useAssemblyBlobs) { + return ReadBlobEntry (path); + } + + return ReadZipEntry (path); + } + + Stream ReadZipEntry (string path) + { + using (var zip = ZipHelper.OpenZip (archivePath)) { + ZipEntry entry = zip.ReadEntry (path); + var ret = new MemoryStream (); + entry.Extract (ret); + ret.Flush (); + return ret; + } + } + + Stream ReadBlobEntry (string path) + { + BlobReader blobReader = null; + BlobAssembly assembly = null; + string name = Path.GetFileNameWithoutExtension (path); + var explorer = new BlobExplorer (archivePath); + + foreach (var asm in explorer.Assemblies) { + if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { + continue; + } + assembly = asm; + blobReader = asm.Blob; + break; + } + + if (blobReader == null) { + Console.WriteLine ($"Blob for entry {path} not found, will try a standard Zip read"); + return ReadZipEntry (path); + } + + string blobEntryName; + if (String.IsNullOrEmpty (blobReader.Arch)) { + blobEntryName = $"{assembliesRootDir}assemblies.blob"; + } else { + blobEntryName = $"{assembliesRootDir}assemblies_{blobReader.Arch}.blob"; + } + + Stream blob = ReadZipEntry (blobEntryName); + if (blob == null) { + Console.WriteLine ($"Blob zip entry {blobEntryName} does not exist"); + return null; + } + + blob.Seek (assembly.DataOffset, SeekOrigin.Begin); + var ret = new MemoryStream (); + byte[] buffer = buffers.Rent (BlobReadBufferSize); + int toRead = (int)assembly.DataSize; + while (toRead > 0) { + int nread = blob.Read (buffer, 0, BlobReadBufferSize); + if (nread <= 0) { + break; + } + + ret.Write (buffer, 0, nread); + toRead -= nread; + } + ret.Flush (); + blob.Dispose (); + buffers.Return (buffer); + + return ret; + } + public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix, bool forceRefresh = false) { if (!forceRefresh && archiveContents != null) { @@ -97,7 +175,7 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } } - Console.WriteLine ("Archive entries with synthetised assembly blob entries:"); + Console.WriteLine ("Archive entries with synthetised assembly blobReader entries:"); foreach (string e in entries) { Console.WriteLine ($" {e}"); } @@ -170,7 +248,7 @@ void BlobContains (string[] fileNames, out List existingFiles, out List< if (explorer.AssembliesByName.Count != 0) { existingFiles.AddRange (blobAssemblies); - // We need to fake config and debug files since they have no named entries in the blob + // We need to fake config and debug files since they have no named entries in the blobReader foreach (string file in configFiles) { BlobAssembly asm = GetBlobAssembly (file); if (asm == null) { From 4f2a1d3a43b2cb4f04ed760bd3953f5f76e011bb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Sep 2021 12:19:17 +0200 Subject: [PATCH 21/54] [WIP] Even more test (and logic) fixes --- .../Utilities/ArchiveAssemblyHelper.cs | 2 +- .../Utilities/ArchAssemblyBlob.cs | 6 ++-- .../Utilities/AssemblyBlob.cs | 15 ++++------ .../Utilities/AssemblyBlobGenerator.cs | 21 ++++++++++++-- .../Utilities/AssemblyBlobGlobalIndex.cs | 29 +++++++++++++++++++ .../Utilities/CommonAssemblyBlob.cs | 4 +-- 6 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 3e5de3a0744..f6566e70747 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -29,7 +29,7 @@ public class ArchiveAssemblyHelper {"arm64_v8a", "arm64-v8a"}, }; - static readonly ArrayPool buffers = ArrayPool.Create (); + static readonly ArrayPool buffers = ArrayPool.Shared; readonly string archivePath; readonly string assembliesRootDir; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index a9d4bf01278..87e2fb6a071 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -12,8 +12,8 @@ class ArchAssemblyBlob : AssemblyBlob readonly Dictionary> assemblies; HashSet seenArchAssemblyNames; - public ArchAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) - : base (apkName, archiveAssembliesPrefix, log) + public ArchAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) + : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) { assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); } @@ -100,7 +100,7 @@ public override void Generate (string outputDirectory, List bytePool = ArrayPool.Shared; - static readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); - - protected static uint globalAssemblyIndex = 0; string archiveAssembliesPrefix; string indexBlobPath; protected string ApkName { get; } protected TaskLoggingHelper Log { get; } + protected AssemblyBlobGlobalIndex GlobalIndexCounter { get; } public uint ID { get; } public bool IsIndexBlob => ID == 0; - protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) + protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) { if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); @@ -51,11 +49,8 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg throw new ArgumentException ("must not be null or empty", nameof (apkName)); } - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - ID = apkIds[apkName]++; + GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); + ID = id; this.archiveAssembliesPrefix = archiveAssembliesPrefix; ApkName = apkName; @@ -317,7 +312,7 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyBlob.Generate for a special case) - var ret = new AssemblyBlobIndexEntry (assemblyName, ID, globalAssemblyIndex++, localBlobIndex) { + var ret = new AssemblyBlobIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { DataOffset = offset, DataSize = size, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs index 37997e07676..fee2fb1ce6a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs @@ -24,6 +24,14 @@ sealed class Blob AssemblyBlob indexBlob; + // IDs must be counted per AssemblyBlobGenerator instance because it's possible that a single build will create more than one instance of the class and each time + // the blobs must be assigned IDs starting from 0, or there will be errors due to "missing" index blob + readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); + + // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** + // blob, thus we need to keep the state here + AssemblyBlobGlobalIndex globalIndexCounter = new AssemblyBlobGlobalIndex (); + public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) { if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { @@ -46,8 +54,8 @@ public void Add (string apkName, BlobAssemblyInfo blobAssembly) Blob blob; if (!blobs.ContainsKey (apkName)) { blob = new Blob { - Common = new CommonAssemblyBlob (apkName, archiveAssembliesPrefix, log), - Arch = new ArchAssemblyBlob (apkName, archiveAssembliesPrefix, log) + Common = new CommonAssemblyBlob (apkName, archiveAssembliesPrefix, log, GetNextBlobID (apkName), globalIndexCounter), + Arch = new ArchAssemblyBlob (apkName, archiveAssembliesPrefix, log, GetNextBlobID (apkName), globalIndexCounter) }; blobs.Add (apkName, blob); @@ -79,6 +87,15 @@ void SetIndexBlob (AssemblyBlob b) } } + uint GetNextBlobID (string apkName) + { + // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock + if (!apkIds.ContainsKey (apkName)) { + apkIds.Add (apkName, 0); + } + return apkIds[apkName]++; + } + public Dictionary> Generate (string outputDirectory) { if (blobs.Count == 0) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs new file mode 100644 index 00000000000..3db2888a8e1 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs @@ -0,0 +1,29 @@ +namespace Xamarin.Android.Tasks +{ + // This class may seem weird, but it's designed with the specific needs of AssemblyBlob instances in mind and also prepared for thread-safe use in the future, should the + // need arise + sealed class AssemblyBlobGlobalIndex + { + uint value = 0; + + public uint Value => value; + + /// + /// Increments the counter and returns its previous value + /// + public uint Increment () + { + uint ret = value++; + return ret; + } + + public void Subtract (uint count) + { + if (value < count) { + return; + } + + value -= count; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs index abf16479399..4a4cddacebd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs @@ -12,8 +12,8 @@ class CommonAssemblyBlob : AssemblyBlob { readonly List assemblies; - public CommonAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log) - : base (apkName, archiveAssembliesPrefix, log) + public CommonAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) + : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) { assemblies = new List (); } From e9bcd9d931fdebe88a0dce922cc712de77ad4c2a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Sep 2021 16:14:58 +0200 Subject: [PATCH 22/54] [WIP] Don't overwrite logs --- tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index e2326d6783a..7018f402f1d 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -613,6 +613,7 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) proj.SetAndroidSupportedAbis (abis); proj.SetProperty (proj.CommonProperties, "UseInterpreter", "True"); builder = CreateApkBuilder (); + builder.BuildLogFile = "install.log"; Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); if (!Builder.UseDotNet) { @@ -627,6 +628,7 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) var logProp = RunAdbCommand ("shell getprop debug.mono.log")?.Trim (); Assert.AreEqual (logProp, "all", "The debug.mono.log prop was not set correctly."); + builder.BuildLogFile = "run.log"; if (CommercialBuildAvailable) Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); else From a7f84602cede2938f193b7498a468db47fdf5a11 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Sep 2021 20:50:13 +0200 Subject: [PATCH 23/54] [WIP] Disable assembly blobs when fastdev is active --- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 2ac049e8ced..ea728eb966c 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,6 +179,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False + False True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> From 284a8f9061bd1ff4ac0d3dafa3b58dd67f61f3d1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 4 Oct 2021 14:53:02 +0200 Subject: [PATCH 24/54] [WIP] Add some debug info to see why some tests using F# fail --- .../Utilities/ArchAssemblyBlob.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs index 87e2fb6a071..9262d123f5f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs @@ -60,18 +60,27 @@ public override void Generate (string outputDirectory, List archAssemblies = kvp.Value; + Log.LogMessage (MessageImportance.Low, $"Blob: assemblies for architecture {abi}:"); // All the architecture blobs must have assemblies in exactly the same order archAssemblies.Sort ((BlobAssemblyInfo a, BlobAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); if (assemblyNames.Count == 0) { + Log.LogMessage (MessageImportance.Low, $" first arch, preparing names list, {archAssemblies.Count} entries"); for (int i = 0; i < archAssemblies.Count; i++) { BlobAssemblyInfo info = archAssemblies[i]; + Log.LogMessage (MessageImportance.Low, $" {Path.GetFileName (info.FilesystemAssemblyPath)}"); assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); } continue; } + Log.LogMessage (MessageImportance.Low, $" {archAssemblies.Count} entries"); + for (int i = 0; i < archAssemblies.Count; i++) { + BlobAssemblyInfo info = archAssemblies[i]; + Log.LogMessage (MessageImportance.Low, $" {Path.GetFileName (info.FilesystemAssemblyPath)}"); + } + if (archAssemblies.Count != assemblyNames.Count) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists"); + throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); } for (int i = 0; i < archAssemblies.Count; i++) { From 09cf6057b49854fe81d4c40a75acf2d71c965931 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Oct 2021 10:38:49 +0200 Subject: [PATCH 25/54] [WIP] Don't add the `Abi` metadata to satellite assemblies Doing so would put them in the arch-specific blob, which is incorrect in and of itself. Also, oddly enough, the assemblies would end up with `Abi` only for the first architecture targetted by the app, which would cause the arch-specific blob generation to fail because of different number of assemblies in different architectures. --- src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index ccdec6cab35..17788900776 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -175,6 +175,12 @@ bool Filter (ITaskItem item) void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol) { var rid = assembly.GetMetadata ("RuntimeIdentifier"); + // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, therefore we need to check it here in + // order to avoid them getting the `Abi` metadata, which would put them in the arch-specific assembly blob. + if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assembly.GetMetadata ("AssetType"), StringComparison.OrdinalIgnoreCase) == 0) { + rid = String.Empty; + } + var abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); if (!string.IsNullOrEmpty (abi)) { string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); From 0cc93b1cbcad539c1dc863e4f279095eb4e20064 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Oct 2021 10:40:59 +0200 Subject: [PATCH 26/54] [WIP] Don't use blobs in tests which run apkdiff Putting assemblies in the blob doesn't affect their size. Forcing individual assembly entries in those tests allows us to not modify apkdiff right now. --- .../Tests/Xamarin.Android.Build.Tests/BuildTest.cs | 1 + .../Tests/Xamarin.Android.Build.Tests/BuildTest2.cs | 1 + .../Tests/Xamarin.Android.Build.Tests/PackagingTest.cs | 1 + .../Droid/Xamarin.Forms.Performance.Integration.Droid.csproj | 1 + 4 files changed, 4 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index de0f4732f19..4edc72324e1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -940,6 +940,7 @@ public void BuildBasicApplicationCheckPdb () var proj = new XamarinAndroidApplicationProject { EmbedAssembliesIntoApk = true, }; + proj.SetProperty ("AndroidUseAssembliesBlob", "False"); using (var b = CreateApkBuilder ()) { var reference = new BuildItem.Reference ("PdbTestLibrary.dll") { WebContentFileNameFromAzure = "PdbTestLibrary.dll" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index f098c481f7e..4488a83b6f0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -77,6 +77,7 @@ public void BuildReleaseArm64 ([Values (false, true)] bool forms) proj.IsRelease = true; proj.SetAndroidSupportedAbis ("arm64-v8a"); proj.SetProperty ("LinkerDumpDependencies", "True"); + proj.SetProperty ("AndroidUseAssembliesBlob", "False"); if (forms) { proj.PackageReferences.Clear (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 9a9ad7e8c0a..25df1c38655 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -638,6 +638,7 @@ protected override void OnResume() }, } }; + app.SetProperty ("AndroidUseAssembliesBlob", "False"); app.MainActivity = @"using System; using Android.App; using Android.Content; diff --git a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj index ed5e8a5dcc2..4500bf8c7b2 100644 --- a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj +++ b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj @@ -20,6 +20,7 @@ armeabi-v7a;x86 arm64-v8a;x86 True + False true <_AndroidCheckedBuild Condition=" '$(UseASAN)' != '' ">asan <_AndroidCheckedBuild Condition=" '$(UseUBSAN)' != '' ">ubsan From 05bd0b4cf8e4265266ec22288bb9d4116a6d37c8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Oct 2021 13:12:12 +0200 Subject: [PATCH 27/54] [WIP] Update BuildReleaseArm64XFormsLegacy.apkdesc --- .../BuildReleaseArm64XFormsLegacy.apkdesc | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc index e7a51b2bc49..2219d8765fe 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc @@ -2,19 +2,19 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3120 + "Size": 3140 }, "assemblies/FormsViewGroup.dll": { "Size": 7200 }, "assemblies/Java.Interop.dll": { - "Size": 68710 + "Size": 68903 }, "assemblies/Mono.Android.dll": { - "Size": 562181 + "Size": 562556 }, "assemblies/Mono.Security.dll": { - "Size": 68422 + "Size": 68421 }, "assemblies/mscorlib.dll": { "Size": 915407 @@ -47,61 +47,61 @@ "Size": 116883 }, "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 7686 + "Size": 7691 }, "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6635 + "Size": 6639 }, "assemblies/Xamarin.AndroidX.AppCompat.dll": { "Size": 125318 }, "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 7354 + "Size": 7357 }, "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 18259 + "Size": 18263 }, "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 131919 + "Size": 131923 }, "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 15417 + "Size": 15413 }, "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 43119 + "Size": 43132 }, "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6704 + "Size": 6705 }, "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 7050 + "Size": 7059 }, "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 7178 + "Size": 7183 }, "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 4860 + "Size": 4864 }, "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 13574 + "Size": 13579 }, "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 102317 + "Size": 102321 }, "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 6262 + "Size": 6270 }, "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 11261 + "Size": 11264 }, "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 19409 + "Size": 19414 }, "assemblies/Xamarin.Forms.Core.dll": { "Size": 524723 }, "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 384855 + "Size": 384866 }, "assemblies/Xamarin.Forms.Platform.dll": { "Size": 56878 @@ -110,10 +110,10 @@ "Size": 55786 }, "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 43488 + "Size": 43494 }, "classes.dex": { - "Size": 3483748 + "Size": 3483824 }, "lib/arm64-v8a/libmono-btls-shared.so": { "Size": 1613872 @@ -122,7 +122,7 @@ "Size": 707024 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 281352 + "Size": 290768 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 4037584 @@ -131,7 +131,7 @@ "Size": 65624 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 140560 + "Size": 142176 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -1883,5 +1883,5 @@ "Size": 341040 } }, - "PackageSize": 9496734 + "PackageSize": 9504926 } \ No newline at end of file From ffdab978abf51c2441981337a7cf16c416e8ba11 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Oct 2021 17:50:16 +0200 Subject: [PATCH 28/54] [WIP] Properly register .dll.config files --- .../Utilities/AssemblyBlob.cs | 15 +++++++++++---- src/monodroid/jni/embedded-assemblies.cc | 6 +++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 531afb0dd49..011fc1e8c7a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -317,13 +317,14 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse DataSize = size, }; - (offset, size) = WriteFile (assembly.DebugInfoPath, false); + (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); if (offset != 0 && size != 0) { ret.DebugDataOffset = offset; ret.DebugDataSize = size; } - (offset, size) = WriteFile (assembly.ConfigPath, false); + // Config files must end with \0 (nul) + (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); if (offset != 0 && size != 0) { ret.ConfigDataOffset = offset; ret.ConfigDataSize = size; @@ -331,7 +332,7 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse return ret; - (uint offset, uint size) WriteFile (string filePath, bool required) + (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) { if (!File.Exists (filePath)) { if (required) { @@ -355,7 +356,13 @@ AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo asse fs.CopyTo (writer.BaseStream); } - return (offset, (uint)fi.Length); + uint length = (uint)fi.Length; + if (appendNul) { + length++; + writer.Write ((byte)0); + } + + return (offset, length); } } } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 3170a898355..dcd0e2677cb 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -421,7 +421,11 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_stringconfig_data_size != 0) { assembly_runtime_info.config_data = rd.data_start + bba->config_data_offset; - mono_register_config_for_assembly (name.get (), reinterpret_cast(assembly_runtime_info.config_data)); + // Mono takes ownership of the pointers + mono_register_config_for_assembly ( + utils.string_concat (name.get (), ".dll"), + utils.strdup_new (reinterpret_cast(assembly_runtime_info.config_data)) + ); } #endif // NET6 From 5a4b53606ac3d9a2722189ab5ba6d39f51e55fa2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Oct 2021 10:22:02 +0200 Subject: [PATCH 29/54] [WIP] Implement some TODOs --- .../Utilities/AssemblyBlob.cs | 26 +++++++++++++++++-- src/monodroid/jni/embedded-assemblies.hh | 9 +++++++ src/monodroid/jni/monodroid-glue.cc | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs index 011fc1e8c7a..46dd2b85001 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs @@ -111,16 +111,28 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List (); manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); - // TODO: check if there are no duplicates here + + var seenHashes32 = new HashSet (); + var seenHashes64 = new HashSet (); + bool haveDuplicates = false; foreach (AssemblyBlobIndexEntry assembly in globalIndex) { if (assembly.BlobID == ID) { localEntryCount++; localAssemblies.Add (assembly); } + if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || + WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { + haveDuplicates = true; + } + manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.BlobID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); } + if (haveDuplicates) { + throw new InvalidOperationException ("Duplicate assemblies encountered"); + } + uint globalAssemblyCount = (uint)globalIndex.Count; Log.LogMessage (MessageImportance.Low, $"Index blob, writing header (local assemblies: {localAssemblies.Count}; global assemblies: {globalIndex.Count})"); @@ -152,6 +164,17 @@ void WriteHash (AssemblyBlobIndexEntry entry, ulong hash) blobWriter.Write (entry.LocalBlobIndex); blobWriter.Write (entry.BlobID); } + + bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) + { + if (seenHashes.Contains (hash)) { + Log.LogMessage (MessageImportance.High, "Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); + return true; + } + + seenHashes.Add (hash); + return false; + } } protected string GetAssemblyName (BlobAssemblyInfo assembly) @@ -180,7 +203,6 @@ protected void Generate (string outputFilePath, List assemblie blobPaths.Add (outputFilePath); Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - // TODO: test with satellite assemblies, their name must include the culture prefix using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index fa844e0bef3..87f334dd5d3 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -146,6 +146,15 @@ namespace xamarin::android::internal { return need_to_scan_more_apks; } + void ensure_valid_blobs () const noexcept + { + if (!application_config.have_assemblies_blob) { + return; + } + + abort_unless (index_blob_header != nullptr && blob_assembly_hashes != nullptr, "Invalid or incomplete assembly blob data"); + } + private: const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid); MonoReflectionType* typemap_java_to_managed (const char *java_type_name); diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 1eb74b5a775..893cd67220d 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -438,7 +438,7 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, } } - // TODO: ensure that we have all we need (blob indexes etc) + embeddedAssemblies.ensure_valid_blobs (); } #if defined (DEBUG) && !defined (WINDOWS) From 8117dc0a94fedea1e6bf852954090a8534615b89 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Oct 2021 15:54:51 +0200 Subject: [PATCH 30/54] [WIP] update another apkdesc --- ...nyname.vsandroidapp-Signed-Release.apkdesc | 78 +++---------------- 1 file changed, 12 insertions(+), 66 deletions(-) diff --git a/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc b/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc index 14987f2e37b..2dafe6e3ccd 100644 --- a/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc +++ b/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc @@ -4,68 +4,14 @@ "AndroidManifest.xml": { "Size": 2896 }, - "assemblies/Java.Interop.dll": { - "Size": 68160 + "assemblies/assemblies.blob": { + "Size": 2026794 }, - "assemblies/Mono.Android.dll": { - "Size": 330421 - }, - "assemblies/Mono.Security.dll": { - "Size": 61400 - }, - "assemblies/mscorlib.dll": { - "Size": 843651 - }, - "assemblies/System.Core.dll": { - "Size": 33884 - }, - "assemblies/System.dll": { - "Size": 208052 - }, - "assemblies/System.Net.Http.dll": { - "Size": 110901 - }, - "assemblies/System.Numerics.dll": { - "Size": 15656 - }, - "assemblies/VSAndroidApp.dll": { - "Size": 60901 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.Common.dll": { - "Size": 6971 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.LiveData.Core.dll": { - "Size": 7017 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.ViewModel.dll": { - "Size": 4779 - }, - "assemblies/Xamarin.Android.Support.Compat.dll": { - "Size": 50728 - }, - "assemblies/Xamarin.Android.Support.CoordinaterLayout.dll": { - "Size": 17769 - }, - "assemblies/Xamarin.Android.Support.Design.dll": { - "Size": 28949 - }, - "assemblies/Xamarin.Android.Support.DrawerLayout.dll": { - "Size": 14618 - }, - "assemblies/Xamarin.Android.Support.Fragment.dll": { - "Size": 41190 - }, - "assemblies/Xamarin.Android.Support.Loader.dll": { - "Size": 13489 - }, - "assemblies/Xamarin.Android.Support.v7.AppCompat.dll": { - "Size": 92177 - }, - "assemblies/Xamarin.Essentials.dll": { - "Size": 14090 + "assemblies/assemblies.manifest": { + "Size": 1574 }, "classes.dex": { - "Size": 2940492 + "Size": 2940564 }, "lib/armeabi-v7a/libmono-btls-shared.so": { "Size": 1112688 @@ -74,7 +20,7 @@ "Size": 736396 }, "lib/armeabi-v7a/libmonodroid.so": { - "Size": 229116 + "Size": 232320 }, "lib/armeabi-v7a/libmonosgen-2.0.so": { "Size": 4456332 @@ -83,7 +29,7 @@ "Size": 48844 }, "lib/armeabi-v7a/libxamarin-app.so": { - "Size": 46912 + "Size": 46832 }, "lib/x86/libmono-btls-shared.so": { "Size": 1459584 @@ -92,7 +38,7 @@ "Size": 803352 }, "lib/x86/libmonodroid.so": { - "Size": 297800 + "Size": 303316 }, "lib/x86/libmonosgen-2.0.so": { "Size": 4212220 @@ -101,7 +47,7 @@ "Size": 61112 }, "lib/x86/libxamarin-app.so": { - "Size": 45580 + "Size": 45500 }, "META-INF/android.arch.core_runtime.version": { "Size": 6 @@ -125,7 +71,7 @@ "Size": 1213 }, "META-INF/ANDROIDD.SF": { - "Size": 67139 + "Size": 65121 }, "META-INF/androidx.appcompat_appcompat.version": { "Size": 6 @@ -206,7 +152,7 @@ "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 67012 + "Size": 64994 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 308 @@ -1664,5 +1610,5 @@ "Size": 320016 } }, - "PackageSize": 9498135 + "PackageSize": 9500681 } \ No newline at end of file From 25cf170ad3f5b8d218fd8ba3a77819877627f5a2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Oct 2021 20:27:40 +0200 Subject: [PATCH 31/54] [WIP] A handful of updates --- .../Tasks/BuildApk.cs | 2 +- .../Tasks/GeneratePackageManagerJava.cs | 11 ++-- tools/assembly-blob-reader/BlobExplorer.cs | 1 + .../Directory.Build.targets | 6 ++ tools/assembly-blob-reader/Program.cs | 46 +++++++++++++- .../assembly-blob-reader.csproj | 2 +- .../decompress-assemblies.csproj | 4 ++ tools/decompress-assemblies/main.cs | 61 ++++++++++++++----- tools/scripts/read-blob | 10 +++ 9 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 tools/assembly-blob-reader/Directory.Build.targets create mode 100755 tools/scripts/read-blob diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index a793a86a650..f12cf2d1c01 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -698,7 +698,7 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log); if (ndk == null) { - return; // NdkTools.Create will Log appropriate error + return; // NdkTools.Create will log appropriate error } string clangDir = ndk.GetClangDeviceLibraryPath (); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 059f660d22d..59e56dde015 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -287,8 +287,6 @@ void AddEnvironment () }; int assemblyCount = 0; - int blobCommonAssemblyCount = 0; - int blobArchAssemblyCount = 0; HashSet archAssemblyNames = null; Action updateAssemblyCount = (ITaskItem assembly) => { @@ -301,9 +299,7 @@ void AddEnvironment () if (String.IsNullOrEmpty (abi)) { assemblyCount++; } else { - if (archAssemblyNames == null) { - archAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); - } + archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); string assemblyName = Path.GetFileName (assembly.ItemSpec); if (!archAssemblyNames.Contains (assemblyName)) { @@ -362,7 +358,10 @@ void AddEnvironment () HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, - NumberOfAssemblyBlobsInApk = 2, // Until feature APKs are a thing, we're going to have just two blobs in each app + NumberOfAssemblyBlobsInApk = 2, // Until feature APKs are a thing, we're going to have just two blobs in each app - one for arch-agnostic + // and up to 4 other for arch-specific assemblies. Only **one** arch-specific blob is ever loaded on the app + // runtime, thus the number 2 here. All architecture specific blobs contain assemblies with the same names + // and in the same order. HaveAssembliesBlob = UseAssembliesBlob, }; diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index 0e9c1c4311a..5d454948192 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -53,6 +53,7 @@ public BlobExplorer (string blobPath, Action? cust throw new ArgumentException ($"'{blobPath}' points to a directory", nameof (blobPath)); } + logger = customLogger; BlobPath = blobPath; string? extension = Path.GetExtension (blobPath); string? baseName = null; diff --git a/tools/assembly-blob-reader/Directory.Build.targets b/tools/assembly-blob-reader/Directory.Build.targets new file mode 100644 index 00000000000..ec0761f8371 --- /dev/null +++ b/tools/assembly-blob-reader/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/assembly-blob-reader/Program.cs b/tools/assembly-blob-reader/Program.cs index 82f10371acd..83717b6b678 100644 --- a/tools/assembly-blob-reader/Program.cs +++ b/tools/assembly-blob-reader/Program.cs @@ -7,9 +7,9 @@ namespace Xamarin.Android.AssemblyBlobReader { class Program { - static void Main (string[] args) + static void ShowBlobInfo (string blobPath) { - var explorer = new BlobExplorer (args[0]); + var explorer = new BlobExplorer (blobPath); string yesno = explorer.IsCompleteSet ? "yes" : "no"; Console.WriteLine ($"Blob set '{explorer.BlobSetName}':"); @@ -70,6 +70,48 @@ void WriteHashValue (ulong hash) } } + static int Main (string[] args) + { + if (args.Length == 0) { + Console.Error.WriteLine ("Usage: read-blob BLOB_PATH [BLOB_PATH ...]"); + Console.Error.WriteLine (); + Console.Error.WriteLine (@" where each BLOB_PATH can point to: + * aab file + * apk file + * index blob file (e.g. base_assemblies.blob) + * arch blob file (e.g. base_assemblies.arm64_v8a.blob) + * blob manifest file (e.g. base_assemblies.manifest) + * blob base name (e.g. base or base_assemblies) + + In each case the whole set of blobs and manifests will be read (if available). Search for the + various members of the blob set (common/main blob, arch blobs, manifest) is based on this naming + convention: + + {BASE_NAME}[.ARCH_NAME].{blob|manifest} + + Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read. + If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies` + +"); + return 1; + } + + bool first = true; + foreach (string path in args) { + ShowBlobInfo (path); + if (first) { + first = false; + continue; + } + + Console.WriteLine (); + Console.WriteLine ("***********************************"); + Console.WriteLine (); + } + + return 0; + } + static void WriteAssemblySegment (string label, uint offset, uint size) { if (offset == 0) { diff --git a/tools/assembly-blob-reader/assembly-blob-reader.csproj b/tools/assembly-blob-reader/assembly-blob-reader.csproj index b2098204f66..be33b8df2eb 100644 --- a/tools/assembly-blob-reader/assembly-blob-reader.csproj +++ b/tools/assembly-blob-reader/assembly-blob-reader.csproj @@ -3,7 +3,7 @@ Microsoft Corporation - 2020 Microsoft Corporation + 2021 Microsoft Corporation 0.0.1 false ../../bin/$(Configuration)/bin/assembly-blob-reader diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index e79f0d93510..311920b146c 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -16,6 +16,10 @@ + + + + diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index 749c3a691db..8c4e0287939 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -4,6 +4,7 @@ using K4os.Compression.LZ4; using Xamarin.Tools.Zip; +using Xamarin.Android.AssemblyBlobReader; namespace Xamarin.Android.Tools.DecompressAssemblies { @@ -17,7 +18,7 @@ static int Usage () { Console.WriteLine ("Usage: decompress-assemblies {file.{dll,apk,aab}} [{file.{dll,apk,aab} ...]"); Console.WriteLine (); - Console.WriteLine ("DLL files passed on command are uncompressed to the current directory with the `uncompressed-` prefix added to their name."); + Console.WriteLine ("DLL files passed on command line are uncompressed to the current directory with the `uncompressed-` prefix added to their name."); Console.WriteLine ("DLL files from AAB/APK archives are uncompressed to a subdirectory of the current directory named after the archive with extension removed"); return 1; } @@ -78,30 +79,58 @@ static bool UncompressDLL (string filePath, string prefix) } } - static bool UncompressFromAPK (string filePath, string assembliesPath) + static bool UncompressFromAPK_IndividualEntries (ZipArchive apk, string filePath, string assembliesPath, string prefix) { - string prefix = $"uncompressed-{Path.GetFileNameWithoutExtension (filePath)}{Path.DirectorySeparatorChar}"; - using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { - foreach (ZipEntry entry in apk) { - if (!entry.FullName.StartsWith (assembliesPath, StringComparison.Ordinal)) { - continue; - } + foreach (ZipEntry entry in apk) { + if (!entry.FullName.StartsWith (assembliesPath, StringComparison.Ordinal)) { + continue; + } - if (!entry.FullName.EndsWith (".dll", StringComparison.Ordinal)) { - continue; - } + if (!entry.FullName.EndsWith (".dll", StringComparison.Ordinal)) { + continue; + } - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Seek (0, SeekOrigin.Begin); - UncompressDLL (stream, $"{filePath}!{entry.FullName}", entry.FullName, prefix); - } + using (var stream = new MemoryStream ()) { + entry.Extract (stream); + stream.Seek (0, SeekOrigin.Begin); + UncompressDLL (stream, $"{filePath}!{entry.FullName}", entry.FullName, prefix); } } return true; } + static bool UncompressFromAPK_Blobs (string filePath, string prefix) + { + var explorer = new BlobExplorer (filePath); + foreach (BlobAssembly assembly in explorer.Assemblies) { + string assemblyName = assembly.Name; + if (String.IsNullOrEmpty (assemblyName)) { + assemblyName = $"{assembly.Hash32:x}_{assembly.Hash64:x}.dll"; + } + + if (!String.IsNullOrEmpty (assembly.Blob.Arch)) { + assemblyName = $"{assembly.Blob.Arch}/{assemblyName}"; + } + + throw new NotImplementedException (); + } + + return true; + } + + static bool UncompressFromAPK (string filePath, string assembliesPath) + { + string prefix = $"uncompressed-{Path.GetFileNameWithoutExtension (filePath)}{Path.DirectorySeparatorChar}"; + using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { + if (!apk.ContainsEntry ($"{assembliesPath}/assemblies.blob")) { + return UncompressFromAPK_IndividualEntries (apk, filePath, assembliesPath, prefix); + } + } + + return UncompressFromAPK_Blobs (filePath, prefix); + } + static int Main (string[] args) { if (args.Length == 0) { diff --git a/tools/scripts/read-blob b/tools/scripts/read-blob new file mode 100755 index 00000000000..ca2f7580c82 --- /dev/null +++ b/tools/scripts/read-blob @@ -0,0 +1,10 @@ +#!/bin/bash +truepath=$(readlink "$0" || echo "$0") +mydir=$(dirname ${truepath}) +binariesdir="${mydir}/assembly-blob-reader" + +if [ -x "${binariesdir}/tmt" ]; then + exec "${binariesdir}/assembly-blob-reader" "$@" +else + exec dotnet "${binariesdir}/assembly-blob-reader.dll" +fi From 33aa5c7eacc55a6325b0e12135cf66098da32fcf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Oct 2021 22:09:09 +0200 Subject: [PATCH 32/54] [WIP] Update assembly decompressor to read blobs --- tools/assembly-blob-reader/BlobAssembly.cs | 53 ++++++++++++ tools/assembly-blob-reader/BlobExplorer.cs | 8 +- tools/assembly-blob-reader/BlobReader.cs | 93 +++++++++++++++++++++- tools/decompress-assemblies/main.cs | 22 ++--- 4 files changed, 162 insertions(+), 14 deletions(-) diff --git a/tools/assembly-blob-reader/BlobAssembly.cs b/tools/assembly-blob-reader/BlobAssembly.cs index 9b87619bb70..2a6fdf836b0 100644 --- a/tools/assembly-blob-reader/BlobAssembly.cs +++ b/tools/assembly-blob-reader/BlobAssembly.cs @@ -18,6 +18,9 @@ class BlobAssembly public uint RuntimeIndex { get; set; } public BlobReader Blob { get; } + public string DllName => MakeFileName ("dll"); + public string PdbName => MakeFileName ("pdb"); + public string ConfigName => MakeFileName ("dll.config"); internal BlobAssembly (BinaryReader reader, BlobReader blob) { @@ -30,5 +33,55 @@ internal BlobAssembly (BinaryReader reader, BlobReader blob) ConfigDataOffset = reader.ReadUInt32 (); ConfigDataSize = reader.ReadUInt32 (); } + + public void ExtractImage (string outputDirPath, string? fileName = null) + { + Blob.ExtractAssemblyImage (this, MakeOutputFilePath (outputDirPath, "dll", fileName)); + } + + public void ExtractImage (Stream output) + { + Blob.ExtractAssemblyImage (this, output); + } + + public void ExtractDebugData (string outputDirPath, string? fileName = null) + { + Blob.ExtractAssemblyDebugData (this, MakeOutputFilePath (outputDirPath, "pdb", fileName)); + } + + public void ExtractDebugData (Stream output) + { + Blob.ExtractAssemblyDebugData (this, output); + } + + public void ExtractConfig (string outputDirPath, string? fileName = null) + { + Blob.ExtractAssemblyConfig (this, MakeOutputFilePath (outputDirPath, "dll.config", fileName)); + } + + public void ExtractConfig (Stream output) + { + Blob.ExtractAssemblyConfig (this, output); + } + + string MakeOutputFilePath (string outputDirPath, string extension, string? fileName) + { + return Path.Combine (outputDirPath, MakeFileName (extension, fileName)); + } + + string MakeFileName (string extension, string? fileName = null) + { + if (String.IsNullOrEmpty (fileName)) { + fileName = Name; + + if (String.IsNullOrEmpty (fileName)) { + fileName = $"{Hash32:x}_{Hash64:x}"; + } + + fileName = $"{fileName}.{extension}"; + } + + return fileName!; + } } } diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index 5d454948192..37574173c18 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -12,6 +12,7 @@ class BlobExplorer BlobManifestReader? manifest; int numberOfBlobs = 0; Action? logger; + bool keepBlobInMemory; public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); public IDictionary AssembliesByHash32 { get; } = new Dictionary (); @@ -43,7 +44,7 @@ class BlobExplorer // Whichever file is referenced in `blobPath`, the BASE_NAME component is extracted and all the found files are read. // If `blobPath` points to an aab or an apk, BASE_NAME will always be `assemblies` // - public BlobExplorer (string blobPath, Action? customLogger = null) + public BlobExplorer (string blobPath, Action? customLogger = null, bool keepBlobInMemory = false) { if (String.IsNullOrEmpty (blobPath)) { throw new ArgumentException ("must not be null or empty", nameof (blobPath)); @@ -54,6 +55,7 @@ public BlobExplorer (string blobPath, Action? cust } logger = customLogger; + this.keepBlobInMemory = keepBlobInMemory; BlobPath = blobPath; string? extension = Path.GetExtension (blobPath); string? baseName = null; @@ -216,7 +218,7 @@ void ReadBlobSetFromArchive (ZipArchive archive, string basePathInArchive) entry.Extract (stream); if (entry.FullName.EndsWith (".blob", StringComparison.Ordinal)) { - AddBlob (new BlobReader (stream, GetBlobArch (entry.FullName))); + AddBlob (new BlobReader (stream, GetBlobArch (entry.FullName), keepBlobInMemory)); } else if (entry.FullName.EndsWith (".manifest", StringComparison.Ordinal)) { manifest = new BlobManifestReader (stream); } @@ -287,7 +289,7 @@ BlobManifestReader ReadManifest (string filePath) BlobReader CreateBlobReader (Stream input, string? arch) { numberOfBlobs++; - return new BlobReader (input, arch); + return new BlobReader (input, arch, keepBlobInMemory); } bool IsAndroidArchive (string extension) diff --git a/tools/assembly-blob-reader/BlobReader.cs b/tools/assembly-blob-reader/BlobReader.cs index 86fc8da714c..da9680ecddc 100644 --- a/tools/assembly-blob-reader/BlobReader.cs +++ b/tools/assembly-blob-reader/BlobReader.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Text; @@ -11,6 +12,8 @@ class BlobReader const uint BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian const uint BUNDLED_ASSEMBLIES_BLOB_VERSION = 1; // The highest format version this reader understands + MemoryStream? blobData; + public uint Version { get; private set; } public uint LocalEntryCount { get; private set; } public uint GlobalEntryCount { get; private set; } @@ -22,11 +25,18 @@ class BlobReader public bool HasGlobalIndex => BlobID == 0; - public BlobReader (Stream blob, string? arch = null) + public BlobReader (Stream blob, string? arch = null, bool keepBlobInMemory = false) { Arch = arch ?? String.Empty; blob.Seek (0, SeekOrigin.Begin); + if (keepBlobInMemory) { + blobData = new MemoryStream (); + blob.CopyTo (blobData); + blobData.Flush (); + blob.Seek (0, SeekOrigin.Begin); + } + using (var reader = new BinaryReader (blob, Encoding.UTF8, leaveOpen: true)) { ReadHeader (reader); @@ -38,6 +48,87 @@ public BlobReader (Stream blob, string? arch = null) } } + internal void ExtractAssemblyImage (BlobAssembly assembly, string outputFilePath) + { + SaveDataToFile (outputFilePath, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyImage (BlobAssembly assembly, Stream output) + { + SaveDataToStream (output, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyDebugData (BlobAssembly assembly, string outputFilePath) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToFile (outputFilePath, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyDebugData (BlobAssembly assembly, Stream output) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyConfig (BlobAssembly assembly, string outputFilePath) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + + SaveDataToFile (outputFilePath, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + internal void ExtractAssemblyConfig (BlobAssembly assembly, Stream output) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + void SaveDataToFile (string outputFilePath, uint offset, uint size) + { + EnsureBlobDataAvailable (); + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + SaveDataToStream (fs, offset, size); + } + } + + void SaveDataToStream (Stream output, uint offset, uint size) + { + EnsureBlobDataAvailable (); + ArrayPool pool = ArrayPool.Shared; + + blobData!.Seek (offset, SeekOrigin.Begin); + byte[] buf = pool.Rent (16384); + int nread; + long toRead = size; + while (toRead > 0 && (nread = blobData.Read (buf, 0, buf.Length)) > 0) { + if (nread > toRead) { + nread = (int)toRead; + } + + output.Write (buf, 0, nread); + toRead -= nread; + } + output.Flush (); + pool.Return (buf); + } + + void EnsureBlobDataAvailable () + { + if (blobData != null) { + return; + } + + throw new InvalidOperationException ("Blob data not available. BlobReader/BlobExplorer must be instantiated with the `keepBlobInMemory` argument set to `true`"); + } + public bool HasIdenticalContent (BlobReader other) { return diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index 8c4e0287939..e7a2074767d 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -25,7 +25,7 @@ static int Usage () static bool UncompressDLL (Stream inputStream, string fileName, string filePath, string prefix) { - string outputFile = $"{prefix}{Path.GetFileName (filePath)}"; + string outputFile = $"{prefix}{filePath}"; bool retVal = true; Console.WriteLine ($"Processing {fileName}"); @@ -75,7 +75,7 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, static bool UncompressDLL (string filePath, string prefix) { using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read)) { - return UncompressDLL (fs, filePath, filePath, prefix); + return UncompressDLL (fs, filePath, Path.GetFileName (filePath), prefix); } } @@ -93,7 +93,8 @@ static bool UncompressFromAPK_IndividualEntries (ZipArchive apk, string filePath using (var stream = new MemoryStream ()) { entry.Extract (stream); stream.Seek (0, SeekOrigin.Begin); - UncompressDLL (stream, $"{filePath}!{entry.FullName}", entry.FullName, prefix); + string fileName = entry.FullName.Substring (assembliesPath.Length); + UncompressDLL (stream, $"{filePath}!{entry.FullName}", fileName, prefix); } } @@ -102,18 +103,19 @@ static bool UncompressFromAPK_IndividualEntries (ZipArchive apk, string filePath static bool UncompressFromAPK_Blobs (string filePath, string prefix) { - var explorer = new BlobExplorer (filePath); + var explorer = new BlobExplorer (filePath, keepBlobInMemory: true); foreach (BlobAssembly assembly in explorer.Assemblies) { - string assemblyName = assembly.Name; - if (String.IsNullOrEmpty (assemblyName)) { - assemblyName = $"{assembly.Hash32:x}_{assembly.Hash64:x}.dll"; - } + string assemblyName = assembly.DllName; if (!String.IsNullOrEmpty (assembly.Blob.Arch)) { assemblyName = $"{assembly.Blob.Arch}/{assemblyName}"; } - throw new NotImplementedException (); + using (var stream = new MemoryStream ()) { + assembly.ExtractImage (stream); + stream.Seek (0, SeekOrigin.Begin); + UncompressDLL (stream, $"{filePath}!{assemblyName}", assemblyName, prefix); + } } return true; @@ -123,7 +125,7 @@ static bool UncompressFromAPK (string filePath, string assembliesPath) { string prefix = $"uncompressed-{Path.GetFileNameWithoutExtension (filePath)}{Path.DirectorySeparatorChar}"; using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { - if (!apk.ContainsEntry ($"{assembliesPath}/assemblies.blob")) { + if (!apk.ContainsEntry ($"{assembliesPath}assemblies.blob")) { return UncompressFromAPK_IndividualEntries (apk, filePath, assembliesPath, prefix); } } From c1110013ff40becf01b7176834be9c252a73fedf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 7 Oct 2021 14:21:54 +0200 Subject: [PATCH 33/54] [WIP] Add blob format documentation --- Documentation/README.md | 1 + Documentation/project-docs/AssemblyBlobs.md | 214 ++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 Documentation/project-docs/AssemblyBlobs.md diff --git a/Documentation/README.md b/Documentation/README.md index fef9409c8d9..cbbf682b0fc 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -11,6 +11,7 @@ * [Submitting Bugs, Feature Requests, and Pull Requests][bugs] * [Directory Structure](project-docs/ExploringSources.md) + * [Assembly blob format](project-docs/AssemblyBlobs.md) [bugs]: https://github.com/xamarin/xamarin-android/wiki/Submitting-Bugs,-Feature-Requests,-and-Pull-Requests diff --git a/Documentation/project-docs/AssemblyBlobs.md b/Documentation/project-docs/AssemblyBlobs.md new file mode 100644 index 00000000000..8ef1223d232 --- /dev/null +++ b/Documentation/project-docs/AssemblyBlobs.md @@ -0,0 +1,214 @@ + +**Table of Contents** + +- [Assembly Blobs format and purpose](#assembly-blobs-format-and-purpose) + - [Rationale](#rationale) +- [Blob kinds and locations](#blob-kinds-and-locations) +- [Blob format](#blob-format) + - [Common header](#common-header) + - [Assembly descriptor table](#assembly-descriptor-table) + - [Index blob](#index-blob) + - [Hash table format](#hash-table-format) + + + +# Assembly Blobs format and purpose + +Assembly blobs are binary files which contain inside them the managed +assemblies, their debug data (optionally) and the associated config +file (optionally). They are placed inside the Android APK/AAB +archives, replacing individual assemblies/pdb/config files. + +Blobs are an optional form of assembly storage in the archive, they +can be used in all build configurations **except** when Fast +Deployment is in effect (in which case assemblies aren't placed in the +archives at all, they are instead synchronized from the host to the +device/emulator filesystem) + +## Rationale + +During native startup, the Xamarin.Android runtime looks inside the +application APK file for the managed assemblies (and their associated +pdb and config files, if applicable) in order to map them (using the +`mmap(2)` call) into memory so that they can be given to the Mono +runtime when it requests a given assembly is loaded. The reason for +the memory mapping is that, as far as Android is concerned, managed +assembly files are just data/resources and, thus, aren't extracted to +the filesystem. As a result, Mono wouldn't be able to find the +assemblies by scanning the filesystem - the host application +(Xamarin.Android) must give it a hand in finding them. + +Applications can contain hundreds of assemblies (for instance a Hello +World MAUI application currently contains over 120 assemblies) and +each of them would have to be mmapped at startup, together with its +pdb and config files, if found. This not only costs time (each `mmap` +invocation is a system call) but it also makes the assembly discovery +an O(n) algorithm, which takes more time as more assemblies are added +to the APK/AAB archive. + +An assembly blob, however, needs to be mapped only once and any +further operations are merely pointer arithmetic, making the process +not only faster but also reducing the algorithm complexity to O(1). + +# Blob kinds and locations + +Each application will contain at least a single blob, with assemblies +that are architecture-agnostics and any number of +architecture-specific blobs. dotnet ships with a handful of +assemblies that **are** architecture-specific - those assemblies are +placed in an architecture specific blob, one per architecture +supported by and enabled for the application. On the execution time, +the Xamarin.Android runtime will always map the architecture-agnostic +blob and one, and **only** one, of the architecture-specific blobs. + +Blobs are placed in the same location in the APK/AAB archive where the +individual assemblies traditionally live, the `assemblies/` (for APK) +and `base/root/assemblies/` (for AAB) folders. + +The architecture agnostic blob is always named `assemblies.blob` while +the architecture-specific one is called `assemblies.[ARCH].blob`. + +Currently, Xamarin.Android applications will produce only one set of +blobs but when Xamarin.Android adds support for Android Features, each +feature APK will contain its own set of blobs. All of the APKs will +follow the location, format and naming conventions described above. + +# Blob format + +Each blob is a structured binary file, using little-endian byte order +and aligned to a byte boundary. Each blob consists of a header, an +assembly descriptor table and, optionally (see below), two tables with +assembly name hashes. All the blobs are assigned a unique ID, with +the blob having ID equal to `0` being the [Index blob](#index-blob) + +Assemblies are stored as adjacent byte streams: + + - **Image data** + Required to be present for all assemblies, contains the actual + assembly PE image. + - **Debug data** + Optional. Contains the assembly's PDB or MDB debug data. + - **Config data** + Optional. Contains the assembly's .config file. Config data + **must** be terminated with a `NUL` character (`0`), this is to + make runtime code slightly more efficient. + +All the structures described here are defined in the +[`xamarin-app.hh`](../../src/monodroid/jni/xamarin-app.hh) file. +Should there be any difference between this document and the +structures in the header file, the information from the header is the +one that should be trusted. + +## Common header + +All kinds of blobs share the following header format: + + struct BundledAssemblyBlobHeader + { + uint32_t magic; + uint32_t version; + uint32_t local_entry_count; + uint32_t global_entry_count; + uint32_t blob_id; + ; + +Individual fields have the following meanings: + + - `magic`: has the value of 0x41424158 (`XABA`) + - `version`: a value increased every time blob format changes. + - `local_entry_count`: number of assemblies stored in this blob (also + the number of entries in the assembly descriptor table, see below) + - `global_entry_count`: number of entries in the index blob's (see + below) hash tables, all the other blobs store `0` in this field + - `blob_id`: a unique ID of this blob. + +## Assembly descriptor table + +Each blob header is followed by a table of +`BundledAssemblyBlobHeader.local_entry_count` entries, each entry +defined by the following structure: + + struct BlobBundledAssembly + { + uint32_t data_offset; + uint32_t data_size; + uint32_t debug_data_offset; + uint32_t debug_data_size; + uint32_t config_data_offset; + uint32_t config_data_size; + }; + +Only the `data_offset` and `data_size` fields must have a non-zero +value, other fields describe optional data and can be set to `0`. + +Individual fields have the following meanings: + + - `data_offset`: offset of the assembly image data from the + beginning of the blob file + - `data_size`: number of bytes of the image data + - `debug_data_offset`: offset of the assembly's debug data from the + beginning of the blob file. A value of `0` indicates there's no + debug data for this assembly. + - `debug_data_size`: number of bytes of debug data. Can be `0` only + if `debug_data_offset` is `0` + - `config_data_offset`: offset of the assembly's config file data + from the beginning of the blob file. A value of `0` indicates + there's no config file data for this assembly. + - `config_data_size`: number of bytes of config file data. Can be + `0` only if `config_data_offset` is `0` + +## Index blob + +Each application will contain exactly one blob with a global index - +two tables with assembly name hashes. All the other blobs **do not** +contain these tables. Two hash tables are necessary because hashes +for 32-bit and 64-bit devices are different. + +The hash tables follow the [Assembly descriptor +table](#assembly-descriptor-table) and precede the individual assembly +streams. + +Placing the hash tables in a single index blob, while "wasting" a +certain amount of memory (since 32-bit devices won't use the 64-bit +table and vice versa), makes for simpler and faster runtime +implementation and the amount of memory wasted isn't big (1000 +two tables which are 8kb long each, this being the amount of memory +wasted) + +### Hash table format + +Both tables share the same format, despite the hashes themselves being +of different sizes. This is done to make handling of the tables +easier on the runtime. + +Each entry contains, among other fields, the assembly name hash. The +hash value is obtained using the +[xxHash](https://cyan4973.github.io/xxHash/) algorithm and is +calculated **without** including the `.dll` extension. This is done +for runtime efficiency as the vast majority of Mono requests to load +an assembly does not include the `.dll` suffix, thus saving us time of +appending it in order to generate the hash for index lookup. + +Each entry is represented by the following structure: + + struct BlobHashEntry + { + union { + uint64_t hash64; + uint32_t hash32; + }; + uint32_t mapping_index; + uint32_t local_blob_index; + uint32_t blob_id; + }; + +Individual fields have the following meanings: + + - `hash64`/`hash32`: the 32-bit or 64-bit hash of the assembly's name + **without** the `.dll` suffix + - `mapping_index`: index into a compile-time generated array of + assembly data pointers. This is a global index, unique across + **all** the APK files comprising the application. + - `local_blob_index`: index into blob [Assembly descriptor table](#assembly-descriptor-table) + describing the assembly. + - `blob_id`: ID of the blob containing the assembly From 025be5a74e34edeae1e6e318c64c6526cce8ac45 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 7 Oct 2021 17:20:28 +0200 Subject: [PATCH 34/54] [WIP] This should be a warning --- src/monodroid/jni/embedded-assemblies.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index dcd0e2677cb..eaf5a7ac70d 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -385,7 +385,7 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_string Date: Mon, 11 Oct 2021 13:18:55 +0200 Subject: [PATCH 35/54] [WIP] Update docs + fix an issue detected by the Roslyn analyzer --- Documentation/project-docs/AssemblyBlobs.md | 17 +++++++++++++---- tools/assembly-blob-reader/BlobExplorer.cs | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Documentation/project-docs/AssemblyBlobs.md b/Documentation/project-docs/AssemblyBlobs.md index 8ef1223d232..d05e0cdd2ed 100644 --- a/Documentation/project-docs/AssemblyBlobs.md +++ b/Documentation/project-docs/AssemblyBlobs.md @@ -68,6 +68,10 @@ and `base/root/assemblies/` (for AAB) folders. The architecture agnostic blob is always named `assemblies.blob` while the architecture-specific one is called `assemblies.[ARCH].blob`. +Each APK in the application (e.g. the future Feature APKs) **may** +contain the above two assembly blob files (some APKs may contain only +resources, other may contain only native libraries etc) + Currently, Xamarin.Android applications will produce only one set of blobs but when Xamarin.Android adds support for Android Features, each feature APK will contain its own set of blobs. All of the APKs will @@ -119,7 +123,10 @@ Individual fields have the following meanings: - `local_entry_count`: number of assemblies stored in this blob (also the number of entries in the assembly descriptor table, see below) - `global_entry_count`: number of entries in the index blob's (see - below) hash tables, all the other blobs store `0` in this field + below) hash tables and, thus, the number of assemblies stored in + **all** of the blobs across **all** of the application's APK files, + all the other blobs store `0` in this field since they do **not** + have the hash tables. - `blob_id`: a unique ID of this blob. ## Assembly descriptor table @@ -181,13 +188,15 @@ Both tables share the same format, despite the hashes themselves being of different sizes. This is done to make handling of the tables easier on the runtime. -Each entry contains, among other fields, the assembly name hash. The -hash value is obtained using the +Each entry contains, among other fields, the assembly name hash. In +case of satellite assemblies, the assembly culture (e.g. `en/` or +`fr/`) is treated as part of the assembly name, thus resulting in a +unique hash. The hash value is obtained using the [xxHash](https://cyan4973.github.io/xxHash/) algorithm and is calculated **without** including the `.dll` extension. This is done for runtime efficiency as the vast majority of Mono requests to load an assembly does not include the `.dll` suffix, thus saving us time of -appending it in order to generate the hash for index lookup. +appending it in order to generate the hash for index lookup. Each entry is represented by the following structure: diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs index 37574173c18..c2b46b13aba 100644 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ b/tools/assembly-blob-reader/BlobExplorer.cs @@ -318,7 +318,7 @@ string GetBaseNameHaveExtension (string blobPath, string extension) string GetBaseNameNoExtension (string blobPath) { string fileName = Path.GetFileName (blobPath); - if (fileName.EndsWith ("_assemblies")) { + if (fileName.EndsWith ("_assemblies", StringComparison.OrdinalIgnoreCase)) { return fileName; } return $"{fileName}_assemblies"; From 0ee96ef0af7fddb8a21c426462599e9312bfe5c8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 11 Oct 2021 15:14:07 +0200 Subject: [PATCH 36/54] [WIP] Code cleanup Remove some commented-out code, fix formatting --- .../Xamarin.Android.Build.Tests/XASdkTests.cs | 18 +++++++-------- .../Xamarin.Android.Common.targets | 4 ++-- src/monodroid/jni/application_dso_stub.cc | 2 +- src/monodroid/jni/embedded-assemblies-zip.cc | 23 ------------------- src/monodroid/jni/embedded-assemblies.cc | 1 - 5 files changed, 11 insertions(+), 37 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index 49465ffc193..cedc9e46439 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -624,17 +624,15 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); helper.AssertContainsEntry ($"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); helper.AssertContainsEntry ($"assemblies/es/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); -// using (var apk = ZipHelper.OpenZip (apkPath)) { - foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { - helper.AssertContainsEntry ($"lib/{abi}/libmonodroid.so"); - helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); - if (rids.Length > 1) { - helper.AssertContainsEntry ($"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); - } else { - helper.AssertContainsEntry ("assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); - } + foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { + helper.AssertContainsEntry ($"lib/{abi}/libmonodroid.so"); + helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); + if (rids.Length > 1) { + helper.AssertContainsEntry ($"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); + } else { + helper.AssertContainsEntry ("assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); } -// } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index ea728eb966c..0a897143596 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,8 +179,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False - False - True + False + True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 5b2e931a4f2..7daa45587c2 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -43,7 +43,7 @@ ApplicationConfig application_config = { .instant_run_enabled = false, .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, - .have_assemblies_blob = false, + .have_assemblies_blob = true, .bound_exception_type = 0, // System .package_naming_policy = 0, .environment_variable_count = 0, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 5e7a4f1af3f..b2b19b69879 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -219,37 +219,14 @@ EmbeddedAssemblies::map_assembly_blob (dynamic_local_string c constexpr size_t bundled_assembly_size = sizeof(BlobBundledAssembly); constexpr size_t hash_entry_size = sizeof(BlobHashEntry); - // log_warn (LOG_ASSEMBLY, "header_size == %u; bundled_assembly_size == %u; hash_entry_size == %u", header_size, bundled_assembly_size, hash_entry_size); - // log_warn (LOG_ASSEMBLY, "local_entry_count == %u; global_entry_count == %u", header->local_entry_count, header->global_entry_count); - // log_warn (LOG_ASSEMBLY, "Found index blob ('%s');", entry_name.get ()); - // log_warn (LOG_ASSEMBLY, "Local assemblies array starts at offset %zu (%p)", reinterpret_cast(rd.assemblies) - rd.data_start, rd.assemblies); - // log_warn (LOG_ASSEMBLY, "Local assemblies array is %zu bytes long", bundled_assembly_size * rd.assembly_count); - // log_warn (LOG_ASSEMBLY, "Global hashes start at offset %zu", (reinterpret_cast(rd.assemblies) + (bundled_assembly_size * rd.assembly_count)) - rd.data_start); - index_blob_header = header; size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); if constexpr (std::is_same_v) { blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - // log_warn (LOG_ASSEMBLY, "64-bit hashes at %p", blob_assembly_hashes); } else { blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - // log_warn (LOG_ASSEMBLY, "32-bit hashes at %p", blob_assembly_hashes); } - - // log_warn (LOG_ASSEMBLY, "Global hash index contents:"); - // hash_t entry_hash; - // for (size_t i = 0; i < header->global_entry_count; i++) { - // BlobHashEntry &he = blob_assembly_hashes[i]; - - // if constexpr (std::is_same_v) { - // entry_hash = he.hash64; - // } else { - // entry_hash = he.hash32; - // } - - // log_warn (LOG_ASSEMBLY, " [%u]: hash == 0x%zx; mapping index == %u; local blob index == %u; blob ID == %u", i, entry_hash, he.mapping_index, he.local_blob_index, he.blob_id); - // } } number_of_mapped_blobs++; diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index eaf5a7ac70d..7fbfa47356b 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -389,7 +389,6 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_stringblob_id, hash_entry->mapping_index); if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); abort (); From c8673b9f2d4724b39cbb2f5836a32528b1a140bf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 12 Oct 2021 11:54:19 +0200 Subject: [PATCH 37/54] [WIP] Replace all instances of 'blob' with 'store' --- Documentation/README.md | 2 +- .../{AssemblyBlobs.md => AssemblyStores.md} | 109 +++--- Xamarin.Android.sln | 2 +- .../Tasks/BuildApk.cs | 74 ++-- .../Tasks/GeneratePackageManagerJava.cs | 20 +- .../Xamarin.Android.Build.Tests/AotTests.cs | 6 +- .../Xamarin.Android.Build.Tests/BuildTest.cs | 4 +- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 2 +- .../PackagingTest.cs | 4 +- .../Tasks/LinkerTests.cs | 2 +- .../Utilities/ArchiveAssemblyHelper.cs | 96 ++--- .../Xamarin.Android.Build.Tests/XASdkTests.cs | 2 +- .../Xamarin.Android.Build.Tests.csproj | 2 +- ...pplicationConfigNativeAssemblyGenerator.cs | 46 +-- .../Utilities/ApplicationConfigTaskState.cs | 2 +- ...chAssemblyBlob.cs => ArchAssemblyStore.cs} | 30 +- .../Utilities/AssemblyBlobGenerator.cs | 139 -------- .../{AssemblyBlob.cs => AssemblyStore.cs} | 72 ++-- ...lyInfo.cs => AssemblyStoreAssemblyInfo.cs} | 4 +- .../Utilities/AssemblyStoreGenerator.cs | 139 ++++++++ ...alIndex.cs => AssemblyStoreGlobalIndex.cs} | 4 +- ...dexEntry.cs => AssemblyStoreIndexEntry.cs} | 8 +- ...AssemblyBlob.cs => CommonAssemblyStore.cs} | 12 +- .../Xamarin.Android.Common.targets | 8 +- src/monodroid/jni/application_dso_stub.cc | 10 +- src/monodroid/jni/embedded-assemblies-zip.cc | 78 ++--- src/monodroid/jni/embedded-assemblies.cc | 39 ++- src/monodroid/jni/embedded-assemblies.hh | 34 +- src/monodroid/jni/monodroid-glue.cc | 2 +- src/monodroid/jni/xamarin-app.hh | 61 ++-- .../MSBuildDeviceIntegration.csproj | 2 +- .../Tests/BundleToolTests.cs | 8 +- ...Forms.Performance.Integration.Droid.csproj | 2 +- tools/assembly-blob-reader/BlobExplorer.cs | 327 ------------------ .../BlobExplorerLogLevel.cs | 10 - tools/assembly-blob-reader/BlobHashEntry.cs | 25 -- tools/assembly-blob-reader/BlobReader.cs | 184 ---------- .../Directory.Build.targets | 6 - .../AssemblyStoreAssembly.cs} | 22 +- .../AssemblyStoreExplorer.cs | 327 ++++++++++++++++++ .../AssemblyStoreExplorerLogLevel.cs | 10 + .../AssemblyStoreHashEntry.cs | 25 ++ .../AssemblyStoreManifestEntry.cs} | 20 +- .../AssemblyStoreManifestReader.cs} | 14 +- .../AssemblyStoreReader.cs | 184 ++++++++++ .../Directory.Build.targets | 6 + .../Program.cs | 34 +- .../assembly-store-reader.csproj} | 4 +- .../decompress-assemblies.csproj | 2 +- tools/decompress-assemblies/main.cs | 14 +- tools/scripts/read-assembly-store | 10 + tools/scripts/read-blob | 10 - 52 files changed, 1133 insertions(+), 1126 deletions(-) rename Documentation/project-docs/{AssemblyBlobs.md => AssemblyStores.md} (67%) rename src/Xamarin.Android.Build.Tasks/Utilities/{ArchAssemblyBlob.cs => ArchAssemblyStore.cs} (76%) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs rename src/Xamarin.Android.Build.Tasks/Utilities/{AssemblyBlob.cs => AssemblyStore.cs} (81%) rename src/Xamarin.Android.Build.Tasks/Utilities/{BlobAssemblyInfo.cs => AssemblyStoreAssemblyInfo.cs} (88%) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs rename src/Xamarin.Android.Build.Tasks/Utilities/{AssemblyBlobGlobalIndex.cs => AssemblyStoreGlobalIndex.cs} (75%) rename src/Xamarin.Android.Build.Tasks/Utilities/{AssemblyBlobIndexEntry.cs => AssemblyStoreIndexEntry.cs} (84%) rename src/Xamarin.Android.Build.Tasks/Utilities/{CommonAssemblyBlob.cs => CommonAssemblyStore.cs} (67%) delete mode 100644 tools/assembly-blob-reader/BlobExplorer.cs delete mode 100644 tools/assembly-blob-reader/BlobExplorerLogLevel.cs delete mode 100644 tools/assembly-blob-reader/BlobHashEntry.cs delete mode 100644 tools/assembly-blob-reader/BlobReader.cs delete mode 100644 tools/assembly-blob-reader/Directory.Build.targets rename tools/{assembly-blob-reader/BlobAssembly.cs => assembly-store-reader/AssemblyStoreAssembly.cs} (73%) create mode 100644 tools/assembly-store-reader/AssemblyStoreExplorer.cs create mode 100644 tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs create mode 100644 tools/assembly-store-reader/AssemblyStoreHashEntry.cs rename tools/{assembly-blob-reader/BlobManifestEntry.cs => assembly-store-reader/AssemblyStoreManifestEntry.cs} (71%) rename tools/{assembly-blob-reader/BlobManifestReader.cs => assembly-store-reader/AssemblyStoreManifestReader.cs} (62%) create mode 100644 tools/assembly-store-reader/AssemblyStoreReader.cs create mode 100644 tools/assembly-store-reader/Directory.Build.targets rename tools/{assembly-blob-reader => assembly-store-reader}/Program.cs (70%) rename tools/{assembly-blob-reader/assembly-blob-reader.csproj => assembly-store-reader/assembly-store-reader.csproj} (86%) create mode 100755 tools/scripts/read-assembly-store delete mode 100755 tools/scripts/read-blob diff --git a/Documentation/README.md b/Documentation/README.md index cbbf682b0fc..cb645dcbbe1 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -11,7 +11,7 @@ * [Submitting Bugs, Feature Requests, and Pull Requests][bugs] * [Directory Structure](project-docs/ExploringSources.md) - * [Assembly blob format](project-docs/AssemblyBlobs.md) + * [Assembly store format](project-docs/AssemblyStores.md) [bugs]: https://github.com/xamarin/xamarin-android/wiki/Submitting-Bugs,-Feature-Requests,-and-Pull-Requests diff --git a/Documentation/project-docs/AssemblyBlobs.md b/Documentation/project-docs/AssemblyStores.md similarity index 67% rename from Documentation/project-docs/AssemblyBlobs.md rename to Documentation/project-docs/AssemblyStores.md index d05e0cdd2ed..57be490322b 100644 --- a/Documentation/project-docs/AssemblyBlobs.md +++ b/Documentation/project-docs/AssemblyStores.md @@ -1,29 +1,29 @@ **Table of Contents** -- [Assembly Blobs format and purpose](#assembly-blobs-format-and-purpose) +- [Assembly Store format and purpose](#assembly-store-format-and-purpose) - [Rationale](#rationale) -- [Blob kinds and locations](#blob-kinds-and-locations) -- [Blob format](#blob-format) +- [Store kinds and locations](#store-kinds-and-locations) +- [Store format](#store-format) - [Common header](#common-header) - [Assembly descriptor table](#assembly-descriptor-table) - - [Index blob](#index-blob) + - [Index store](#index-store) - [Hash table format](#hash-table-format) -# Assembly Blobs format and purpose +# Assembly Store format and purpose -Assembly blobs are binary files which contain inside them the managed +Assembly stores are binary files which contain the managed assemblies, their debug data (optionally) and the associated config file (optionally). They are placed inside the Android APK/AAB archives, replacing individual assemblies/pdb/config files. -Blobs are an optional form of assembly storage in the archive, they -can be used in all build configurations **except** when Fast -Deployment is in effect (in which case assemblies aren't placed in the -archives at all, they are instead synchronized from the host to the -device/emulator filesystem) +Assembly stores are an optional form of assembly storage in the +archive, they can be used in all build configurations **except** when +Fast Deployment is in effect (in which case assemblies aren't placed +in the archives at all, they are instead synchronized from the host to +the device/emulator filesystem) ## Rationale @@ -46,44 +46,44 @@ invocation is a system call) but it also makes the assembly discovery an O(n) algorithm, which takes more time as more assemblies are added to the APK/AAB archive. -An assembly blob, however, needs to be mapped only once and any +An assembly store, however, needs to be mapped only once and any further operations are merely pointer arithmetic, making the process not only faster but also reducing the algorithm complexity to O(1). -# Blob kinds and locations +# Store kinds and locations -Each application will contain at least a single blob, with assemblies -that are architecture-agnostics and any number of -architecture-specific blobs. dotnet ships with a handful of +Each application will contain at least a single assembly store, with +assemblies that are architecture-agnostics and any number of +architecture-specific stores. dotnet ships with a handful of assemblies that **are** architecture-specific - those assemblies are -placed in an architecture specific blob, one per architecture +placed in an architecture specific store, one per architecture supported by and enabled for the application. On the execution time, the Xamarin.Android runtime will always map the architecture-agnostic -blob and one, and **only** one, of the architecture-specific blobs. +store and one, and **only** one, of the architecture-specific stores. -Blobs are placed in the same location in the APK/AAB archive where the +Stores are placed in the same location in the APK/AAB archive where the individual assemblies traditionally live, the `assemblies/` (for APK) and `base/root/assemblies/` (for AAB) folders. -The architecture agnostic blob is always named `assemblies.blob` while +The architecture agnostic store is always named `assemblies.blob` while the architecture-specific one is called `assemblies.[ARCH].blob`. Each APK in the application (e.g. the future Feature APKs) **may** -contain the above two assembly blob files (some APKs may contain only +contain the above two assembly store files (some APKs may contain only resources, other may contain only native libraries etc) Currently, Xamarin.Android applications will produce only one set of -blobs but when Xamarin.Android adds support for Android Features, each -feature APK will contain its own set of blobs. All of the APKs will +stores but when Xamarin.Android adds support for Android Features, each +feature APK will contain its own set of stores. All of the APKs will follow the location, format and naming conventions described above. -# Blob format +# Store format -Each blob is a structured binary file, using little-endian byte order -and aligned to a byte boundary. Each blob consists of a header, an +Each store is a structured binary file, using little-endian byte order +and aligned to a byte boundary. Each store consists of a header, an assembly descriptor table and, optionally (see below), two tables with -assembly name hashes. All the blobs are assigned a unique ID, with -the blob having ID equal to `0` being the [Index blob](#index-blob) +assembly name hashes. All the stores are assigned a unique ID, with +the store having ID equal to `0` being the [Index store](#index-store) Assemblies are stored as adjacent byte streams: @@ -105,37 +105,38 @@ one that should be trusted. ## Common header -All kinds of blobs share the following header format: +All kinds of stores share the following header format: - struct BundledAssemblyBlobHeader + struct AssemblyStoreHeader { uint32_t magic; uint32_t version; uint32_t local_entry_count; uint32_t global_entry_count; - uint32_t blob_id; + uint32_t store_id; ; Individual fields have the following meanings: - `magic`: has the value of 0x41424158 (`XABA`) - - `version`: a value increased every time blob format changes. - - `local_entry_count`: number of assemblies stored in this blob (also - the number of entries in the assembly descriptor table, see below) - - `global_entry_count`: number of entries in the index blob's (see + - `version`: a value increased every time assembly store format changes. + - `local_entry_count`: number of assemblies stored in this assembly + store (also the number of entries in the assembly descriptor + table, see below) + - `global_entry_count`: number of entries in the index store's (see below) hash tables and, thus, the number of assemblies stored in - **all** of the blobs across **all** of the application's APK files, - all the other blobs store `0` in this field since they do **not** - have the hash tables. - - `blob_id`: a unique ID of this blob. + **all** of the assembly stores across **all** of the application's + APK files, all the other assembly stores have `0` in this field + since they do **not** have the hash tables. + - `store_id`: a unique ID of this store. ## Assembly descriptor table -Each blob header is followed by a table of -`BundledAssemblyBlobHeader.local_entry_count` entries, each entry +Each store header is followed by a table of +`AssemblyStoreHeader.local_entry_count` entries, each entry defined by the following structure: - struct BlobBundledAssembly + struct AssemblyStoreAssemblyDescriptor { uint32_t data_offset; uint32_t data_size; @@ -151,23 +152,23 @@ value, other fields describe optional data and can be set to `0`. Individual fields have the following meanings: - `data_offset`: offset of the assembly image data from the - beginning of the blob file + beginning of the store file - `data_size`: number of bytes of the image data - `debug_data_offset`: offset of the assembly's debug data from the - beginning of the blob file. A value of `0` indicates there's no + beginning of the store file. A value of `0` indicates there's no debug data for this assembly. - `debug_data_size`: number of bytes of debug data. Can be `0` only if `debug_data_offset` is `0` - `config_data_offset`: offset of the assembly's config file data - from the beginning of the blob file. A value of `0` indicates + from the beginning of the store file. A value of `0` indicates there's no config file data for this assembly. - `config_data_size`: number of bytes of config file data. Can be `0` only if `config_data_offset` is `0` -## Index blob +## Index store -Each application will contain exactly one blob with a global index - -two tables with assembly name hashes. All the other blobs **do not** +Each application will contain exactly one store with a global index - +two tables with assembly name hashes. All the other stores **do not** contain these tables. Two hash tables are necessary because hashes for 32-bit and 64-bit devices are different. @@ -175,7 +176,7 @@ The hash tables follow the [Assembly descriptor table](#assembly-descriptor-table) and precede the individual assembly streams. -Placing the hash tables in a single index blob, while "wasting" a +Placing the hash tables in a single index store, while "wasting" a certain amount of memory (since 32-bit devices won't use the 64-bit table and vice versa), makes for simpler and faster runtime implementation and the amount of memory wasted isn't big (1000 @@ -200,15 +201,15 @@ appending it in order to generate the hash for index lookup. Each entry is represented by the following structure: - struct BlobHashEntry + struct AssemblyStoreHashEntry { union { uint64_t hash64; uint32_t hash32; }; uint32_t mapping_index; - uint32_t local_blob_index; - uint32_t blob_id; + uint32_t local_store_index; + uint32_t store_id; }; Individual fields have the following meanings: @@ -218,6 +219,6 @@ Individual fields have the following meanings: - `mapping_index`: index into a compile-time generated array of assembly data pointers. This is a global index, unique across **all** the APK files comprising the application. - - `local_blob_index`: index into blob [Assembly descriptor table](#assembly-descriptor-table) + - `local_store_index`: index into assembly store [Assembly descriptor table](#assembly-descriptor-table) describing the assembly. - - `blob_id`: ID of the blob containing the assembly + - `store_id`: ID of the assembly store containing the assembly diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 46b98910697..608787bbea4 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -148,7 +148,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "to EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-blob-reader", "tools\assembly-blob-reader\assembly-blob-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index f12cf2d1c01..32a5adebc66 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -120,7 +120,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List existingEntries = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string blobApkName) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -180,7 +180,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } if (EmbedAssemblies && !BundleAssemblies) - AddAssemblies (apk, debug, compress, compressedAssembliesInfo, blobApkName); + AddAssemblies (apk, debug, compress, compressedAssembliesInfo, assemblyStoreApkName); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); @@ -301,7 +301,7 @@ public override bool RunTask () throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); } - ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, blobApkName: null); + ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null); outputFiles.Add (ApkOutputPath); if (CreatePackagePerAbi && SupportedAbis.Length > 1) { foreach (var abi in SupportedAbis) { @@ -310,7 +310,7 @@ public override bool RunTask () var apk = Path.GetFileNameWithoutExtension (ApkOutputPath); ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi), Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)), - debug, compress, compressedAssembliesInfo, blobApkName: abi); + debug, compress, compressedAssembliesInfo, assemblyStoreApkName: abi); outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi))); } } @@ -322,37 +322,36 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string blobApkName) + void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) { var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); - bool useAssembliesBlob = appConfState != null ? appConfState.UseAssembliesBlob : false; + bool useAssemblyStores = appConfState != null ? appConfState.UseAssemblyStore : false; string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); - AssemblyBlobGenerator blobGenerator; + AssemblyStoreGenerator storeGenerator; - if (useAssembliesBlob) { - Log.LogDebugMessage ("Creating AssemblyBlobGenerator instance"); - blobGenerator = new AssemblyBlobGenerator (AssembliesPath, Log); + if (useAssemblyStores) { + storeGenerator = new AssemblyStoreGenerator (AssembliesPath, Log); } else { - blobGenerator = null; + storeGenerator = null; } int count = 0; - BlobAssemblyInfo blobAssembly = null; + AssemblyStoreAssemblyInfo storeAssembly = null; // // NOTE // - // The very first blob (ID 0) **must** contain an index of all the assemblies included in the application, even if they - // are included in other APKs than the base one. The ID 0 blob **must** be placed in the base assembly + // The very first store (ID 0) **must** contain an index of all the assemblies included in the application, even if they + // are included in other APKs than the base one. The ID 0 store **must** be placed in the base assembly // - // Currently, all the assembly blobs end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we + // Currently, all the assembly stores end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we // partition assemblies into "feature" APKs const string DefaultBaseApkName = "base"; - if (String.IsNullOrEmpty (blobApkName)) { - blobApkName = DefaultBaseApkName; + if (String.IsNullOrEmpty (assemblyStoreApkName)) { + assemblyStoreApkName = DefaultBaseApkName; } // Add user assemblies @@ -362,25 +361,25 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); - if (blobPaths == null) { - throw new InvalidOperationException ("Blob generator did not generate any blobs"); + Dictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); + if (assemblyStorePaths == null) { + throw new InvalidOperationException ("Assembly store generator did not generate any stores"); } - if (!blobPaths.TryGetValue (blobApkName, out List baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) { - throw new InvalidOperationException ("Blob generator didn't generate the required base blobs"); + if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { + throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); } - string blobPrefix = $"{blobApkName}_"; - foreach (string blobPath in baseBlobs) { - string inArchiveName = Path.GetFileName (blobPath); + string assemblyStorePrefix = $"{assemblyStoreApkName}_"; + foreach (string assemblyStorePath in baseAssemblyStores) { + string inArchiveName = Path.GetFileName (assemblyStorePath); - if (inArchiveName.StartsWith (blobPrefix, StringComparison.Ordinal)) { - inArchiveName = inArchiveName.Substring (blobPrefix.Length); + if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { + inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); } CompressionMethod compressionMethod; @@ -390,7 +389,7 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary= ZipArchiveEx.ZipFlushFilesLimit) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 59e56dde015..9e8af4e1f3c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -30,7 +30,7 @@ public class GeneratePackageManagerJava : AndroidTask public ITaskItem[] SatelliteAssemblies { get; set; } - public bool UseAssembliesBlob { get; set; } + public bool UseAssemblyStore { get; set; } [Required] public string OutputDirectory { get; set; } @@ -275,7 +275,7 @@ void AddEnvironment () Encoding assemblyNameEncoding = Encoding.UTF8; Action updateNameWidth = (ITaskItem assembly) => { - if (UseAssembliesBlob) { + if (UseAssemblyStore) { return; } @@ -290,7 +290,7 @@ void AddEnvironment () HashSet archAssemblyNames = null; Action updateAssemblyCount = (ITaskItem assembly) => { - if (!UseAssembliesBlob) { + if (!UseAssemblyStore) { assemblyCount++; return; } @@ -321,7 +321,7 @@ void AddEnvironment () updateAssemblyCount (assembly); } - if (!UseAssembliesBlob) { + if (!UseAssemblyStore) { int abiNameLength = 0; foreach (string abi in SupportedAbis) { if (abi.Length <= abiNameLength) { @@ -335,7 +335,7 @@ void AddEnvironment () bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); if (appConfState != null) { - appConfState.UseAssembliesBlob = UseAssembliesBlob; + appConfState.UseAssemblyStore = UseAssemblyStore; }; foreach (string abi in SupportedAbis) { @@ -358,11 +358,11 @@ void AddEnvironment () HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, - NumberOfAssemblyBlobsInApk = 2, // Until feature APKs are a thing, we're going to have just two blobs in each app - one for arch-agnostic - // and up to 4 other for arch-specific assemblies. Only **one** arch-specific blob is ever loaded on the app - // runtime, thus the number 2 here. All architecture specific blobs contain assemblies with the same names - // and in the same order. - HaveAssembliesBlob = UseAssembliesBlob, + NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic + // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app + // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names + // and in the same order. + HaveAssemblyStore = UseAssemblyStore, }; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index 9ff86704647..f9a88389426 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -182,7 +182,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); if (checkMinLlvmPath) { // Set //uses-sdk/@android:minSdkVersion so that LLVM uses the right libc.so @@ -261,7 +261,7 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { if (!b.CrossCompilerAvailable (supportedAbis)) Assert.Ignore ("Cross compiler was not available"); @@ -405,7 +405,7 @@ public static void Foo () { proj.SetProperty ("AndroidAotMode", "Hybrid"); // So we can use Mono.Cecil to open assemblies directly proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); proj.SetAndroidSupportedAbis (abis); using (var b = CreateApkBuilder ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 4edc72324e1..6beed1e2407 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -109,7 +109,7 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive proj.SetProperty (proj.ActiveConfigurationProperties, "MonoSymbolArchive", monoSymbolArchive); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugSymbols", debugSymbols); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugType", debugType); - proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder ()) { if (aotAssemblies && !b.CrossCompilerAvailable (string.Join (";", abis))) Assert.Ignore ("Cross compiler was not available"); @@ -940,7 +940,7 @@ public void BuildBasicApplicationCheckPdb () var proj = new XamarinAndroidApplicationProject { EmbedAssembliesIntoApk = true, }; - proj.SetProperty ("AndroidUseAssembliesBlob", "False"); + proj.SetProperty ("AndroidUseAssemblyStore", "False"); using (var b = CreateApkBuilder ()) { var reference = new BuildItem.Reference ("PdbTestLibrary.dll") { WebContentFileNameFromAzure = "PdbTestLibrary.dll" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 4488a83b6f0..05af0db36b5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -77,7 +77,7 @@ public void BuildReleaseArm64 ([Values (false, true)] bool forms) proj.IsRelease = true; proj.SetAndroidSupportedAbis ("arm64-v8a"); proj.SetProperty ("LinkerDumpDependencies", "True"); - proj.SetProperty ("AndroidUseAssembliesBlob", "False"); + proj.SetProperty ("AndroidUseAssemblyStore", "False"); if (forms) { proj.PackageReferences.Clear (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 25df1c38655..a1e51c23446 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -72,7 +72,7 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssembliesB var proj = new XamarinAndroidApplicationProject { IsRelease = true }; - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssembliesBlob.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssembliesBlob.ToString ()); proj.SetAndroidSupportedAbis ("armeabi-v7a"); if (!Builder.UseDotNet) { proj.PackageReferences.Add (new Package { @@ -638,7 +638,7 @@ protected override void OnResume() }, } }; - app.SetProperty ("AndroidUseAssembliesBlob", "False"); + app.SetProperty ("AndroidUseAssemblyStore", "False"); app.MainActivity = @"using System; using Android.App; using Android.Content; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index e85fdeac118..70c864082c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -218,7 +218,7 @@ public void RemoveDesigner ([Values (true, false)] bool usesAssemblyBlobs) }; proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); proj.SetProperty ("AndroidLinkResources", "True"); - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); string assemblyName = proj.ProjectName; using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "build should have succeeded."); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index f6566e70747..a510d009c17 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -4,7 +4,7 @@ using System.IO; using System.Linq; -using Xamarin.Android.AssemblyBlobReader; +using Xamarin.Android.AssemblyStore; using Xamarin.ProjectTools; using Xamarin.Tools.Zip; @@ -12,8 +12,8 @@ namespace Xamarin.Android.Build.Tests { public class ArchiveAssemblyHelper { - public const string DefaultBlobEntryPrefix = "{blobReader}"; - const int BlobReadBufferSize = 8192; + public const string DefaultAssemblyStoreEntryPrefix = "{storeReader}"; + const int AssemblyStoreReadBufferSize = 8192; static readonly HashSet SpecialExtensions = new HashSet (StringComparer.OrdinalIgnoreCase) { ".dll", @@ -33,19 +33,19 @@ public class ArchiveAssemblyHelper readonly string archivePath; readonly string assembliesRootDir; - bool useAssemblyBlobs; + bool useAssemblyStores; List archiveContents; public string ArchivePath => archivePath; - public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) + public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores) { if (String.IsNullOrEmpty (archivePath)) { throw new ArgumentException ("must not be null or empty", nameof (archivePath)); } this.archivePath = archivePath; - this.useAssemblyBlobs = useAssemblyBlobs; + this.useAssemblyStores = useAssemblyStores; string extension = Path.GetExtension (archivePath) ?? String.Empty; if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { @@ -61,8 +61,8 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) public Stream ReadEntry (string path) { - if (useAssemblyBlobs) { - return ReadBlobEntry (path); + if (useAssemblyStores) { + return ReadStoreEntry (path); } return ReadZipEntry (path); @@ -79,46 +79,46 @@ Stream ReadZipEntry (string path) } } - Stream ReadBlobEntry (string path) + Stream ReadStoreEntry (string path) { - BlobReader blobReader = null; - BlobAssembly assembly = null; + AssemblyStoreReader storeReader = null; + AssemblyStoreAssembly assembly = null; string name = Path.GetFileNameWithoutExtension (path); - var explorer = new BlobExplorer (archivePath); + var explorer = new AssemblyStoreExplorer (archivePath); foreach (var asm in explorer.Assemblies) { if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { continue; } assembly = asm; - blobReader = asm.Blob; + storeReader = asm.Store; break; } - if (blobReader == null) { - Console.WriteLine ($"Blob for entry {path} not found, will try a standard Zip read"); + if (storeReader == null) { + Console.WriteLine ($"Store for entry {path} not found, will try a standard Zip read"); return ReadZipEntry (path); } - string blobEntryName; - if (String.IsNullOrEmpty (blobReader.Arch)) { - blobEntryName = $"{assembliesRootDir}assemblies.blob"; + string storeEntryName; + if (String.IsNullOrEmpty (storeReader.Arch)) { + storeEntryName = $"{assembliesRootDir}assemblies.store"; } else { - blobEntryName = $"{assembliesRootDir}assemblies_{blobReader.Arch}.blob"; + storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.store"; } - Stream blob = ReadZipEntry (blobEntryName); - if (blob == null) { - Console.WriteLine ($"Blob zip entry {blobEntryName} does not exist"); + Stream store = ReadZipEntry (storeEntryName); + if (store == null) { + Console.WriteLine ($"Store zip entry {storeEntryName} does not exist"); return null; } - blob.Seek (assembly.DataOffset, SeekOrigin.Begin); + store.Seek (assembly.DataOffset, SeekOrigin.Begin); var ret = new MemoryStream (); - byte[] buffer = buffers.Rent (BlobReadBufferSize); + byte[] buffer = buffers.Rent (AssemblyStoreReadBufferSize); int toRead = (int)assembly.DataSize; while (toRead > 0) { - int nread = blob.Read (buffer, 0, BlobReadBufferSize); + int nread = store.Read (buffer, 0, AssemblyStoreReadBufferSize); if (nread <= 0) { break; } @@ -127,20 +127,20 @@ Stream ReadBlobEntry (string path) toRead -= nread; } ret.Flush (); - blob.Dispose (); + store.Dispose (); buffers.Return (buffer); return ret; } - public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix, bool forceRefresh = false) + public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false) { if (!forceRefresh && archiveContents != null) { return archiveContents; } - if (String.IsNullOrEmpty (blobEntryPrefix)) { - throw new ArgumentException (nameof (blobEntryPrefix), "must not be null or empty"); + if (String.IsNullOrEmpty (storeEntryPrefix)) { + throw new ArgumentException (nameof (storeEntryPrefix), "must not be null or empty"); } var entries = new List (); @@ -151,17 +151,17 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } archiveContents = entries; - if (!useAssemblyBlobs) { - Console.WriteLine ("Not using assembly blobs"); + if (!useAssemblyStores) { + Console.WriteLine ("Not using assembly stores"); return entries; } - var explorer = new BlobExplorer (archivePath); + var explorer = new AssemblyStoreExplorer (archivePath); foreach (var asm in explorer.Assemblies) { - string prefix = blobEntryPrefix; + string prefix = storeEntryPrefix; - if (!String.IsNullOrEmpty (asm.Blob.Arch)) { - string arch = ArchToAbi[asm.Blob.Arch]; + if (!String.IsNullOrEmpty (asm.Store.Arch)) { + string arch = ArchToAbi[asm.Store.Arch]; prefix = $"{prefix}{arch}/"; } @@ -175,7 +175,7 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } } - Console.WriteLine ("Archive entries with synthetised assembly blobReader entries:"); + Console.WriteLine ("Archive entries with synthetised assembly storeReader entries:"); foreach (string e in entries) { Console.WriteLine ($" {e}"); } @@ -203,8 +203,8 @@ public void Contains (string[] fileNames, out List existingFiles, out Li throw new ArgumentException ("must not be empty", nameof (fileNames)); } - if (useAssemblyBlobs) { - BlobContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + if (useAssemblyStores) { + StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); } else { ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); } @@ -219,7 +219,7 @@ void ArchiveContains (string[] fileNames, out List existingFiles, out Li } } - void BlobContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) { var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); @@ -241,16 +241,16 @@ void BlobContains (string[] fileNames, out List existingFiles, out List< } } - var explorer = new BlobExplorer (archivePath); + var explorer = new AssemblyStoreExplorer (archivePath); - // Blobs don't store the assembly extension - var blobAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); + // Assembly stores don't store the assembly extension + var storeAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); if (explorer.AssembliesByName.Count != 0) { - existingFiles.AddRange (blobAssemblies); + existingFiles.AddRange (storeAssemblies); - // We need to fake config and debug files since they have no named entries in the blobReader + // We need to fake config and debug files since they have no named entries in the storeReader foreach (string file in configFiles) { - BlobAssembly asm = GetBlobAssembly (file); + AssemblyStoreAssembly asm = GetStoreAssembly (file); if (asm == null) { continue; } @@ -261,7 +261,7 @@ void BlobContains (string[] fileNames, out List existingFiles, out List< } foreach (string file in debugFiles) { - BlobAssembly asm = GetBlobAssembly (file); + AssemblyStoreAssembly asm = GetStoreAssembly (file); if (asm == null) { continue; } @@ -281,14 +281,14 @@ void BlobContains (string[] fileNames, out List existingFiles, out List< additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); - BlobAssembly GetBlobAssembly (string file) + AssemblyStoreAssembly GetStoreAssembly (string file) { string assemblyName = Path.GetFileNameWithoutExtension (file); if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { assemblyName = Path.GetFileNameWithoutExtension (assemblyName); } - if (!explorer.AssembliesByName.TryGetValue (assemblyName, out BlobAssembly asm) || asm == null) { + if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreAssembly asm) || asm == null) { return null; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index cedc9e46439..c7545bceedb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -530,7 +530,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo } }; proj.MainActivity = proj.DefaultMainActivity.Replace (": Activity", ": AndroidX.AppCompat.App.AppCompatActivity"); - proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); if (aot) { proj.SetProperty ("RunAOTCompilation", "true"); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index a57da49b8fb..0ffc5865252 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -32,7 +32,7 @@ - + ..\Expected\GenerateDesignerFileExpected.cs PreserveNewest diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index d90f40da386..4d3a2806c05 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -24,9 +24,9 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public bool InstantRunEnabled { get; set; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } public bool HaveRuntimeConfigBlob { get; set; } - public bool HaveAssembliesBlob { get; set; } + public bool HaveAssemblyStore { get; set; } public int NumberOfAssembliesInApk { get; set; } - public int NumberOfAssemblyBlobsInApk { get; set; } + public int NumberOfAssemblyStoresInApks { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public PackageNamingPolicy PackageNamingPolicy { get; set; } @@ -79,8 +79,8 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "have_runtime_config_blob"); size += WriteData (output, HaveRuntimeConfigBlob); - WriteCommentLine (output, "have_assemblies_blob"); - size += WriteData (output, HaveAssembliesBlob); + WriteCommentLine (output, "have_assembly_store"); + size += WriteData (output, HaveAssemblyStore); WriteCommentLine (output, "bound_exception_type"); size += WriteData (output, (byte)BoundExceptionType); @@ -100,8 +100,8 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "bundled_assembly_name_width"); size += WriteData (output, BundledAssemblyNameWidth); - WriteCommentLine (output, "number_of_assembly_blobs"); - size += WriteData (output, NumberOfAssemblyBlobsInApk); + WriteCommentLine (output, "number_of_assembly_store_files"); + size += WriteData (output, NumberOfAssemblyStoresInApks); WriteCommentLine (output, "android_package_name"); size += WritePointer (output, MakeLocalLabel (stringLabel)); @@ -118,46 +118,46 @@ protected override void WriteSymbols (StreamWriter output) WriteNameValueStringArray (output, "app_system_properties", systemProperties); WriteBundledAssemblies (output); - WriteBlobAssemblies (output); + WriteAssemblyStoreAssemblies (output); } - void WriteBlobAssemblies (StreamWriter output) + void WriteAssemblyStoreAssemblies (StreamWriter output) { output.WriteLine (); - string label = "blob_bundled_assemblies"; - WriteCommentLine (output, "Individual blob assembly data"); + string label = "assembly_store_bundled_assemblies"; + WriteCommentLine (output, "Assembly store individual assembly data"); WriteDataSection (output, label); WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); uint size = 0; - if (HaveAssembliesBlob) { + if (HaveAssemblyStore) { for (int i = 0; i < NumberOfAssembliesInApk; i++) { - size += WriteStructure (output, packed: false, structureWriter: () => WriteBlobAssembly (output)); + size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyStoreAssembly (output)); } } WriteStructureSize (output, label, size); output.WriteLine (); - label = "assembly_blobs"; - WriteCommentLine (output, "Assembly blob data"); + label = "assembly_stores"; + WriteCommentLine (output, "Assembly store data"); WriteDataSection (output, label); WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); size = 0; - if (HaveAssembliesBlob) { - for (int i = 0; i < NumberOfAssemblyBlobsInApk; i++) { - size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyBlob (output)); + if (HaveAssemblyStore) { + for (int i = 0; i < NumberOfAssemblyStoresInApks; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyStore (output)); } } WriteStructureSize (output, label, size); } - uint WriteBlobAssembly (StreamWriter output) + uint WriteAssemblyStoreAssembly (StreamWriter output) { // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh BlobAssemblyRuntimeData structure + // src/monodroid/jni/xamarin-app.hh AssemblyStoreSingleAssemblyRuntimeData structure WriteCommentLine (output, "image_data"); uint size = WritePointer (output); @@ -175,10 +175,10 @@ uint WriteBlobAssembly (StreamWriter output) return size; } - uint WriteAssemblyBlob (StreamWriter output) + uint WriteAssemblyStore (StreamWriter output) { // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyBlobRuntimeData structure + // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure WriteCommentLine (output, "data_start"); uint size = WritePointer (output); @@ -201,7 +201,7 @@ void WriteBundledAssemblies (StreamWriter output) WriteSection (output, ".bss.bundled_assembly_names", hasStrings: false, writable: true, nobits: true); List name_labels = null; - if (!HaveAssembliesBlob) { + if (!HaveAssemblyStore) { name_labels = new List (); for (int i = 0; i < NumberOfAssembliesInApk; i++) { string bufferLabel = GetBufferLabel (); @@ -218,7 +218,7 @@ void WriteBundledAssemblies (StreamWriter output) WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); uint size = 0; - if (!HaveAssembliesBlob) { + if (!HaveAssemblyStore) { for (int i = 0; i < NumberOfAssembliesInApk; i++) { size += WriteStructure (output, packed: false, structureWriter: () => WriteBundledAssembly (output, MakeLocalLabel (name_labels[i]))); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs index 0212933b3ab..3de3f75a830 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs @@ -5,6 +5,6 @@ class ApplicationConfigTaskState public const string RegisterTaskObjectKey = "Xamarin.Android.Tasks.ApplicationConfigTaskState"; public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } = false; - public bool UseAssembliesBlob { get; set; } = false; + public bool UseAssemblyStore { get; set; } = false; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs similarity index 76% rename from src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs index 9262d123f5f..49e928ca9da 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs @@ -7,34 +7,34 @@ namespace Xamarin.Android.Tasks { - class ArchAssemblyBlob : AssemblyBlob + class ArchAssemblyStore : AssemblyStore { - readonly Dictionary> assemblies; + readonly Dictionary> assemblies; HashSet seenArchAssemblyNames; - public ArchAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) + public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) { - assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); + assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); } - public override string WriteIndex (List globalIndex) + public override string WriteIndex (List globalIndex) { throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); } - public override void Add (BlobAssemblyInfo blobAssembly) + public override void Add (AssemblyStoreAssemblyInfo blobAssembly) { if (String.IsNullOrEmpty (blobAssembly.Abi)) { throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); } if (!assemblies.ContainsKey (blobAssembly.Abi)) { - assemblies.Add (blobAssembly.Abi, new List ()); + assemblies.Add (blobAssembly.Abi, new List ()); } Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Arch '{blobAssembly.Abi}' assembly {blobAssembly.FilesystemAssemblyPath}"); - List blobAssemblies = assemblies[blobAssembly.Abi]; + List blobAssemblies = assemblies[blobAssembly.Abi]; blobAssemblies.Add (blobAssembly); if (seenArchAssemblyNames == null) { @@ -49,7 +49,7 @@ public override void Add (BlobAssemblyInfo blobAssembly) seenArchAssemblyNames.Add (assemblyName); } - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) { if (assemblies.Count == 0) { return; @@ -58,15 +58,15 @@ public override void Generate (string outputDirectory, List (); foreach (var kvp in assemblies) { string abi = kvp.Key; - List archAssemblies = kvp.Value; + List archAssemblies = kvp.Value; Log.LogMessage (MessageImportance.Low, $"Blob: assemblies for architecture {abi}:"); // All the architecture blobs must have assemblies in exactly the same order - archAssemblies.Sort ((BlobAssemblyInfo a, BlobAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); + archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); if (assemblyNames.Count == 0) { Log.LogMessage (MessageImportance.Low, $" first arch, preparing names list, {archAssemblies.Count} entries"); for (int i = 0; i < archAssemblies.Count; i++) { - BlobAssemblyInfo info = archAssemblies[i]; + AssemblyStoreAssemblyInfo info = archAssemblies[i]; Log.LogMessage (MessageImportance.Low, $" {Path.GetFileName (info.FilesystemAssemblyPath)}"); assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); } @@ -75,7 +75,7 @@ public override void Generate (string outputDirectory, List archAssemblies = kvp.Value; + List archAssemblies = kvp.Value; if (archAssemblies.Count == 0) { continue; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs deleted file mode 100644 index fee2fb1ce6a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGenerator.cs +++ /dev/null @@ -1,139 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class AssemblyBlobGenerator - { - sealed class Blob - { - public AssemblyBlob Common; - public AssemblyBlob Arch; - } - - readonly string archiveAssembliesPrefix; - readonly TaskLoggingHelper log; - - // NOTE: when/if we have parallel BuildApk this should become a ConcurrentDictionary - readonly Dictionary blobs; - - AssemblyBlob indexBlob; - - // IDs must be counted per AssemblyBlobGenerator instance because it's possible that a single build will create more than one instance of the class and each time - // the blobs must be assigned IDs starting from 0, or there will be errors due to "missing" index blob - readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); - - // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** - // blob, thus we need to keep the state here - AssemblyBlobGlobalIndex globalIndexCounter = new AssemblyBlobGlobalIndex (); - - public AssemblyBlobGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - this.log = log; - - blobs = new Dictionary (StringComparer.Ordinal); - } - - public void Add (string apkName, BlobAssemblyInfo blobAssembly) - { - log.LogMessage (MessageImportance.Low, $"Add: apkName == '{apkName}'"); - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - Blob blob; - if (!blobs.ContainsKey (apkName)) { - blob = new Blob { - Common = new CommonAssemblyBlob (apkName, archiveAssembliesPrefix, log, GetNextBlobID (apkName), globalIndexCounter), - Arch = new ArchAssemblyBlob (apkName, archiveAssembliesPrefix, log, GetNextBlobID (apkName), globalIndexCounter) - }; - - blobs.Add (apkName, blob); - SetIndexBlob (blob.Common); - SetIndexBlob (blob.Arch); - } - - blob = blobs[apkName]; - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - blob.Common.Add (blobAssembly); - } else { - blob.Arch.Add (blobAssembly); - } - - void SetIndexBlob (AssemblyBlob b) - { - log.LogMessage (MessageImportance.Low, $"Checking if {b} is an index blob"); - if (!b.IsIndexBlob) { - log.LogMessage (MessageImportance.Low, $" it is not (ID: {b.ID})"); - return; - } - - log.LogMessage (MessageImportance.Low, $" it is"); - if (indexBlob != null) { - throw new InvalidOperationException ("Index blob already set!"); - } - - indexBlob = b; - } - } - - uint GetNextBlobID (string apkName) - { - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - return apkIds[apkName]++; - } - - public Dictionary> Generate (string outputDirectory) - { - if (blobs.Count == 0) { - return null; - } - - if (indexBlob == null) { - throw new InvalidOperationException ("Index blob not found"); - } - - var globalIndex = new List (); - var ret = new Dictionary> (StringComparer.Ordinal); - string indexBlobApkName = null; - foreach (var kvp in blobs) { - string apkName = kvp.Key; - Blob blob = kvp.Value; - - if (!ret.ContainsKey (apkName)) { - ret.Add (apkName, new List ()); - } - - if (blob.Common == indexBlob || blob.Arch == indexBlob) { - indexBlobApkName = apkName; - } - - GenerateBlob (blob.Common, apkName); - GenerateBlob (blob.Arch, apkName); - } - - string manifestPath = indexBlob.WriteIndex (globalIndex); - ret[indexBlobApkName].Add (manifestPath); - - return ret; - - void GenerateBlob (AssemblyBlob blob, string apkName) - { - blob.Generate (outputDirectory, globalIndex, ret[apkName]); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs similarity index 81% rename from src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs index 46dd2b85001..9346cb5fc78 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs @@ -9,7 +9,7 @@ namespace Xamarin.Android.Tasks { - abstract class AssemblyBlob + abstract class AssemblyStore { // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant @@ -34,12 +34,12 @@ abstract class AssemblyBlob protected string ApkName { get; } protected TaskLoggingHelper Log { get; } - protected AssemblyBlobGlobalIndex GlobalIndexCounter { get; } + protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } public uint ID { get; } - public bool IsIndexBlob => ID == 0; + public bool IsIndexStore => ID == 0; - protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) + protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) { if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); @@ -57,12 +57,12 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg Log = log; } - public abstract void Add (BlobAssemblyInfo blobAssembly); - public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); + public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); + public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - public virtual string WriteIndex (List globalIndex) + public virtual string WriteIndex (List globalIndex) { - if (!IsIndexBlob) { + if (!IsIndexStore) { throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); } @@ -95,7 +95,7 @@ public virtual string WriteIndex (List globalIndex) return indexBlobManifestPath; } - void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) + void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) { using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { @@ -105,18 +105,18 @@ void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) + void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) { uint localEntryCount = 0; - var localAssemblies = new List (); + var localAssemblies = new List (); manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); var seenHashes32 = new HashSet (); var seenHashes64 = new HashSet (); bool haveDuplicates = false; - foreach (AssemblyBlobIndexEntry assembly in globalIndex) { - if (assembly.BlobID == ID) { + foreach (AssemblyStoreIndexEntry assembly in globalIndex) { + if (assembly.StoreID == ID) { localEntryCount++; localAssemblies.Add (assembly); } @@ -126,7 +126,7 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List (globalIndex); - sortedIndex.Sort ((AssemblyBlobIndexEntry a, AssemblyBlobIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (AssemblyBlobIndexEntry entry in sortedIndex) { + var sortedIndex = new List (globalIndex); + sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); + foreach (AssemblyStoreIndexEntry entry in sortedIndex) { WriteHash (entry, entry.NameHash32); } - sortedIndex.Sort ((AssemblyBlobIndexEntry a, AssemblyBlobIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (AssemblyBlobIndexEntry entry in sortedIndex) { + sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); + foreach (AssemblyStoreIndexEntry entry in sortedIndex) { WriteHash (entry, entry.NameHash64); } - void WriteHash (AssemblyBlobIndexEntry entry, ulong hash) + void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) { blobWriter.Write (hash); blobWriter.Write (entry.MappingIndex); blobWriter.Write (entry.LocalBlobIndex); - blobWriter.Write (entry.BlobID); + blobWriter.Write (entry.StoreID); } bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) @@ -177,7 +177,7 @@ bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, Ha } } - protected string GetAssemblyName (BlobAssemblyInfo assembly) + protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) { string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { @@ -187,7 +187,7 @@ protected string GetAssemblyName (BlobAssemblyInfo assembly) return assemblyName; } - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) + protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) { if (globalIndex == null) { throw new ArgumentNullException (nameof (globalIndex)); @@ -197,7 +197,7 @@ protected void Generate (string outputFilePath, List assemblie throw new ArgumentNullException (nameof (blobPaths)); } - if (IsIndexBlob) { + if (IsIndexStore) { indexBlobPath = outputFilePath; } @@ -212,11 +212,11 @@ protected void Generate (string outputFilePath, List assemblie } } - void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) + void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) { - var localAssemblies = new List (); + var localAssemblies = new List (); - if (!IsIndexBlob) { + if (!IsIndexStore) { // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different // way. uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); @@ -225,7 +225,7 @@ void Generate (BinaryWriter writer, List assemblies, List assemblies, List assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); if (addToGlobalIndex) { globalIndex.Add (entry); @@ -262,7 +262,7 @@ void Generate (BinaryWriter writer, List assemblies, List assemblies, uint offsetFixup = 0) + void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) { // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh - foreach (AssemblyBlobIndexEntry assembly in assemblies) { + foreach (AssemblyStoreIndexEntry assembly in assemblies) { Log.LogMessage (MessageImportance.Low, $" => {assembly.Name} before adjustment: data offset == {assembly.DataOffset}"); AdjustOffsets (assembly, offsetFixup); Log.LogMessage (MessageImportance.Low, $" => {assembly.Name} after adjustment: data offset == {assembly.DataOffset}"); @@ -309,7 +309,7 @@ void WriteAssemblyDescriptors (BinaryWriter writer, List } } - void AdjustOffsets (AssemblyBlobIndexEntry assembly, uint offsetFixup) + void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) { if (offsetFixup == 0) { return; @@ -326,15 +326,15 @@ void AdjustOffsets (AssemblyBlobIndexEntry assembly, uint offsetFixup) } } - AssemblyBlobIndexEntry WriteAssembly (BinaryWriter writer, BlobAssemblyInfo assembly, string assemblyName, uint localBlobIndex) + AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) { uint offset; uint size; (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyBlob.Generate for a special case) - var ret = new AssemblyBlobIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { + // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) + var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { DataOffset = offset, DataSize = size, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs similarity index 88% rename from src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index 7e635462c1a..c5c166fb787 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/BlobAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -3,7 +3,7 @@ namespace Xamarin.Android.Tasks { - class BlobAssemblyInfo + class AssemblyStoreAssemblyInfo { public string FilesystemAssemblyPath { get; } public string ArchiveAssemblyPath { get; } @@ -11,7 +11,7 @@ class BlobAssemblyInfo public string ConfigPath { get; private set; } public string Abi { get; } - public BlobAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) + public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) { if (String.IsNullOrEmpty (filesystemAssemblyPath)) { throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs new file mode 100644 index 00000000000..36c2bbf3027 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class AssemblyStoreGenerator + { + sealed class Store + { + public AssemblyStore Common; + public AssemblyStore Arch; + } + + readonly string archiveAssembliesPrefix; + readonly TaskLoggingHelper log; + + // NOTE: when/if we have parallel BuildApk this should become a ConcurrentDictionary + readonly Dictionary stores; + + AssemblyStore indexStore; + + // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time + // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store + readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); + + // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** + // assembly store, thus we need to keep the state here + AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); + + public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) + { + if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); + } + + this.archiveAssembliesPrefix = archiveAssembliesPrefix; + this.log = log; + + stores = new Dictionary (StringComparer.Ordinal); + } + + public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) + { + log.LogMessage (MessageImportance.Low, $"Add: apkName == '{apkName}'"); + if (String.IsNullOrEmpty (apkName)) { + throw new ArgumentException ("must not be null or empty", nameof (apkName)); + } + + Store store; + if (!stores.ContainsKey (apkName)) { + store = new Store { + Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), + Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) + }; + + stores.Add (apkName, store); + SetIndexStore (store.Common); + SetIndexStore (store.Arch); + } + + store = stores[apkName]; + if (String.IsNullOrEmpty (storeAssembly.Abi)) { + store.Common.Add (storeAssembly); + } else { + store.Arch.Add (storeAssembly); + } + + void SetIndexStore (AssemblyStore b) + { + log.LogMessage (MessageImportance.Low, $"Checking if {b} is an index store"); + if (!b.IsIndexStore) { + log.LogMessage (MessageImportance.Low, $" it is not (ID: {b.ID})"); + return; + } + + log.LogMessage (MessageImportance.Low, $" it is"); + if (indexStore != null) { + throw new InvalidOperationException ("Index store already set!"); + } + + indexStore = b; + } + } + + uint GetNextStoreID (string apkName) + { + // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock + if (!apkIds.ContainsKey (apkName)) { + apkIds.Add (apkName, 0); + } + return apkIds[apkName]++; + } + + public Dictionary> Generate (string outputDirectory) + { + if (stores.Count == 0) { + return null; + } + + if (indexStore == null) { + throw new InvalidOperationException ("Index store not found"); + } + + var globalIndex = new List (); + var ret = new Dictionary> (StringComparer.Ordinal); + string indexStoreApkName = null; + foreach (var kvp in stores) { + string apkName = kvp.Key; + Store store = kvp.Value; + + if (!ret.ContainsKey (apkName)) { + ret.Add (apkName, new List ()); + } + + if (store.Common == indexStore || store.Arch == indexStore) { + indexStoreApkName = apkName; + } + + GenerateStore (store.Common, apkName); + GenerateStore (store.Arch, apkName); + } + + string manifestPath = indexStore.WriteIndex (globalIndex); + ret[indexStoreApkName].Add (manifestPath); + + return ret; + + void GenerateStore (AssemblyStore store, string apkName) + { + store.Generate (outputDirectory, globalIndex, ret[apkName]); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs similarity index 75% rename from src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs index 3db2888a8e1..6ce93f11f9d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobGlobalIndex.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs @@ -1,8 +1,8 @@ namespace Xamarin.Android.Tasks { - // This class may seem weird, but it's designed with the specific needs of AssemblyBlob instances in mind and also prepared for thread-safe use in the future, should the + // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the // need arise - sealed class AssemblyBlobGlobalIndex + sealed class AssemblyStoreGlobalIndex { uint value = 0; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs similarity index 84% rename from src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs index 41d5c3bd3d6..59f012f5142 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlobIndexEntry.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs @@ -5,10 +5,10 @@ namespace Xamarin.Android.Tasks { - class AssemblyBlobIndexEntry + class AssemblyStoreIndexEntry { public string Name { get; } - public uint BlobID { get; } + public uint StoreID { get; } public uint MappingIndex { get; } public uint LocalBlobIndex { get; } @@ -25,14 +25,14 @@ class AssemblyBlobIndexEntry public uint ConfigDataOffset { get; set; } public uint ConfigDataSize { get; set; } - public AssemblyBlobIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) + public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) { if (String.IsNullOrEmpty (name)) { throw new ArgumentException ("must not be null or empty", nameof (name)); } Name = name; - BlobID = blobID; + StoreID = blobID; MappingIndex = mappingIndex; LocalBlobIndex = localBlobIndex; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs similarity index 67% rename from src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs index 4a4cddacebd..9b640ccc1a1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyBlob.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs @@ -8,17 +8,17 @@ namespace Xamarin.Android.Tasks { - class CommonAssemblyBlob : AssemblyBlob + class CommonAssemblyStore : AssemblyStore { - readonly List assemblies; + readonly List assemblies; - public CommonAssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyBlobGlobalIndex globalIndexCounter) + public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) { - assemblies = new List (); + assemblies = new List (); } - public override void Add (BlobAssemblyInfo blobAssembly) + public override void Add (AssemblyStoreAssemblyInfo blobAssembly) { if (!String.IsNullOrEmpty (blobAssembly.Abi)) { throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); @@ -28,7 +28,7 @@ public override void Add (BlobAssemblyInfo blobAssembly) assemblies.Add (blobAssembly); } - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) { if (assemblies.Count == 0) { return; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 0a897143596..5013df1de02 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,8 +179,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False - False - True + False + True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> @@ -1512,7 +1512,7 @@ because xbuild doesn't support framework reference assemblies. .so;$(AndroidStoreUncompressedFileExtensions) .dex;$(AndroidStoreUncompressedFileExtensions) - .blob;$(AndroidStoreUncompressedFileExtensions) + .blob;$(AndroidStoreUncompressedFileExtensions) @@ -1599,7 +1599,7 @@ because xbuild doesn't support framework reference assemblies. InstantRunEnabled="$(_InstantRunEnabled)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" - UseAssembliesBlob="$(AndroidUseAssembliesBlob)" + UseAssemblyStore="$(AndroidUseAssemblyStore)" > diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 7daa45587c2..d58e3f3e220 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -43,14 +43,14 @@ ApplicationConfig application_config = { .instant_run_enabled = false, .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, - .have_assemblies_blob = true, + .have_assembly_store = true, .bound_exception_type = 0, // System .package_naming_policy = 0, .environment_variable_count = 0, .system_property_count = 0, .number_of_assemblies_in_apk = 2, .bundled_assembly_name_width = 0, - .number_of_assembly_blobs = 2, + .number_of_assembly_store_files = 2, .android_package_name = "com.xamarin.test", }; @@ -83,21 +83,23 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { }, }; -BlobAssemblyRuntimeData blob_bundled_assemblies[] = { +AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[] = { { .image_data = nullptr, .debug_info_data = nullptr, .config_data = nullptr, + .descriptor = nullptr, }, { .image_data = nullptr, .debug_info_data = nullptr, .config_data = nullptr, + .descriptor = nullptr, } }; -AssemblyBlobRuntimeData assembly_blobs[] = { +AssemblyStoreRuntimeData assembly_stores[] = { { .data_start = nullptr, .assembly_count = 0, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index b2b19b69879..81b617e747e 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -170,83 +170,83 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c } force_inline void -EmbeddedAssemblies::map_assembly_blob (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept { - if (number_of_mapped_blobs >= application_config.number_of_assembly_blobs) { - log_fatal (LOG_ASSEMBLY, "Too many assembly blobs. Expected at most %u", application_config.number_of_assembly_blobs); + if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); abort (); } - md_mmap_info blob_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - auto header = static_cast(blob_map.area); + md_mmap_info assembly_store_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); + auto header = static_cast(assembly_store_map.area); - if (header->magic != BUNDLED_ASSEMBLIES_BLOB_MAGIC) { - log_fatal (LOG_ASSEMBLY, "Blob '%s' is not a valid Xamarin.Android assembly blob file", entry_name.get ()); + if (header->magic != ASSEMBLY_STORE_MAGIC) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid Xamarin.Android assembly store file", entry_name.get ()); abort (); } - if (header->version > BUNDLED_ASSEMBLIES_BLOB_VERSION) { - log_fatal (LOG_ASSEMBLY, "Blob '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); + if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); abort (); } - if (header->blob_id >= application_config.number_of_assembly_blobs) { + if (header->store_id >= application_config.number_of_assembly_store_files) { log_fatal ( LOG_ASSEMBLY, - "Blob '%s' index %u exceeds the number of blobs known at application build time, %u", + "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", entry_name.get (), - header->blob_id, - application_config.number_of_assembly_blobs + header->store_id, + application_config.number_of_assembly_store_files ); abort (); } - AssemblyBlobRuntimeData &rd = assembly_blobs[header->blob_id]; + AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; if (rd.data_start != nullptr) { - log_fatal (LOG_ASSEMBLY, "Blob '%s' has a duplicate ID (%u)", entry_name.get (), header->blob_id); + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); abort (); } - constexpr size_t header_size = sizeof(BundledAssemblyBlobHeader); + constexpr size_t header_size = sizeof(AssemblyStoreHeader); - rd.data_start = static_cast(blob_map.area); + rd.data_start = static_cast(assembly_store_map.area); rd.assembly_count = header->local_entry_count; - rd.assemblies = reinterpret_cast(rd.data_start + header_size); + rd.assemblies = reinterpret_cast(rd.data_start + header_size); number_of_found_assemblies += rd.assembly_count; - if (header->blob_id == 0) { - constexpr size_t bundled_assembly_size = sizeof(BlobBundledAssembly); - constexpr size_t hash_entry_size = sizeof(BlobHashEntry); + if (header->store_id == 0) { + constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); + constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); - index_blob_header = header; + index_assembly_store_header = header; size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); if constexpr (std::is_same_v) { - blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); + assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); } else { - blob_assembly_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); + assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); } } - number_of_mapped_blobs++; + number_of_mapped_assembly_stores++; have_and_want_debug_symbols = register_debug_symbols; } force_inline void -EmbeddedAssemblies::zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { - if (all_blobs_found ()) { + if (all_required_zip_entries_found ()) { return; } dynamic_local_string entry_name; - bool common_assembly_blob_found = false; - bool arch_assembly_blob_found = false; + bool common_assembly_store_found = false; + bool arch_assembly_store_found = false; - log_debug (LOG_ASSEMBLY, "Looking for blobs in APK (common: '%s'; arch-specific: '%s')", bundled_assemblies_common_blob_name.data (), bundled_assemblies_arch_blob_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); for (size_t i = 0; i < num_entries; i++) { - if (all_blobs_found ()) { + if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; break; } @@ -256,14 +256,14 @@ EmbeddedAssemblies::zip_load_blob_assembly_entries (std::vector const& continue; } - if (!common_assembly_blob_found && utils.ends_with (entry_name, bundled_assemblies_common_blob_name)) { - common_assembly_blob_found = true; - map_assembly_blob (entry_name, state); + if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { + common_assembly_store_found = true; + map_assembly_store (entry_name, state); } - if (!arch_assembly_blob_found && utils.ends_with (entry_name, bundled_assemblies_arch_blob_name)) { - arch_assembly_blob_found = true; - map_assembly_blob (entry_name, state); + if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { + arch_assembly_store_found = true; + map_assembly_store (entry_name, state); } } } @@ -305,8 +305,8 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus exit (FATAL_EXIT_NO_ASSEMBLIES); } - if (application_config.have_assemblies_blob) { - zip_load_blob_assembly_entries (buf, cd_entries, state); + if (application_config.have_assembly_store) { + zip_load_assembly_store_entries (buf, cd_entries, state); } else { zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 7fbfa47356b..698ea504581 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -141,9 +141,9 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u } force_inline void -EmbeddedAssemblies::get_assembly_data (BlobAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +EmbeddedAssemblies::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { - get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); + get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); } #if defined (NET6) @@ -342,11 +342,11 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin return nullptr; } -force_inline const BlobHashEntry* -EmbeddedAssemblies::find_blob_assembly_entry (hash_t hash, const BlobHashEntry *entries, size_t entry_count) noexcept +force_inline const AssemblyStoreHashEntry* +EmbeddedAssemblies::find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept { hash_t entry_hash; - const BlobHashEntry *ret; + const AssemblyStoreHashEntry *ret; while (entry_count > 0) { ret = entries + (entry_count / 2); @@ -372,7 +372,7 @@ EmbeddedAssemblies::find_blob_assembly_entry (hash_t hash, const BlobHashEntry * } force_inline MonoAssembly* -EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept +EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { size_t len = name.length (); @@ -381,9 +381,9 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_stringmapping_index]; + AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; if (assembly_runtime_info.image_data == nullptr) { - if (hash_entry->blob_id >= application_config.number_of_assembly_blobs) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly blob ID %u, exceeds the maximum of %u", hash_entry->blob_id, application_config.number_of_assembly_blobs - 1); + if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); abort (); } - AssemblyBlobRuntimeData &rd = assembly_blobs[hash_entry->blob_id]; - if (hash_entry->local_blob_index >= rd.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid index %u into local blob descriptor array", hash_entry->local_blob_index); + AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; + if (hash_entry->local_store_index >= rd.assembly_count) { + log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); } - BlobBundledAssembly *bba = &rd.assemblies[hash_entry->local_blob_index]; + AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; // The assignments here don't need to be atomic, the value will always be the same, so even if two threads // arrive here at the same time, nothing bad will happen. @@ -444,6 +444,11 @@ EmbeddedAssemblies::blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; - MonoAssembly* blob_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; + MonoAssembly* assembly_store_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; MonoAssembly* open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only); template @@ -199,11 +199,11 @@ namespace xamarin::android::internal { #endif // ndef NET6 static void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; - static void get_assembly_data (BlobAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; + static void get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; - void zip_load_blob_assembly_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; + void zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); @@ -249,10 +249,10 @@ namespace xamarin::android::internal { return assemblies_prefix_override != nullptr ? static_cast(strlen (assemblies_prefix_override)) : sizeof(assemblies_prefix) - 1; } - bool all_blobs_found () const noexcept + bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_blobs == application_config.number_of_assembly_blobs + number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files #if defined (NET6) && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) #endif // NET6 @@ -275,8 +275,8 @@ namespace xamarin::android::internal { void set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; - void map_assembly_blob (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; - const BlobHashEntry* find_blob_assembly_entry (hash_t hash, const BlobHashEntry *entries, size_t entry_count) noexcept; + void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; + const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -298,11 +298,11 @@ namespace xamarin::android::internal { md_mmap_info runtime_config_blob_mmap{}; bool runtime_config_blob_found = false; #endif // def NET6 - uint32_t number_of_mapped_blobs = 0; + uint32_t number_of_mapped_assembly_stores = 0; bool need_to_scan_more_apks = true; - BundledAssemblyBlobHeader *index_blob_header = nullptr; - BlobHashEntry *blob_assembly_hashes; + AssemblyStoreHeader *index_assembly_store_header = nullptr; + AssemblyStoreHashEntry *assembly_store_hashes; }; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 893cd67220d..c3c2441c140 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -438,7 +438,7 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, } } - embeddedAssemblies.ensure_valid_blobs (); + embeddedAssemblies.ensure_valid_assembly_stores (); } #if defined (DEBUG) && !defined (WINDOWS) diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 64a4aae7da0..41a5aee422d 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -10,8 +10,9 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian -static constexpr uint32_t BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian -static constexpr uint32_t BUNDLED_ASSEMBLIES_BLOB_VERSION = 1; // Increase whenever an incompatible change is made to the blob format +static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the + // assembly store format static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -109,41 +110,41 @@ struct XamarinAndroidBundledAssembly final }; // -// Blob format +// Assembly store format // // The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. // The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the -// blob but also into the `uint8_t*` `blob_bundled_assemblies*` arrays. +// store but also into the `uint8_t*` `assembly_store_bundled_assemblies*` arrays. // -// This way the `assemblies` array in the blob can remain read only, because we write the "mapped" assembly +// This way the `assemblies` array in the store can remain read only, because we write the "mapped" assembly // pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. // -// Each blob has a unique ID assigned, which is an index into an array of pointers to arrays which store -// individual assembly addresses. Only blob with ID 0 comes with the hashes32 and hashes64 arrays. This is -// done to make it possible to use a single sorted array to find assemblies insted of each blob having its +// Each store has a unique ID assigned, which is an index into an array of pointers to arrays which store +// individual assembly addresses. Only store with ID 0 comes with the hashes32 and hashes64 arrays. This is +// done to make it possible to use a single sorted array to find assemblies insted of each store having its // own sorted array of hashes, which would require several binary searches instead of just one. // -// BundledAssemblyBlobHeader header; -// BlobBundledAssembly assemblies[header.local_entry_count]; -// BlobHashEntry hashes32[header.global_entry_count]; // only in blob with ID 0 -// BlobHashEntry hashes64[header.global_entry_count]; // only in blob with ID 0 +// AssemblyStoreHeader header; +// AssemblyStoreAssemblyDescriptor assemblies[header.local_entry_count]; +// AssemblyStoreHashEntry hashes32[header.global_entry_count]; // only in assembly store with ID 0 +// AssemblyStoreHashEntry hashes64[header.global_entry_count]; // only in assembly store with ID 0 // [DATA] // // -// The structures which are found in the blob files must be packed, to avoid problems when calculating offsets (runtime +// The structures which are found in the store files must be packed to avoid problems when calculating offsets (runtime // size of a structure can be different than the real data size) // -struct [[gnu::packed]] BundledAssemblyBlobHeader final +struct [[gnu::packed]] AssemblyStoreHeader final { uint32_t magic; uint32_t version; uint32_t local_entry_count; uint32_t global_entry_count; - uint32_t blob_id; + uint32_t store_id; }; -struct [[gnu::packed]] BlobHashEntry final +struct [[gnu::packed]] AssemblyStoreHashEntry final { union { uint64_t hash64; @@ -151,17 +152,17 @@ struct [[gnu::packed]] BlobHashEntry final }; // Index into the array with pointers to assembly data. - // It **must** be unique across all the blobs from all the apks + // It **must** be unique across all the stores from all the apks uint32_t mapping_index; - // Index into the array with assembly descriptors inside a blob - uint32_t local_blob_index; + // Index into the array with assembly descriptors inside a store + uint32_t local_store_index; - // Index into the array with blob mmap addresses - uint32_t blob_id; + // Index into the array with assembly store mmap addresses + uint32_t store_id; }; -struct [[gnu::packed]] BlobBundledAssembly final +struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final { uint32_t data_offset; uint32_t data_size; @@ -173,19 +174,19 @@ struct [[gnu::packed]] BlobBundledAssembly final uint32_t config_data_size; }; -struct AssemblyBlobRuntimeData final +struct AssemblyStoreRuntimeData final { uint8_t *data_start; uint32_t assembly_count; - BlobBundledAssembly *assemblies; + AssemblyStoreAssemblyDescriptor *assemblies; }; -struct BlobAssemblyRuntimeData final +struct AssemblyStoreSingleAssemblyRuntimeData final { uint8_t *image_data; uint8_t *debug_info_data; uint8_t *config_data; - BlobBundledAssembly *descriptor; + AssemblyStoreAssemblyDescriptor *descriptor; }; struct ApplicationConfig @@ -198,14 +199,14 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; - bool have_assemblies_blob; + bool have_assembly_store; uint8_t bound_exception_type; uint32_t package_naming_policy; uint32_t environment_variable_count; uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; - uint32_t number_of_assembly_blobs; + uint32_t number_of_assembly_store_files; const char *android_package_name; }; @@ -229,7 +230,7 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API BlobAssemblyRuntimeData blob_bundled_assemblies[]; -MONO_API AssemblyBlobRuntimeData assembly_blobs[]; +MONO_API AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; +MONO_API AssemblyStoreRuntimeData assembly_stores[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index ab5c0b9fe88..a180a6b851d 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -22,7 +22,7 @@ - + diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index e0eff168a85..7722a698b13 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -63,7 +63,7 @@ public void OneTimeSetUp () } }; - lib.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + lib.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); var bytes = new byte [1024]; app = new XamarinFormsMapsApplicationProject { @@ -87,7 +87,7 @@ public void OneTimeSetUp () app.SetProperty (app.ReleaseProperties, "AndroidPackageFormat", "aab"); app.SetAndroidSupportedAbis (Abis); app.SetProperty ("AndroidBundleConfigurationFile", "buildConfig.json"); - app.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); + app.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); libBuilder = CreateDllBuilder (Path.Combine (path, lib.ProjectName), cleanupOnDispose: true); Assert.IsTrue (libBuilder.Build (lib), "Library build should have succeeded."); @@ -143,7 +143,7 @@ public void BaseZip () "resources.pb", }; - string blobEntryPrefix = ArchiveAssemblyHelper.DefaultBlobEntryPrefix; + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; if (usesAssemblyBlobs) { expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); @@ -230,7 +230,7 @@ public void AppBundle () "BundleConfig.pb", }; - string blobEntryPrefix = ArchiveAssemblyHelper.DefaultBlobEntryPrefix; + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; if (usesAssemblyBlobs) { expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); diff --git a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj index 4500bf8c7b2..a10607edb93 100644 --- a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj +++ b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj @@ -20,7 +20,7 @@ armeabi-v7a;x86 arm64-v8a;x86 True - False + False true <_AndroidCheckedBuild Condition=" '$(UseASAN)' != '' ">asan <_AndroidCheckedBuild Condition=" '$(UseUBSAN)' != '' ">ubsan diff --git a/tools/assembly-blob-reader/BlobExplorer.cs b/tools/assembly-blob-reader/BlobExplorer.cs deleted file mode 100644 index c2b46b13aba..00000000000 --- a/tools/assembly-blob-reader/BlobExplorer.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Tools.Zip; - -namespace Xamarin.Android.AssemblyBlobReader -{ - class BlobExplorer - { - BlobReader? indexBlob; - BlobManifestReader? manifest; - int numberOfBlobs = 0; - Action? logger; - bool keepBlobInMemory; - - public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); - public IDictionary AssembliesByHash32 { get; } = new Dictionary (); - public IDictionary AssembliesByHash64 { get; } = new Dictionary (); - public List Assemblies { get; } = new List (); - public IDictionary> Blobs { get; } = new SortedDictionary> (); - public string BlobPath { get; } - public string BlobSetName { get; } - public bool HasErrors { get; private set; } - public bool HasWarnings { get; private set; } - - public bool IsCompleteSet => indexBlob != null && manifest != null; - public int NumberOfBlobs => numberOfBlobs; - - // blobPath can point to: - // - // aab - // apk - // index blob (e.g. base_assemblies.blob) - // arch blob (e.g. base_assemblies.arm64_v8a.blob) - // blob manifest (e.g. base_assemblies.manifest) - // blob base name (e.g. base or base_assemblies) - // - // In each case the whole set of blobs and manifests will be read (if available). Search for the various members of the blob set (common/main blob, arch blobs, - // manifest) is based on this naming convention: - // - // {BASE_NAME}[.ARCH_NAME].{blob|manifest} - // - // Whichever file is referenced in `blobPath`, the BASE_NAME component is extracted and all the found files are read. - // If `blobPath` points to an aab or an apk, BASE_NAME will always be `assemblies` - // - public BlobExplorer (string blobPath, Action? customLogger = null, bool keepBlobInMemory = false) - { - if (String.IsNullOrEmpty (blobPath)) { - throw new ArgumentException ("must not be null or empty", nameof (blobPath)); - } - - if (Directory.Exists (blobPath)) { - throw new ArgumentException ($"'{blobPath}' points to a directory", nameof (blobPath)); - } - - logger = customLogger; - this.keepBlobInMemory = keepBlobInMemory; - BlobPath = blobPath; - string? extension = Path.GetExtension (blobPath); - string? baseName = null; - - if (String.IsNullOrEmpty (extension)) { - baseName = GetBaseNameNoExtension (blobPath); - } else { - baseName = GetBaseNameHaveExtension (blobPath, extension); - } - - if (String.IsNullOrEmpty (baseName)) { - throw new InvalidOperationException ($"Unable to determine base name of a blob set from path '{blobPath}'"); - } - - BlobSetName = baseName; - if (!IsAndroidArchive (extension)) { - string? directoryName = Path.GetDirectoryName (blobPath); - if (String.IsNullOrEmpty (directoryName)) { - directoryName = "."; - } - - ReadBlobSetFromFilesystem (baseName, directoryName); - } else { - ReadBlobSetFromArchive (baseName, blobPath, extension); - } - - ProcessBlobs (); - } - - void Logger (BlobExplorerLogLevel level, string message) - { - if (level == BlobExplorerLogLevel.Error) { - HasErrors = true; - } else if (level == BlobExplorerLogLevel.Warning) { - HasWarnings = true; - } - - if (logger != null) { - logger (level, message); - } else { - DefaultLogger (level, message); - } - } - - void DefaultLogger (BlobExplorerLogLevel level, string message) - { - Console.WriteLine ($"{level}: {message}"); - } - - void ProcessBlobs () - { - if (Blobs.Count == 0 || indexBlob == null) { - return; - } - - ProcessIndex (indexBlob.GlobalIndex32, "32", (BlobHashEntry he, BlobAssembly assembly) => { - assembly.Hash32 = (uint)he.Hash; - assembly.RuntimeIndex = he.MappingIndex; - - if (manifest != null && manifest.EntriesByHash32.TryGetValue (assembly.Hash32, out BlobManifestEntry? me) && me != null) { - assembly.Name = me.Name; - } - - if (!AssembliesByHash32.ContainsKey (assembly.Hash32)) { - AssembliesByHash32.Add (assembly.Hash32, assembly); - } - }); - - ProcessIndex (indexBlob.GlobalIndex64, "64", (BlobHashEntry he, BlobAssembly assembly) => { - assembly.Hash64 = he.Hash; - if (assembly.RuntimeIndex != he.MappingIndex) { - Logger (BlobExplorerLogLevel.Warning, $"assembly with hashes 0x{assembly.Hash32} and 0x{assembly.Hash64} has a different 32-bit runtime index ({assembly.RuntimeIndex}) than the 64-bit runtime index({he.MappingIndex})"); - } - - if (manifest != null && manifest.EntriesByHash64.TryGetValue (assembly.Hash64, out BlobManifestEntry? me) && me != null) { - if (String.IsNullOrEmpty (assembly.Name)) { - Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} did not match any assembly name in the manifest"); - assembly.Name = me.Name; - if (String.IsNullOrEmpty (assembly.Name)) { - Logger (BlobExplorerLogLevel.Warning, $"64-bit hash 0x{assembly.Hash64:x} did not match any assembly name in the manifest"); - } - } else if (String.Compare (assembly.Name, me.Name, StringComparison.Ordinal) != 0) { - Logger (BlobExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} maps to assembly name '{assembly.Name}', however 64-bit hash 0x{assembly.Hash64:x} for the same entry matches assembly name '{me.Name}'"); - } - } - - if (!AssembliesByHash64.ContainsKey (assembly.Hash64)) { - AssembliesByHash64.Add (assembly.Hash64, assembly); - } - }); - - foreach (var kvp in Blobs) { - List list = kvp.Value; - if (list.Count < 2) { - continue; - } - - BlobReader template = list[0]; - for (int i = 1; i < list.Count; i++) { - BlobReader other = list[i]; - if (!template.HasIdenticalContent (other)) { - Logger (BlobExplorerLogLevel.Error, $"Blob ID {template.BlobID} for architecture {other.Arch} is not identical to other blobs with the same ID"); - } - } - } - - void ProcessIndex (List index, string bitness, Action assemblyHandler) - { - foreach (BlobHashEntry he in index) { - if (!Blobs.TryGetValue (he.BlobID, out List? blobList) || blobList == null) { - Logger (BlobExplorerLogLevel.Warning, $"blob with id {he.BlobID} not part of the set"); - continue; - } - - foreach (BlobReader blob in blobList) { - if (he.LocalBlobIndex >= (uint)blob.Assemblies.Count) { - Logger (BlobExplorerLogLevel.Warning, $"{bitness}-bit index entry with hash 0x{he.Hash:x} has invalid blob {blob.BlobID} index {he.LocalBlobIndex} (maximum allowed is {blob.Assemblies.Count})"); - continue; - } - - BlobAssembly assembly = blob.Assemblies[(int)he.LocalBlobIndex]; - assemblyHandler (he, assembly); - - if (!AssembliesByName.ContainsKey (assembly.Name)) { - AssembliesByName.Add (assembly.Name, assembly); - } - } - } - } - } - - void ReadBlobSetFromArchive (string baseName, string archivePath, string extension) - { - string basePathInArchive; - - if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { - basePathInArchive = "base/root/assemblies"; - } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { - basePathInArchive = "assemblies"; - } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { - basePathInArchive = "root/assemblies"; - } else { - throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); - } - - basePathInArchive = $"{basePathInArchive}/{baseName}."; - using (ZipArchive archive = ZipArchive.Open (archivePath, FileMode.Open)) { - ReadBlobSetFromArchive (archive, basePathInArchive); - } - } - - void ReadBlobSetFromArchive (ZipArchive archive, string basePathInArchive) - { - foreach (ZipEntry entry in archive) { - if (!entry.FullName.StartsWith (basePathInArchive, StringComparison.Ordinal)) { - continue; - } - - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - - if (entry.FullName.EndsWith (".blob", StringComparison.Ordinal)) { - AddBlob (new BlobReader (stream, GetBlobArch (entry.FullName), keepBlobInMemory)); - } else if (entry.FullName.EndsWith (".manifest", StringComparison.Ordinal)) { - manifest = new BlobManifestReader (stream); - } - } - } - } - - void AddBlob (BlobReader reader) - { - if (reader.HasGlobalIndex) { - indexBlob = reader; - } - - List? blobList; - if (!Blobs.TryGetValue (reader.BlobID, out blobList)) { - blobList = new List (); - Blobs.Add (reader.BlobID, blobList); - } - blobList.Add (reader); - - Assemblies.AddRange (reader.Assemblies); - } - - string? GetBlobArch (string path) - { - string? arch = Path.GetFileNameWithoutExtension (path); - if (!String.IsNullOrEmpty (arch)) { - arch = Path.GetExtension (arch); - if (!String.IsNullOrEmpty (arch)) { - arch = arch.Substring (1); - } - } - - return arch; - } - - void ReadBlobSetFromFilesystem (string baseName, string setPath) - { - foreach (string de in Directory.EnumerateFiles (setPath, $"{baseName}.*", SearchOption.TopDirectoryOnly)) { - string? extension = Path.GetExtension (de); - if (String.IsNullOrEmpty (extension)) { - continue; - } - - if (String.Compare (".blob", extension, StringComparison.OrdinalIgnoreCase) == 0) { - AddBlob (ReadBlob (de)); - } else if (String.Compare (".manifest", extension, StringComparison.OrdinalIgnoreCase) == 0) { - manifest = ReadManifest (de); - } - } - - BlobReader ReadBlob (string filePath) - { - string? arch = GetBlobArch (filePath); - using (var fs = File.OpenRead (filePath)) { - return CreateBlobReader (fs, arch); - } - } - - BlobManifestReader ReadManifest (string filePath) - { - using (var fs = File.OpenRead (filePath)) { - return new BlobManifestReader (fs); - } - } - } - - BlobReader CreateBlobReader (Stream input, string? arch) - { - numberOfBlobs++; - return new BlobReader (input, arch, keepBlobInMemory); - } - - bool IsAndroidArchive (string extension) - { - return - String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0 || - String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0 || - String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0; - } - - string GetBaseNameHaveExtension (string blobPath, string extension) - { - if (IsAndroidArchive (extension)) { - return "assemblies"; - } - - string fileName = Path.GetFileNameWithoutExtension (blobPath); - int dot = fileName.IndexOf ('.'); - if (dot >= 0) { - return fileName.Substring (0, dot); - } - - return fileName; - } - - string GetBaseNameNoExtension (string blobPath) - { - string fileName = Path.GetFileName (blobPath); - if (fileName.EndsWith ("_assemblies", StringComparison.OrdinalIgnoreCase)) { - return fileName; - } - return $"{fileName}_assemblies"; - } - } -} diff --git a/tools/assembly-blob-reader/BlobExplorerLogLevel.cs b/tools/assembly-blob-reader/BlobExplorerLogLevel.cs deleted file mode 100644 index 203be0a0429..00000000000 --- a/tools/assembly-blob-reader/BlobExplorerLogLevel.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Xamarin.Android.AssemblyBlobReader -{ - enum BlobExplorerLogLevel - { - Debug, - Info, - Warning, - Error, - } -} diff --git a/tools/assembly-blob-reader/BlobHashEntry.cs b/tools/assembly-blob-reader/BlobHashEntry.cs deleted file mode 100644 index 856466e3bc7..00000000000 --- a/tools/assembly-blob-reader/BlobHashEntry.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using System.IO; - -namespace Xamarin.Android.AssemblyBlobReader -{ - class BlobHashEntry - { - public bool Is32Bit { get; } - - public ulong Hash { get; } - public uint MappingIndex { get; } - public uint LocalBlobIndex { get; } - public uint BlobID { get; } - - internal BlobHashEntry (BinaryReader reader, bool is32Bit) - { - Is32Bit = is32Bit; - - Hash = reader.ReadUInt64 (); - MappingIndex = reader.ReadUInt32 (); - LocalBlobIndex = reader.ReadUInt32 (); - BlobID = reader.ReadUInt32 (); - } - } -} diff --git a/tools/assembly-blob-reader/BlobReader.cs b/tools/assembly-blob-reader/BlobReader.cs deleted file mode 100644 index da9680ecddc..00000000000 --- a/tools/assembly-blob-reader/BlobReader.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Text; - -namespace Xamarin.Android.AssemblyBlobReader -{ - class BlobReader - { - // These two constants must be identical to the native ones in src/monodroid/jni/xamarin-app.hh - const uint BUNDLED_ASSEMBLIES_BLOB_MAGIC = 0x41424158; // 'XABA', little-endian - const uint BUNDLED_ASSEMBLIES_BLOB_VERSION = 1; // The highest format version this reader understands - - MemoryStream? blobData; - - public uint Version { get; private set; } - public uint LocalEntryCount { get; private set; } - public uint GlobalEntryCount { get; private set; } - public uint BlobID { get; private set; } - public List Assemblies { get; } - public List GlobalIndex32 { get; } = new List (); - public List GlobalIndex64 { get; } = new List (); - public string Arch { get; } - - public bool HasGlobalIndex => BlobID == 0; - - public BlobReader (Stream blob, string? arch = null, bool keepBlobInMemory = false) - { - Arch = arch ?? String.Empty; - - blob.Seek (0, SeekOrigin.Begin); - if (keepBlobInMemory) { - blobData = new MemoryStream (); - blob.CopyTo (blobData); - blobData.Flush (); - blob.Seek (0, SeekOrigin.Begin); - } - - using (var reader = new BinaryReader (blob, Encoding.UTF8, leaveOpen: true)) { - ReadHeader (reader); - - Assemblies = new List (); - ReadLocalEntries (reader, Assemblies); - if (HasGlobalIndex) { - ReadGlobalIndex (reader, GlobalIndex32, GlobalIndex64); - } - } - } - - internal void ExtractAssemblyImage (BlobAssembly assembly, string outputFilePath) - { - SaveDataToFile (outputFilePath, assembly.DataOffset, assembly.DataSize); - } - - internal void ExtractAssemblyImage (BlobAssembly assembly, Stream output) - { - SaveDataToStream (output, assembly.DataOffset, assembly.DataSize); - } - - internal void ExtractAssemblyDebugData (BlobAssembly assembly, string outputFilePath) - { - if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { - return; - } - SaveDataToFile (outputFilePath, assembly.DebugDataOffset, assembly.DebugDataSize); - } - - internal void ExtractAssemblyDebugData (BlobAssembly assembly, Stream output) - { - if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { - return; - } - SaveDataToStream (output, assembly.DebugDataOffset, assembly.DebugDataSize); - } - - internal void ExtractAssemblyConfig (BlobAssembly assembly, string outputFilePath) - { - if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { - return; - } - - SaveDataToFile (outputFilePath, assembly.ConfigDataOffset, assembly.ConfigDataSize); - } - - internal void ExtractAssemblyConfig (BlobAssembly assembly, Stream output) - { - if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { - return; - } - SaveDataToStream (output, assembly.ConfigDataOffset, assembly.ConfigDataSize); - } - - void SaveDataToFile (string outputFilePath, uint offset, uint size) - { - EnsureBlobDataAvailable (); - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - SaveDataToStream (fs, offset, size); - } - } - - void SaveDataToStream (Stream output, uint offset, uint size) - { - EnsureBlobDataAvailable (); - ArrayPool pool = ArrayPool.Shared; - - blobData!.Seek (offset, SeekOrigin.Begin); - byte[] buf = pool.Rent (16384); - int nread; - long toRead = size; - while (toRead > 0 && (nread = blobData.Read (buf, 0, buf.Length)) > 0) { - if (nread > toRead) { - nread = (int)toRead; - } - - output.Write (buf, 0, nread); - toRead -= nread; - } - output.Flush (); - pool.Return (buf); - } - - void EnsureBlobDataAvailable () - { - if (blobData != null) { - return; - } - - throw new InvalidOperationException ("Blob data not available. BlobReader/BlobExplorer must be instantiated with the `keepBlobInMemory` argument set to `true`"); - } - - public bool HasIdenticalContent (BlobReader other) - { - return - other.Version == Version && - other.LocalEntryCount == LocalEntryCount && - other.GlobalEntryCount == GlobalEntryCount && - other.BlobID == BlobID && - other.Assemblies.Count == Assemblies.Count && - other.GlobalIndex32.Count == GlobalIndex32.Count && - other.GlobalIndex64.Count == GlobalIndex64.Count; - } - - void ReadHeader (BinaryReader reader) - { - uint magic = reader.ReadUInt32 (); - if (magic != BUNDLED_ASSEMBLIES_BLOB_MAGIC) { - throw new InvalidOperationException ("Invalid header magic number"); - } - - Version = reader.ReadUInt32 (); - if (Version == 0) { - throw new InvalidOperationException ("Invalid version number: 0"); - } - - if (Version > BUNDLED_ASSEMBLIES_BLOB_VERSION) { - throw new InvalidOperationException ($"Blob format version {Version} is higher than the one understood by this reader, {BUNDLED_ASSEMBLIES_BLOB_VERSION}"); - } - - LocalEntryCount = reader.ReadUInt32 (); - GlobalEntryCount = reader.ReadUInt32 (); - BlobID = reader.ReadUInt32 (); - } - - void ReadLocalEntries (BinaryReader reader, List assemblies) - { - for (uint i = 0; i < LocalEntryCount; i++) { - assemblies.Add (new BlobAssembly (reader, this)); - } - } - - void ReadGlobalIndex (BinaryReader reader, List index32, List index64) - { - ReadIndex (true, index32); - ReadIndex (true, index64); - - void ReadIndex (bool is32Bit, List index) { - for (uint i = 0; i < GlobalEntryCount; i++) { - index.Add (new BlobHashEntry (reader, is32Bit)); - } - } - } - } -} diff --git a/tools/assembly-blob-reader/Directory.Build.targets b/tools/assembly-blob-reader/Directory.Build.targets deleted file mode 100644 index ec0761f8371..00000000000 --- a/tools/assembly-blob-reader/Directory.Build.targets +++ /dev/null @@ -1,6 +0,0 @@ - - - - - diff --git a/tools/assembly-blob-reader/BlobAssembly.cs b/tools/assembly-store-reader/AssemblyStoreAssembly.cs similarity index 73% rename from tools/assembly-blob-reader/BlobAssembly.cs rename to tools/assembly-store-reader/AssemblyStoreAssembly.cs index 2a6fdf836b0..b83be2acbf4 100644 --- a/tools/assembly-blob-reader/BlobAssembly.cs +++ b/tools/assembly-store-reader/AssemblyStoreAssembly.cs @@ -1,9 +1,9 @@ using System; using System.IO; -namespace Xamarin.Android.AssemblyBlobReader +namespace Xamarin.Android.AssemblyStore { - class BlobAssembly + class AssemblyStoreAssembly { public uint DataOffset { get; } public uint DataSize { get; } @@ -17,14 +17,14 @@ class BlobAssembly public string Name { get; set; } = String.Empty; public uint RuntimeIndex { get; set; } - public BlobReader Blob { get; } + public AssemblyStoreReader Store { get; } public string DllName => MakeFileName ("dll"); public string PdbName => MakeFileName ("pdb"); public string ConfigName => MakeFileName ("dll.config"); - internal BlobAssembly (BinaryReader reader, BlobReader blob) + internal AssemblyStoreAssembly (BinaryReader reader, AssemblyStoreReader store) { - Blob = blob; + Store = store; DataOffset = reader.ReadUInt32 (); DataSize = reader.ReadUInt32 (); @@ -36,32 +36,32 @@ internal BlobAssembly (BinaryReader reader, BlobReader blob) public void ExtractImage (string outputDirPath, string? fileName = null) { - Blob.ExtractAssemblyImage (this, MakeOutputFilePath (outputDirPath, "dll", fileName)); + Store.ExtractAssemblyImage (this, MakeOutputFilePath (outputDirPath, "dll", fileName)); } public void ExtractImage (Stream output) { - Blob.ExtractAssemblyImage (this, output); + Store.ExtractAssemblyImage (this, output); } public void ExtractDebugData (string outputDirPath, string? fileName = null) { - Blob.ExtractAssemblyDebugData (this, MakeOutputFilePath (outputDirPath, "pdb", fileName)); + Store.ExtractAssemblyDebugData (this, MakeOutputFilePath (outputDirPath, "pdb", fileName)); } public void ExtractDebugData (Stream output) { - Blob.ExtractAssemblyDebugData (this, output); + Store.ExtractAssemblyDebugData (this, output); } public void ExtractConfig (string outputDirPath, string? fileName = null) { - Blob.ExtractAssemblyConfig (this, MakeOutputFilePath (outputDirPath, "dll.config", fileName)); + Store.ExtractAssemblyConfig (this, MakeOutputFilePath (outputDirPath, "dll.config", fileName)); } public void ExtractConfig (Stream output) { - Blob.ExtractAssemblyConfig (this, output); + Store.ExtractAssemblyConfig (this, output); } string MakeOutputFilePath (string outputDirPath, string extension, string? fileName) diff --git a/tools/assembly-store-reader/AssemblyStoreExplorer.cs b/tools/assembly-store-reader/AssemblyStoreExplorer.cs new file mode 100644 index 00000000000..ca8722301ea --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreExplorer.cs @@ -0,0 +1,327 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreExplorer + { + AssemblyStoreReader? indexStore; + AssemblyStoreManifestReader? manifest; + int numberOfStores = 0; + Action? logger; + bool keepStoreInMemory; + + public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); + public IDictionary AssembliesByHash32 { get; } = new Dictionary (); + public IDictionary AssembliesByHash64 { get; } = new Dictionary (); + public List Assemblies { get; } = new List (); + public IDictionary> Stores { get; } = new SortedDictionary> (); + public string StorePath { get; } + public string StoreSetName { get; } + public bool HasErrors { get; private set; } + public bool HasWarnings { get; private set; } + + public bool IsCompleteSet => indexStore != null && manifest != null; + public int NumberOfStores => numberOfStores; + + // storePath can point to: + // + // aab + // apk + // index store (e.g. base_assemblies.store) + // arch store (e.g. base_assemblies.arm64_v8a.store) + // store manifest (e.g. base_assemblies.manifest) + // store base name (e.g. base or base_assemblies) + // + // In each case the whole set of stores and manifests will be read (if available). Search for the various members of the store set (common/main store, arch stores, + // manifest) is based on this naming convention: + // + // {BASE_NAME}[.ARCH_NAME].{store|manifest} + // + // Whichever file is referenced in `storePath`, the BASE_NAME component is extracted and all the found files are read. + // If `storePath` points to an aab or an apk, BASE_NAME will always be `assemblies` + // + public AssemblyStoreExplorer (string storePath, Action? customLogger = null, bool keepStoreInMemory = false) + { + if (String.IsNullOrEmpty (storePath)) { + throw new ArgumentException ("must not be null or empty", nameof (storePath)); + } + + if (Directory.Exists (storePath)) { + throw new ArgumentException ($"'{storePath}' points to a directory", nameof (storePath)); + } + + logger = customLogger; + this.keepStoreInMemory = keepStoreInMemory; + StorePath = storePath; + string? extension = Path.GetExtension (storePath); + string? baseName = null; + + if (String.IsNullOrEmpty (extension)) { + baseName = GetBaseNameNoExtension (storePath); + } else { + baseName = GetBaseNameHaveExtension (storePath, extension); + } + + if (String.IsNullOrEmpty (baseName)) { + throw new InvalidOperationException ($"Unable to determine base name of a store set from path '{storePath}'"); + } + + StoreSetName = baseName; + if (!IsAndroidArchive (extension)) { + string? directoryName = Path.GetDirectoryName (storePath); + if (String.IsNullOrEmpty (directoryName)) { + directoryName = "."; + } + + ReadStoreSetFromFilesystem (baseName, directoryName); + } else { + ReadStoreSetFromArchive (baseName, storePath, extension); + } + + ProcessStores (); + } + + void Logger (AssemblyStoreExplorerLogLevel level, string message) + { + if (level == AssemblyStoreExplorerLogLevel.Error) { + HasErrors = true; + } else if (level == AssemblyStoreExplorerLogLevel.Warning) { + HasWarnings = true; + } + + if (logger != null) { + logger (level, message); + } else { + DefaultLogger (level, message); + } + } + + void DefaultLogger (AssemblyStoreExplorerLogLevel level, string message) + { + Console.WriteLine ($"{level}: {message}"); + } + + void ProcessStores () + { + if (Stores.Count == 0 || indexStore == null) { + return; + } + + ProcessIndex (indexStore.GlobalIndex32, "32", (AssemblyStoreHashEntry he, AssemblyStoreAssembly assembly) => { + assembly.Hash32 = (uint)he.Hash; + assembly.RuntimeIndex = he.MappingIndex; + + if (manifest != null && manifest.EntriesByHash32.TryGetValue (assembly.Hash32, out AssemblyStoreManifestEntry? me) && me != null) { + assembly.Name = me.Name; + } + + if (!AssembliesByHash32.ContainsKey (assembly.Hash32)) { + AssembliesByHash32.Add (assembly.Hash32, assembly); + } + }); + + ProcessIndex (indexStore.GlobalIndex64, "64", (AssemblyStoreHashEntry he, AssemblyStoreAssembly assembly) => { + assembly.Hash64 = he.Hash; + if (assembly.RuntimeIndex != he.MappingIndex) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"assembly with hashes 0x{assembly.Hash32} and 0x{assembly.Hash64} has a different 32-bit runtime index ({assembly.RuntimeIndex}) than the 64-bit runtime index({he.MappingIndex})"); + } + + if (manifest != null && manifest.EntriesByHash64.TryGetValue (assembly.Hash64, out AssemblyStoreManifestEntry? me) && me != null) { + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} did not match any assembly name in the manifest"); + assembly.Name = me.Name; + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"64-bit hash 0x{assembly.Hash64:x} did not match any assembly name in the manifest"); + } + } else if (String.Compare (assembly.Name, me.Name, StringComparison.Ordinal) != 0) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} maps to assembly name '{assembly.Name}', however 64-bit hash 0x{assembly.Hash64:x} for the same entry matches assembly name '{me.Name}'"); + } + } + + if (!AssembliesByHash64.ContainsKey (assembly.Hash64)) { + AssembliesByHash64.Add (assembly.Hash64, assembly); + } + }); + + foreach (var kvp in Stores) { + List list = kvp.Value; + if (list.Count < 2) { + continue; + } + + AssemblyStoreReader template = list[0]; + for (int i = 1; i < list.Count; i++) { + AssemblyStoreReader other = list[i]; + if (!template.HasIdenticalContent (other)) { + Logger (AssemblyStoreExplorerLogLevel.Error, $"Store ID {template.StoreID} for architecture {other.Arch} is not identical to other stores with the same ID"); + } + } + } + + void ProcessIndex (List index, string bitness, Action assemblyHandler) + { + foreach (AssemblyStoreHashEntry he in index) { + if (!Stores.TryGetValue (he.StoreID, out List? storeList) || storeList == null) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"store with id {he.StoreID} not part of the set"); + continue; + } + + foreach (AssemblyStoreReader store in storeList) { + if (he.LocalStoreIndex >= (uint)store.Assemblies.Count) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"{bitness}-bit index entry with hash 0x{he.Hash:x} has invalid store {store.StoreID} index {he.LocalStoreIndex} (maximum allowed is {store.Assemblies.Count})"); + continue; + } + + AssemblyStoreAssembly assembly = store.Assemblies[(int)he.LocalStoreIndex]; + assemblyHandler (he, assembly); + + if (!AssembliesByName.ContainsKey (assembly.Name)) { + AssembliesByName.Add (assembly.Name, assembly); + } + } + } + } + } + + void ReadStoreSetFromArchive (string baseName, string archivePath, string extension) + { + string basePathInArchive; + + if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "base/root/assemblies"; + } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "assemblies"; + } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "root/assemblies"; + } else { + throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); + } + + basePathInArchive = $"{basePathInArchive}/{baseName}."; + using (ZipArchive archive = ZipArchive.Open (archivePath, FileMode.Open)) { + ReadStoreSetFromArchive (archive, basePathInArchive); + } + } + + void ReadStoreSetFromArchive (ZipArchive archive, string basePathInArchive) + { + foreach (ZipEntry entry in archive) { + if (!entry.FullName.StartsWith (basePathInArchive, StringComparison.Ordinal)) { + continue; + } + + using (var stream = new MemoryStream ()) { + entry.Extract (stream); + + if (entry.FullName.EndsWith (".store", StringComparison.Ordinal)) { + AddStore (new AssemblyStoreReader (stream, GetStoreArch (entry.FullName), keepStoreInMemory)); + } else if (entry.FullName.EndsWith (".manifest", StringComparison.Ordinal)) { + manifest = new AssemblyStoreManifestReader (stream); + } + } + } + } + + void AddStore (AssemblyStoreReader reader) + { + if (reader.HasGlobalIndex) { + indexStore = reader; + } + + List? storeList; + if (!Stores.TryGetValue (reader.StoreID, out storeList)) { + storeList = new List (); + Stores.Add (reader.StoreID, storeList); + } + storeList.Add (reader); + + Assemblies.AddRange (reader.Assemblies); + } + + string? GetStoreArch (string path) + { + string? arch = Path.GetFileNameWithoutExtension (path); + if (!String.IsNullOrEmpty (arch)) { + arch = Path.GetExtension (arch); + if (!String.IsNullOrEmpty (arch)) { + arch = arch.Substring (1); + } + } + + return arch; + } + + void ReadStoreSetFromFilesystem (string baseName, string setPath) + { + foreach (string de in Directory.EnumerateFiles (setPath, $"{baseName}.*", SearchOption.TopDirectoryOnly)) { + string? extension = Path.GetExtension (de); + if (String.IsNullOrEmpty (extension)) { + continue; + } + + if (String.Compare (".store", extension, StringComparison.OrdinalIgnoreCase) == 0) { + AddStore (ReadStore (de)); + } else if (String.Compare (".manifest", extension, StringComparison.OrdinalIgnoreCase) == 0) { + manifest = ReadManifest (de); + } + } + + AssemblyStoreReader ReadStore (string filePath) + { + string? arch = GetStoreArch (filePath); + using (var fs = File.OpenRead (filePath)) { + return CreateStoreReader (fs, arch); + } + } + + AssemblyStoreManifestReader ReadManifest (string filePath) + { + using (var fs = File.OpenRead (filePath)) { + return new AssemblyStoreManifestReader (fs); + } + } + } + + AssemblyStoreReader CreateStoreReader (Stream input, string? arch) + { + numberOfStores++; + return new AssemblyStoreReader (input, arch, keepStoreInMemory); + } + + bool IsAndroidArchive (string extension) + { + return + String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0; + } + + string GetBaseNameHaveExtension (string storePath, string extension) + { + if (IsAndroidArchive (extension)) { + return "assemblies"; + } + + string fileName = Path.GetFileNameWithoutExtension (storePath); + int dot = fileName.IndexOf ('.'); + if (dot >= 0) { + return fileName.Substring (0, dot); + } + + return fileName; + } + + string GetBaseNameNoExtension (string storePath) + { + string fileName = Path.GetFileName (storePath); + if (fileName.EndsWith ("_assemblies", StringComparison.OrdinalIgnoreCase)) { + return fileName; + } + return $"{fileName}_assemblies"; + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs b/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs new file mode 100644 index 00000000000..19452be507b --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs @@ -0,0 +1,10 @@ +namespace Xamarin.Android.AssemblyStore +{ + enum AssemblyStoreExplorerLogLevel + { + Debug, + Info, + Warning, + Error, + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreHashEntry.cs b/tools/assembly-store-reader/AssemblyStoreHashEntry.cs new file mode 100644 index 00000000000..b4504778807 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreHashEntry.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreHashEntry + { + public bool Is32Bit { get; } + + public ulong Hash { get; } + public uint MappingIndex { get; } + public uint LocalStoreIndex { get; } + public uint StoreID { get; } + + internal AssemblyStoreHashEntry (BinaryReader reader, bool is32Bit) + { + Is32Bit = is32Bit; + + Hash = reader.ReadUInt64 (); + MappingIndex = reader.ReadUInt32 (); + LocalStoreIndex = reader.ReadUInt32 (); + StoreID = reader.ReadUInt32 (); + } + } +} diff --git a/tools/assembly-blob-reader/BlobManifestEntry.cs b/tools/assembly-store-reader/AssemblyStoreManifestEntry.cs similarity index 71% rename from tools/assembly-blob-reader/BlobManifestEntry.cs rename to tools/assembly-store-reader/AssemblyStoreManifestEntry.cs index e77466c09da..53f2752f3a2 100644 --- a/tools/assembly-blob-reader/BlobManifestEntry.cs +++ b/tools/assembly-store-reader/AssemblyStoreManifestEntry.cs @@ -1,26 +1,26 @@ using System; using System.Globalization; -namespace Xamarin.Android.AssemblyBlobReader +namespace Xamarin.Android.AssemblyStore { - class BlobManifestEntry + class AssemblyStoreManifestEntry { // Fields are: - // Hash 32 | Hash 64 | Blob ID | Blob idx | Name + // Hash 32 | Hash 64 | Store ID | Store idx | Name const int NumberOfFields = 5; const int Hash32FieldIndex = 0; const int Hash64FieldIndex = 1; - const int BlobIDFieldIndex = 2; - const int BlobIndexFieldIndex = 3; + const int StoreIDFieldIndex = 2; + const int StoreIndexFieldIndex = 3; const int NameFieldIndex = 4; public uint Hash32 { get; } public ulong Hash64 { get; } - public uint BlobID { get; } - public uint IndexInBlob { get; } + public uint StoreID { get; } + public uint IndexInStore { get; } public string Name { get; } - public BlobManifestEntry (string[] fields) + public AssemblyStoreManifestEntry (string[] fields) { if (fields.Length != NumberOfFields) { throw new ArgumentOutOfRangeException (nameof (fields), "Invalid number of fields"); @@ -28,8 +28,8 @@ public BlobManifestEntry (string[] fields) Hash32 = GetUInt32 (fields[Hash32FieldIndex]); Hash64 = GetUInt64 (fields[Hash64FieldIndex]); - BlobID = GetUInt32 (fields[BlobIDFieldIndex]); - IndexInBlob = GetUInt32 (fields[BlobIndexFieldIndex]); + StoreID = GetUInt32 (fields[StoreIDFieldIndex]); + IndexInStore = GetUInt32 (fields[StoreIndexFieldIndex]); Name = fields[NameFieldIndex].Trim (); } diff --git a/tools/assembly-blob-reader/BlobManifestReader.cs b/tools/assembly-store-reader/AssemblyStoreManifestReader.cs similarity index 62% rename from tools/assembly-blob-reader/BlobManifestReader.cs rename to tools/assembly-store-reader/AssemblyStoreManifestReader.cs index 9c438b6103c..53a8a3c2e19 100644 --- a/tools/assembly-blob-reader/BlobManifestReader.cs +++ b/tools/assembly-store-reader/AssemblyStoreManifestReader.cs @@ -3,17 +3,17 @@ using System.IO; using System.Text; -namespace Xamarin.Android.AssemblyBlobReader +namespace Xamarin.Android.AssemblyStore { - class BlobManifestReader + class AssemblyStoreManifestReader { static readonly char[] fieldSplit = new char[] { ' ' }; - public List Entries { get; } = new List (); - public Dictionary EntriesByHash32 { get; } = new Dictionary (); - public Dictionary EntriesByHash64 { get; } = new Dictionary (); + public List Entries { get; } = new List (); + public Dictionary EntriesByHash32 { get; } = new Dictionary (); + public Dictionary EntriesByHash64 { get; } = new Dictionary (); - public BlobManifestReader (Stream manifest) + public AssemblyStoreManifestReader (Stream manifest) { manifest.Seek (0, SeekOrigin.Begin); using (var sr = new StreamReader (manifest, Encoding.UTF8, detectEncodingFromByteOrderMarks: false)) { @@ -33,7 +33,7 @@ void ReadManifest (StreamReader reader) continue; } - var entry = new BlobManifestEntry (fields); + var entry = new AssemblyStoreManifestEntry (fields); Entries.Add (entry); if (entry.Hash32 != 0) { EntriesByHash32.Add (entry.Hash32, entry); diff --git a/tools/assembly-store-reader/AssemblyStoreReader.cs b/tools/assembly-store-reader/AssemblyStoreReader.cs new file mode 100644 index 00000000000..8bb36bac01c --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreReader.cs @@ -0,0 +1,184 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreReader + { + // These two constants must be identical to the native ones in src/monodroid/jni/xamarin-app.hh + const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian + const uint ASSEMBLY_STORE_FORMAT_VERSION = 1; // The highest format version this reader understands + + MemoryStream? storeData; + + public uint Version { get; private set; } + public uint LocalEntryCount { get; private set; } + public uint GlobalEntryCount { get; private set; } + public uint StoreID { get; private set; } + public List Assemblies { get; } + public List GlobalIndex32 { get; } = new List (); + public List GlobalIndex64 { get; } = new List (); + public string Arch { get; } + + public bool HasGlobalIndex => StoreID == 0; + + public AssemblyStoreReader (Stream store, string? arch = null, bool keepStoreInMemory = false) + { + Arch = arch ?? String.Empty; + + store.Seek (0, SeekOrigin.Begin); + if (keepStoreInMemory) { + storeData = new MemoryStream (); + store.CopyTo (storeData); + storeData.Flush (); + store.Seek (0, SeekOrigin.Begin); + } + + using (var reader = new BinaryReader (store, Encoding.UTF8, leaveOpen: true)) { + ReadHeader (reader); + + Assemblies = new List (); + ReadLocalEntries (reader, Assemblies); + if (HasGlobalIndex) { + ReadGlobalIndex (reader, GlobalIndex32, GlobalIndex64); + } + } + } + + internal void ExtractAssemblyImage (AssemblyStoreAssembly assembly, string outputFilePath) + { + SaveDataToFile (outputFilePath, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyImage (AssemblyStoreAssembly assembly, Stream output) + { + SaveDataToStream (output, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyDebugData (AssemblyStoreAssembly assembly, string outputFilePath) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToFile (outputFilePath, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyDebugData (AssemblyStoreAssembly assembly, Stream output) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyConfig (AssemblyStoreAssembly assembly, string outputFilePath) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + + SaveDataToFile (outputFilePath, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + internal void ExtractAssemblyConfig (AssemblyStoreAssembly assembly, Stream output) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + void SaveDataToFile (string outputFilePath, uint offset, uint size) + { + EnsureStoreDataAvailable (); + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + SaveDataToStream (fs, offset, size); + } + } + + void SaveDataToStream (Stream output, uint offset, uint size) + { + EnsureStoreDataAvailable (); + ArrayPool pool = ArrayPool.Shared; + + storeData!.Seek (offset, SeekOrigin.Begin); + byte[] buf = pool.Rent (16384); + int nread; + long toRead = size; + while (toRead > 0 && (nread = storeData.Read (buf, 0, buf.Length)) > 0) { + if (nread > toRead) { + nread = (int)toRead; + } + + output.Write (buf, 0, nread); + toRead -= nread; + } + output.Flush (); + pool.Return (buf); + } + + void EnsureStoreDataAvailable () + { + if (storeData != null) { + return; + } + + throw new InvalidOperationException ("Store data not available. AssemblyStore/AssemblyStoreExplorer must be instantiated with the `keepStoreInMemory` argument set to `true`"); + } + + public bool HasIdenticalContent (AssemblyStoreReader other) + { + return + other.Version == Version && + other.LocalEntryCount == LocalEntryCount && + other.GlobalEntryCount == GlobalEntryCount && + other.StoreID == StoreID && + other.Assemblies.Count == Assemblies.Count && + other.GlobalIndex32.Count == GlobalIndex32.Count && + other.GlobalIndex64.Count == GlobalIndex64.Count; + } + + void ReadHeader (BinaryReader reader) + { + uint magic = reader.ReadUInt32 (); + if (magic != ASSEMBLY_STORE_MAGIC) { + throw new InvalidOperationException ("Invalid header magic number"); + } + + Version = reader.ReadUInt32 (); + if (Version == 0) { + throw new InvalidOperationException ("Invalid version number: 0"); + } + + if (Version > ASSEMBLY_STORE_FORMAT_VERSION) { + throw new InvalidOperationException ($"Store format version {Version} is higher than the one understood by this reader, {ASSEMBLY_STORE_FORMAT_VERSION}"); + } + + LocalEntryCount = reader.ReadUInt32 (); + GlobalEntryCount = reader.ReadUInt32 (); + StoreID = reader.ReadUInt32 (); + } + + void ReadLocalEntries (BinaryReader reader, List assemblies) + { + for (uint i = 0; i < LocalEntryCount; i++) { + assemblies.Add (new AssemblyStoreAssembly (reader, this)); + } + } + + void ReadGlobalIndex (BinaryReader reader, List index32, List index64) + { + ReadIndex (true, index32); + ReadIndex (true, index64); + + void ReadIndex (bool is32Bit, List index) { + for (uint i = 0; i < GlobalEntryCount; i++) { + index.Add (new AssemblyStoreHashEntry (reader, is32Bit)); + } + } + } + } +} diff --git a/tools/assembly-store-reader/Directory.Build.targets b/tools/assembly-store-reader/Directory.Build.targets new file mode 100644 index 00000000000..e58eed5ca2c --- /dev/null +++ b/tools/assembly-store-reader/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/assembly-blob-reader/Program.cs b/tools/assembly-store-reader/Program.cs similarity index 70% rename from tools/assembly-blob-reader/Program.cs rename to tools/assembly-store-reader/Program.cs index 83717b6b678..d5ee1806f64 100644 --- a/tools/assembly-blob-reader/Program.cs +++ b/tools/assembly-store-reader/Program.cs @@ -3,23 +3,23 @@ using Mono.Options; -namespace Xamarin.Android.AssemblyBlobReader +namespace Xamarin.Android.AssemblyStore { class Program { - static void ShowBlobInfo (string blobPath) + static void ShowAssemblyStoreInfo (string storePath) { - var explorer = new BlobExplorer (blobPath); + var explorer = new AssemblyStoreExplorer (storePath); string yesno = explorer.IsCompleteSet ? "yes" : "no"; - Console.WriteLine ($"Blob set '{explorer.BlobSetName}':"); + Console.WriteLine ($"Store set '{explorer.StoreSetName}':"); Console.WriteLine ($" Is complete set? {yesno}"); - Console.WriteLine ($" Number of blobs in the set: {explorer.NumberOfBlobs}"); + Console.WriteLine ($" Number of stores in the set: {explorer.NumberOfStores}"); Console.WriteLine (); Console.WriteLine ("Assemblies:"); string infoIndent = " "; - foreach (BlobAssembly assembly in explorer.Assemblies) { + foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) { Console.WriteLine ($" {assembly.RuntimeIndex}:"); Console.Write ($"{infoIndent}Name: "); if (String.IsNullOrEmpty (assembly.Name)) { @@ -28,11 +28,11 @@ static void ShowBlobInfo (string blobPath) Console.WriteLine (assembly.Name); } - Console.Write ($"{infoIndent}Blob ID: {assembly.Blob.BlobID} ("); - if (String.IsNullOrEmpty (assembly.Blob.Arch)) { + Console.Write ($"{infoIndent}Store ID: {assembly.Store.StoreID} ("); + if (String.IsNullOrEmpty (assembly.Store.Arch)) { Console.Write ("shared"); } else { - Console.Write (assembly.Blob.Arch); + Console.Write (assembly.Store.Arch); } Console.WriteLine (")"); @@ -73,18 +73,18 @@ void WriteHashValue (ulong hash) static int Main (string[] args) { if (args.Length == 0) { - Console.Error.WriteLine ("Usage: read-blob BLOB_PATH [BLOB_PATH ...]"); + Console.Error.WriteLine ("Usage: read-assembly-store BLOB_PATH [BLOB_PATH ...]"); Console.Error.WriteLine (); Console.Error.WriteLine (@" where each BLOB_PATH can point to: * aab file * apk file - * index blob file (e.g. base_assemblies.blob) - * arch blob file (e.g. base_assemblies.arm64_v8a.blob) - * blob manifest file (e.g. base_assemblies.manifest) - * blob base name (e.g. base or base_assemblies) + * index store file (e.g. base_assemblies.blob) + * arch store file (e.g. base_assemblies.arm64_v8a.blob) + * store manifest file (e.g. base_assemblies.manifest) + * store base name (e.g. base or base_assemblies) - In each case the whole set of blobs and manifests will be read (if available). Search for the - various members of the blob set (common/main blob, arch blobs, manifest) is based on this naming + In each case the whole set of stores and manifests will be read (if available). Search for the + various members of the store set (common/main store, arch stores, manifest) is based on this naming convention: {BASE_NAME}[.ARCH_NAME].{blob|manifest} @@ -98,7 +98,7 @@ various members of the blob set (common/main blob, arch blobs, manifest) is base bool first = true; foreach (string path in args) { - ShowBlobInfo (path); + ShowAssemblyStoreInfo (path); if (first) { first = false; continue; diff --git a/tools/assembly-blob-reader/assembly-blob-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj similarity index 86% rename from tools/assembly-blob-reader/assembly-blob-reader.csproj rename to tools/assembly-store-reader/assembly-store-reader.csproj index be33b8df2eb..66b47ed293b 100644 --- a/tools/assembly-blob-reader/assembly-blob-reader.csproj +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -6,10 +6,10 @@ 2021 Microsoft Corporation 0.0.1 false - ../../bin/$(Configuration)/bin/assembly-blob-reader + ../../bin/$(Configuration)/bin/assembly-store-reader Exe net6.0 - Xamarin.Android.AssemblyBlobReader + Xamarin.Android.AssemblyStoreReader disable enable diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index 311920b146c..a21f994bcc8 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -17,7 +17,7 @@ - + diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index e7a2074767d..63badae7355 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -4,7 +4,7 @@ using K4os.Compression.LZ4; using Xamarin.Tools.Zip; -using Xamarin.Android.AssemblyBlobReader; +using Xamarin.Android.AssemblyStore; namespace Xamarin.Android.Tools.DecompressAssemblies { @@ -101,14 +101,14 @@ static bool UncompressFromAPK_IndividualEntries (ZipArchive apk, string filePath return true; } - static bool UncompressFromAPK_Blobs (string filePath, string prefix) + static bool UncompressFromAPK_AssemblyStores (string filePath, string prefix) { - var explorer = new BlobExplorer (filePath, keepBlobInMemory: true); - foreach (BlobAssembly assembly in explorer.Assemblies) { + var explorer = new AssemblyStoreExplorer (filePath, keepStoreInMemory: true); + foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) { string assemblyName = assembly.DllName; - if (!String.IsNullOrEmpty (assembly.Blob.Arch)) { - assemblyName = $"{assembly.Blob.Arch}/{assemblyName}"; + if (!String.IsNullOrEmpty (assembly.Store.Arch)) { + assemblyName = $"{assembly.Store.Arch}/{assemblyName}"; } using (var stream = new MemoryStream ()) { @@ -130,7 +130,7 @@ static bool UncompressFromAPK (string filePath, string assembliesPath) } } - return UncompressFromAPK_Blobs (filePath, prefix); + return UncompressFromAPK_AssemblyStores (filePath, prefix); } static int Main (string[] args) diff --git a/tools/scripts/read-assembly-store b/tools/scripts/read-assembly-store new file mode 100755 index 00000000000..3386ec91783 --- /dev/null +++ b/tools/scripts/read-assembly-store @@ -0,0 +1,10 @@ +#!/bin/bash +truepath=$(readlink "$0" || echo "$0") +mydir=$(dirname ${truepath}) +binariesdir="${mydir}/assembly-store-reader" + +if [ -x "${binariesdir}/assembly-store-reader" ]; then + exec "${binariesdir}/assembly-store-reader" "$@" +else + exec dotnet "${binariesdir}/assembly-store-reader.dll" +fi diff --git a/tools/scripts/read-blob b/tools/scripts/read-blob deleted file mode 100755 index ca2f7580c82..00000000000 --- a/tools/scripts/read-blob +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -truepath=$(readlink "$0" || echo "$0") -mydir=$(dirname ${truepath}) -binariesdir="${mydir}/assembly-blob-reader" - -if [ -x "${binariesdir}/tmt" ]; then - exec "${binariesdir}/assembly-blob-reader" "$@" -else - exec dotnet "${binariesdir}/assembly-blob-reader.dll" -fi From 3759790617c7f4822a51335e4e615655f9defd32 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 13 Oct 2021 12:34:23 +0200 Subject: [PATCH 38/54] [WIP] The blob -> store renaming changed file extensions for blobs, bad rename, bad --- .../Utilities/ArchiveAssemblyHelper.cs | 6 ++- .../Xamarin.Android.Build.Tests/XASdkTests.cs | 38 +++++++++---------- .../AssemblyStoreExplorer.cs | 16 +++++--- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index a510d009c17..c2d9c8e2e6a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -102,9 +102,9 @@ Stream ReadStoreEntry (string path) string storeEntryName; if (String.IsNullOrEmpty (storeReader.Arch)) { - storeEntryName = $"{assembliesRootDir}assemblies.store"; + storeEntryName = $"{assembliesRootDir}assemblies.blob"; } else { - storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.store"; + storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.blob"; } Stream store = ReadZipEntry (storeEntryName); @@ -156,7 +156,9 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb return entries; } + Console.WriteLine ($"Creating AssemblyStoreExplorer for archive '{archivePath}'"); var explorer = new AssemblyStoreExplorer (archivePath); + Console.WriteLine ($"Explorer found {explorer.Assemblies.Count} assemblies"); foreach (var asm in explorer.Assemblies) { string prefix = storeEntryPrefix; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index c7545bceedb..899e1557951 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -398,104 +398,104 @@ public void DotNetBuildBinding () /* runtimeIdentifiers */ "android-arm", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-x86", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-x64", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, - /* usesAssemblyBlobs */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, - /* usesAssemblyBlobs */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, - /* usesAssemblyBlobs */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ true, - /* usesAssemblyBlobs */ false, + /* usesAssemblyStore */ false, }, }; [Test] [Category ("SmokeTests")] [TestCaseSource (nameof (DotNetBuildSource))] - public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyBlobs) + public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyStore) { var proj = new XASdkProject { IsRelease = isRelease, @@ -530,7 +530,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo } }; proj.MainActivity = proj.DefaultMainActivity.Replace (": Activity", ": AndroidX.AppCompat.App.AppCompatActivity"); - proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStore.ToString ()); if (aot) { proj.SetProperty ("RunAOTCompilation", "true"); } @@ -619,7 +619,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo bool expectEmbeddedAssembies = !(CommercialBuildAvailable && !isRelease); var apkPath = Path.Combine (outputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apkPath); - var helper = new ArchiveAssemblyHelper (apkPath, usesAssemblyBlobs); + var helper = new ArchiveAssemblyHelper (apkPath, usesAssemblyStore); helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.dll", shouldContainEntry: expectEmbeddedAssembies); helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); helper.AssertContainsEntry ($"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); diff --git a/tools/assembly-store-reader/AssemblyStoreExplorer.cs b/tools/assembly-store-reader/AssemblyStoreExplorer.cs index ca8722301ea..f7fb4bfe9ab 100644 --- a/tools/assembly-store-reader/AssemblyStoreExplorer.cs +++ b/tools/assembly-store-reader/AssemblyStoreExplorer.cs @@ -31,15 +31,15 @@ class AssemblyStoreExplorer // // aab // apk - // index store (e.g. base_assemblies.store) - // arch store (e.g. base_assemblies.arm64_v8a.store) + // index store (e.g. base_assemblies.blob) + // arch store (e.g. base_assemblies.arm64_v8a.blob) // store manifest (e.g. base_assemblies.manifest) // store base name (e.g. base or base_assemblies) // // In each case the whole set of stores and manifests will be read (if available). Search for the various members of the store set (common/main store, arch stores, // manifest) is based on this naming convention: // - // {BASE_NAME}[.ARCH_NAME].{store|manifest} + // {BASE_NAME}[.ARCH_NAME].{blob|manifest} // // Whichever file is referenced in `storePath`, the BASE_NAME component is extracted and all the found files are read. // If `storePath` points to an aab or an apk, BASE_NAME will always be `assemblies` @@ -72,6 +72,7 @@ public AssemblyStoreExplorer (string storePath, Action Date: Mon, 18 Oct 2021 12:34:31 +0200 Subject: [PATCH 39/54] [WIP] Mind your extensions + reuse constants more --- src/monodroid/jni/embedded-assemblies.cc | 13 ++++++++----- src/monodroid/jni/monodroid-glue.cc | 7 +++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 698ea504581..8f1ec545217 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -375,8 +375,9 @@ force_inline MonoAssembly* EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { size_t len = name.length (); + bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); - if (utils.ends_with (name, SharedConstants::DLL_EXTENSION)) { + if (have_dll_ext) { len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; } @@ -444,10 +445,12 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string fullpath (override_dir_len + file_name_len); utils.path_combine (fullpath, override_dir, override_dir_len, pname.get (), pname.length ()); if (!is_dll) { - fullpath.append (dll_extension, dll_extension_len); + fullpath.append (SharedConstants::DLL_EXTENSION, dll_extension_len); } log_info (LOG_ASSEMBLY, "open_from_update_dir: trying to open assembly: %s\n", fullpath.get ()); From d57ba721bfe69be3323e958970dd8fb040eabbb6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 18 Oct 2021 16:32:37 +0200 Subject: [PATCH 40/54] [WIP] Design blobs for Design Time Build --- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 5013df1de02..bd088cad434 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,7 +179,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False - False + False True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> From 2bc8d38c2d555731ce0cd9be750d66355c07221c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 18 Oct 2021 18:37:10 +0200 Subject: [PATCH 41/54] [WIP] Try to fix designer tests again --- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 2 +- src/monodroid/jni/application_dso_stub.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index bd088cad434..5013df1de02 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,7 +179,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False - False + False True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index d58e3f3e220..b9fbc5721f7 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -43,7 +43,7 @@ ApplicationConfig application_config = { .instant_run_enabled = false, .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, - .have_assembly_store = true, + .have_assembly_store = false, .bound_exception_type = 0, // System .package_naming_policy = 0, .environment_variable_count = 0, From 4de377940499811d7f86e1b1ab5639ca8cc8e66b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 18 Oct 2021 18:40:27 +0200 Subject: [PATCH 42/54] [WIP] Add a trailing comma --- src/monodroid/jni/application_dso_stub.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index b9fbc5721f7..6a0f3de1d21 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -96,7 +96,7 @@ AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[] = { .debug_info_data = nullptr, .config_data = nullptr, .descriptor = nullptr, - } + }, }; AssemblyStoreRuntimeData assembly_stores[] = { From fd9a54a54c759942a865d9e7c7e1aa704ab288f0 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 18 Oct 2021 23:14:41 +0200 Subject: [PATCH 43/54] [WIP] Force Linux builds to use g++/gcc 10, for full C++20 support --- build-tools/automation/azure-pipelines.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 50558cc8f8c..92b936f8555 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -397,6 +397,9 @@ stages: cancelTimeoutInMinutes: 2 workspace: clean: all + variables: + CXX: g++-10 + CC: gcc-10 steps: - checkout: self clean: true From d930fef231c4fc509ded4fa0f3833d2c9e593a88 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Oct 2021 11:11:37 +0200 Subject: [PATCH 44/54] [WIP] Attempt to work around a mingw bug on the CI bots --- src/monodroid/jni/basic-utilities.hh | 20 ++++++++++++ src/monodroid/jni/cpp-util.hh | 49 +++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index 2af276de223..630e7eba153 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -117,6 +117,13 @@ namespace xamarin::android return p != nullptr && p [N - 1] == '\0'; } + template + bool ends_with (const char *str, helper_char_array const& end) const noexcept + { + char *p = const_cast (strstr (str, end.data ())); + return p != nullptr && p [N - 1] == '\0'; + } + template bool ends_with (internal::string_base const& str, const char (&end)[N]) const noexcept { @@ -143,6 +150,19 @@ namespace xamarin::android return memcmp (str.get () + len - end_length, end.data (), end_length) == 0; } + template + bool ends_with (internal::string_base const& str, helper_char_array const& end) const noexcept + { + constexpr size_t end_length = N - 1; + + size_t len = str.length (); + if (XA_UNLIKELY (len < end_length)) { + return false; + } + + return memcmp (str.get () + len - end_length, end.data (), end_length) == 0; + } + template const TChar* find_last (internal::string_base const& str, const char ch) const noexcept { diff --git a/src/monodroid/jni/cpp-util.hh b/src/monodroid/jni/cpp-util.hh index 8cb9fe6e3ff..759cf4a4282 100644 --- a/src/monodroid/jni/cpp-util.hh +++ b/src/monodroid/jni/cpp-util.hh @@ -56,6 +56,53 @@ namespace xamarin::android template using c_unique_ptr = std::unique_ptr>; + template + struct helper_char_array final + { + constexpr char* data () noexcept + { + return _elems; + } + + constexpr const char* data () const noexcept + { + return _elems; + } + + constexpr char const& operator[] (size_t n) const noexcept + { + return _elems[n]; + } + + constexpr char& operator[] (size_t n) noexcept + { + return _elems[n]; + } + + char _elems[Size]; + }; + + // MinGW 9 on the CI build bots has a bug in std::array which causes builds to fail with: + // + // error G713F753E: ‘constexpr auto xamarin::android::concat_const(const char (&)[Length]...) [with long long unsigned int ...Length = {15, 7, 5}]’ called in a constant expression + // ... + // /usr/lib/gcc/x86_64-w64-mingw32/9.3-win32/include/c++/array:94:12: note: ‘struct std::array’ has no user-provided default constructor + // struct array + // ^~~~~ + // /usr/lib/gcc/x86_64-w64-mingw32/9.3-win32/include/c++/array:110:56: note: and the implicitly-defined constructor does not initialize ‘char std::array::_M_elems [17]’ + // typename _AT_Type::_Type _M_elems; + // ^~~~~~~~ + // + // thus we need to use this workaround here + // +#if defined (__MINGW32__) && __GNUC__ < 10 + template + using char_array = helper_char_array; +#else + template + using char_array = std::array; +#endif + template constexpr auto concat_const (const char (&...parts)[Length]) { @@ -63,7 +110,7 @@ namespace xamarin::android // `sizeof... (Length)` part which subtracts the number of template parameters - the amount of NUL bytes so that // we don't waste space. constexpr size_t total_length = (... + Length) - sizeof... (Length); - std::array ret; + char_array ret; ret[total_length] = 0; size_t i = 0; From df094492c874503daa43ac24c78aebdb7f16ea63 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Oct 2021 12:00:08 +0200 Subject: [PATCH 45/54] [WIP] Another attempt to work around a mingw issue --- src/monodroid/jni/cpp-util.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/monodroid/jni/cpp-util.hh b/src/monodroid/jni/cpp-util.hh index 759cf4a4282..de1ac16b05f 100644 --- a/src/monodroid/jni/cpp-util.hh +++ b/src/monodroid/jni/cpp-util.hh @@ -79,10 +79,10 @@ namespace xamarin::android return _elems[n]; } - char _elems[Size]; + char _elems[Size]{}; }; - // MinGW 9 on the CI build bots has a bug in std::array which causes builds to fail with: + // MinGW 9 on the CI build bots has a bug in the gcc compiler which causes builds to fail with: // // error G713F753E: ‘constexpr auto xamarin::android::concat_const(const char (&)[Length]...) [with long long unsigned int ...Length = {15, 7, 5}]’ called in a constant expression // ... From f12f7aa3a16ba5c8296c8a95a965a323dc86061a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Oct 2021 13:08:25 +0200 Subject: [PATCH 46/54] [WIP] And another attempt to make Linux CI bots mingw work --- src/monodroid/jni/application_dso_stub.cc | 4 ++++ src/monodroid/jni/embedded-assemblies-zip.cc | 13 ++++++++----- src/monodroid/jni/embedded-assemblies.cc | 4 ++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 6a0f3de1d21..51c6e1eecea 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -34,6 +34,10 @@ CompressedAssemblies compressed_assemblies = { /*.descriptors = */ nullptr, }; +// +// Config settings below **must** be valid for Desktop builds as the default `libxamarin-app.{dll,dylib,so}` is used by +// the Designer +// ApplicationConfig application_config = { .uses_mono_llvm = false, .uses_mono_aot = false, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 81b617e747e..b0f581b5b3d 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -292,11 +292,14 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus std::vector buf (cd_size); ZipEntryLoadState state { - .apk_fd = fd, - .apk_name = apk_name, - .prefix = get_assemblies_prefix (), - .prefix_len = get_assemblies_prefix_length (), - .buf_offset = 0, + .apk_fd = fd, + .apk_name = apk_name, + .prefix = get_assemblies_prefix (), + .prefix_len = get_assemblies_prefix_length (), + .buf_offset = 0, + .local_header_offset = 0, + .data_offset = 0, + .file_size = 0, }; ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 8f1ec545217..c8099b2256a 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -1,6 +1,8 @@ #include +#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) #include +#endif #include #include #include @@ -345,6 +347,7 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin force_inline const AssemblyStoreHashEntry* EmbeddedAssemblies::find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept { +#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) hash_t entry_hash; const AssemblyStoreHashEntry *ret; @@ -367,6 +370,7 @@ EmbeddedAssemblies::find_assembly_store_entry (hash_t hash, const AssemblyStoreH return ret; } } +#endif // ndef __MINGW32__ || (def __MINGW32__ && __GNUC__ >= 10) return nullptr; } From f0f8293f4cba72673e8ec6f93e68e3a045f169a4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Oct 2021 21:47:24 +0200 Subject: [PATCH 47/54] [WIP] Bundled apps should not use blobs - remove some debug logging, too --- .../Utilities/ArchAssemblyStore.cs | 10 ---------- .../Utilities/AssemblyStore.cs | 14 -------------- .../Utilities/AssemblyStoreGenerator.cs | 4 ---- .../Utilities/CommonAssemblyStore.cs | 1 - .../Xamarin.Android.Common.targets | 2 +- 5 files changed, 1 insertion(+), 30 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs index 49e928ca9da..a5b5811b7de 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs @@ -33,7 +33,6 @@ public override void Add (AssemblyStoreAssemblyInfo blobAssembly) assemblies.Add (blobAssembly.Abi, new List ()); } - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Arch '{blobAssembly.Abi}' assembly {blobAssembly.FilesystemAssemblyPath}"); List blobAssemblies = assemblies[blobAssembly.Abi]; blobAssemblies.Add (blobAssembly); @@ -60,25 +59,16 @@ public override void Generate (string outputDirectory, List archAssemblies = kvp.Value; - Log.LogMessage (MessageImportance.Low, $"Blob: assemblies for architecture {abi}:"); // All the architecture blobs must have assemblies in exactly the same order archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); if (assemblyNames.Count == 0) { - Log.LogMessage (MessageImportance.Low, $" first arch, preparing names list, {archAssemblies.Count} entries"); for (int i = 0; i < archAssemblies.Count; i++) { AssemblyStoreAssemblyInfo info = archAssemblies[i]; - Log.LogMessage (MessageImportance.Low, $" {Path.GetFileName (info.FilesystemAssemblyPath)}"); assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); } continue; } - Log.LogMessage (MessageImportance.Low, $" {archAssemblies.Count} entries"); - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - Log.LogMessage (MessageImportance.Low, $" {Path.GetFileName (info.FilesystemAssemblyPath)}"); - } - if (archAssemblies.Count != assemblyNames.Count) { throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs index 9346cb5fc78..a2ef8bdeb0c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs @@ -135,17 +135,14 @@ void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List (globalIndex); sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); foreach (AssemblyStoreIndexEntry entry in sortedIndex) { @@ -226,11 +223,7 @@ void Generate (BinaryWriter writer, List assemblies, } foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: assembly hfs path == '{assembly.FilesystemAssemblyPath}'; assembly archive path == '{assembly.ArchiveAssemblyPath}'"); string assemblyName = GetAssemblyName (assembly); - - Log.LogMessage (MessageImportance.Low, $"Got initial assemblyName == '{assemblyName}'"); - string archivePath = assembly.ArchiveAssemblyPath; if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); @@ -249,11 +242,9 @@ void Generate (BinaryWriter writer, List assemblies, } else { assemblyName = $"{archivePath}/{assemblyName}"; } - Log.LogMessage (MessageImportance.Low, $"Got archivePath ('{archivePath}'), assemblyName now == '{assemblyName}'"); } AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); - Log.LogMessage (MessageImportance.Low, $" => assemblyName == '{entry.Name}'; dataOffset == {entry.DataOffset}"); if (addToGlobalIndex) { globalIndex.Add (entry); } @@ -266,11 +257,8 @@ void Generate (BinaryWriter writer, List assemblies, return; } - Log.LogMessage (MessageImportance.Low, $"Not an index blob, writing header (local assemblies: {localAssemblies.Count})"); writer.Seek (0, SeekOrigin.Begin); WriteBlobHeader (writer, (uint)localAssemblies.Count); - - Log.LogMessage (MessageImportance.Low, "Not an index blob, writing assembly descriptors"); WriteAssemblyDescriptors (writer, localAssemblies); } @@ -294,9 +282,7 @@ void WriteAssemblyDescriptors (BinaryWriter writer, List {assembly.Name} before adjustment: data offset == {assembly.DataOffset}"); AdjustOffsets (assembly, offsetFixup); - Log.LogMessage (MessageImportance.Low, $" => {assembly.Name} after adjustment: data offset == {assembly.DataOffset}"); writer.Write (assembly.DataOffset); writer.Write (assembly.DataSize); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 36c2bbf3027..d81f6373fbd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -46,7 +46,6 @@ public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) { - log.LogMessage (MessageImportance.Low, $"Add: apkName == '{apkName}'"); if (String.IsNullOrEmpty (apkName)) { throw new ArgumentException ("must not be null or empty", nameof (apkName)); } @@ -72,13 +71,10 @@ public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) void SetIndexStore (AssemblyStore b) { - log.LogMessage (MessageImportance.Low, $"Checking if {b} is an index store"); if (!b.IsIndexStore) { - log.LogMessage (MessageImportance.Low, $" it is not (ID: {b.ID})"); return; } - log.LogMessage (MessageImportance.Low, $" it is"); if (indexStore != null) { throw new InvalidOperationException ("Index store already set!"); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs index 9b640ccc1a1..9709a200e5a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs @@ -24,7 +24,6 @@ public override void Add (AssemblyStoreAssemblyInfo blobAssembly) throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); } - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: adding Common assembly {blobAssembly.FilesystemAssemblyPath}"); assemblies.Add (blobAssembly); } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 5013df1de02..a566bfd1fbc 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,7 +179,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False - False + False True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> From 03be45204cc2fbda9c935b98e501238424d51920 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 20 Oct 2021 13:47:08 +0200 Subject: [PATCH 48/54] [WIP] Fix single-rid builds --- .../Tasks/BuildApk.cs | 4 +- .../Tasks/GeneratePackageManagerJava.cs | 3 -- .../Tasks/ProcessAssemblies.cs | 41 ++++++++++++++++--- .../Tasks/RegisterAssemblyStoreState.cs | 26 ++++++++++++ .../Utilities/ApplicationConfigTaskState.cs | 1 - .../Xamarin.Android.Common.targets | 6 +++ 6 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/RegisterAssemblyStoreState.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 32a5adebc66..ce13cace6b1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -324,8 +324,8 @@ public override bool RunTask () void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) { - var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); - bool useAssemblyStores = appConfState != null ? appConfState.UseAssemblyStore : false; + object useAssemblyStoresState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (RegisterAssemblyStoreState.RegisteredObjectKey, RegisteredTaskObjectLifetime.Build); + bool useAssemblyStores = useAssemblyStoresState != null ? (bool)useAssemblyStoresState : false; string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 9e8af4e1f3c..4125581f24f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -334,9 +334,6 @@ void AddEnvironment () bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); - if (appConfState != null) { - appConfState.UseAssemblyStore = UseAssemblyStore; - }; foreach (string abi in SupportedAbis) { NativeAssemblerTargetProvider asmTargetProvider = GetAssemblyTargetProvider (abi); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 17788900776..499f96fbc68 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -78,10 +78,35 @@ public override bool RunTask () return !Log.HasLoggedErrors; } + void SetAssemblyAbiMetadata (string abi, string assetType, ITaskItem assembly, ITaskItem? symbol) + { + if (String.IsNullOrEmpty (abi) || String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0) { + return; + } + + assembly.SetMetadata ("Abi", abi); + if (symbol != null) { + symbol.SetMetadata ("Abi", abi); + } + } + + void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol) + { + string assetType = assembly.GetMetadata ("AssetType"); + string rid = assembly.GetMetadata ("RuntimeIdentifier"); + if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { + // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set + return; + } + + SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assetType, assembly, symbol); + } + void SetMetadataForAssemblies (List output, Dictionary symbols) { foreach (var assembly in InputAssemblies) { var symbol = GetOrCreateSymbolItem (symbols, assembly); + SetAssemblyAbiMetadata (assembly, symbol); symbol?.SetDestinationSubPath (); assembly.SetDestinationSubPath (); assembly.SetMetadata ("FrameworkAssembly", IsFrameworkAssembly (assembly).ToString ()); @@ -175,9 +200,15 @@ bool Filter (ITaskItem item) void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol) { var rid = assembly.GetMetadata ("RuntimeIdentifier"); - // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, therefore we need to check it here in - // order to avoid them getting the `Abi` metadata, which would put them in the arch-specific assembly blob. - if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assembly.GetMetadata ("AssetType"), StringComparison.OrdinalIgnoreCase) == 0) { + string assetType = assembly.GetMetadata ("AssetType"); + + // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, so they should have none of the + // abi-specific metadata set + // + // Likewise, only abi-specific assemblies (with `AssetType=native` metadata set) should have the `Destination*` metadata set + if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || + String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0) { rid = String.Empty; } @@ -186,13 +217,13 @@ void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); assembly.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); assembly.SetMetadata ("DestinationSubPath", Path.Combine (destination, fileName)); - assembly.SetMetadata ("Abi", abi); if (symbol != null) { destination = Path.Combine (symbol.GetMetadata ("DestinationSubDirectory"), abi); symbol.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); symbol.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (symbol.ItemSpec))); - symbol.SetMetadata ("Abi", abi); } + + SetAssemblyAbiMetadata (abi, assetType, assembly, symbol); } else { Log.LogDebugMessage ($"Android ABI not found for: {assembly.ItemSpec}"); assembly.SetDestinationSubPath (); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/RegisterAssemblyStoreState.cs b/src/Xamarin.Android.Build.Tasks/Tasks/RegisterAssemblyStoreState.cs new file mode 100644 index 00000000000..b167983937f --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/RegisterAssemblyStoreState.cs @@ -0,0 +1,26 @@ +using System; +using System.IO; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks +{ + public class RegisterAssemblyStoreState : AndroidTask + { + public const string RegisteredObjectKey = ".:RegisterAssemblyStoreState_Key:."; + + public override string TaskPrefix => "RASS"; + + [Required] + public bool UseAssemblyStore { get; set; } + + public override bool RunTask () + { + BuildEngine4.RegisterTaskObjectAssemblyLocal (RegisteredObjectKey, UseAssemblyStore, RegisteredTaskObjectLifetime.Build); + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs index 3de3f75a830..5cdd3a6e6a2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs @@ -5,6 +5,5 @@ class ApplicationConfigTaskState public const string RegisterTaskObjectKey = "Xamarin.Android.Tasks.ApplicationConfigTaskState"; public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } = false; - public bool UseAssemblyStore { get; set; } = false; } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index a566bfd1fbc..87be15e29a5 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -92,6 +92,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. +