diff --git a/Documentation/docs-mobile/TOC.yml b/Documentation/docs-mobile/TOC.yml index ef5e9432fa8..7e6106f9572 100644 --- a/Documentation/docs-mobile/TOC.yml +++ b/Documentation/docs-mobile/TOC.yml @@ -184,6 +184,10 @@ href: messages/xa0139.md - name: XA0140 href: messages/xa0140.md + - name: XA0141 + href: messages/xa0141.md + - name: XA0142 + href: messages/xa0142.md - name: "XA1xxx: Project related" items: - name: "XA1xxx: Project related" diff --git a/Documentation/docs-mobile/messages/index.md b/Documentation/docs-mobile/messages/index.md index 8d46b161ab2..71a9b0f1962 100644 --- a/Documentation/docs-mobile/messages/index.md +++ b/Documentation/docs-mobile/messages/index.md @@ -1,4 +1,4 @@ ---- +-- title: .NET for Android errors and warnings reference description: Build and deployment error and warning codes in .NET for Android, their meanings, and guidance on how to address them. ms.date: 04/11/2024 @@ -104,6 +104,7 @@ or 'Help->Report a Problem' in Visual Studio for Mac. + [XA0139](xa0139.md): `@(AndroidAsset)` `{0}` has invalid `DeliveryType` metadata of `{1}`. Supported values are `installtime`, `ondemand` or `fastfollow` + [XA0140](xa0140.md): + [XA0141](xa0141.md): NuGet package '{0}' version '{1}' contains a shared library '{2}' which is not correctly aligned. See https://developer.android.com/guide/practices/page-sizes for more details ++ [XA0142](xa0142.md): Command '{0}' failed.\n{1} ## XA1xxx: Project related diff --git a/Documentation/docs-mobile/messages/xa0142.md b/Documentation/docs-mobile/messages/xa0142.md new file mode 100644 index 00000000000..a34b20bf12c --- /dev/null +++ b/Documentation/docs-mobile/messages/xa0142.md @@ -0,0 +1,17 @@ +--- +title: .NET for Android error XA0142 +description: XA0141 error code +ms.date: 11/09/2024 +--- +# .NET for Android warning XA0142 + +## Issue + +Command '{0}' failed.\n{1} + +## Solution + +Examine logged output of the failed command for indications of what caused the issue. If no immediate +solution is suggested by the logged messages, please file an issue at https://github.com/dotnet/android +including the full error message and steps that led to the command failing (possibly including a small +repro application). diff --git a/Documentation/project-docs/ApkSharedLibraries.md b/Documentation/project-docs/ApkSharedLibraries.md new file mode 100644 index 00000000000..c09dc4a20f9 --- /dev/null +++ b/Documentation/project-docs/ApkSharedLibraries.md @@ -0,0 +1,190 @@ +# Shared libraries in .NET for Android applications + +Applications contain a number of shared libraries which are placed in the +per-rid directories inside APK/AAB archives (`lib/ABI/lib*.so`). The libraries +have different purposes and come from different sources: + + 1. .NET PAL (Platform Abstraction Layer), used by various Base Class Library + assemblies. + 2. .NET runtime (`libmonosgen-2.0.so` containing the Mono VM) + 3. AOT images (`libaot*.so`, containing pre-JITed **data** which is loaded by + MonoVM at runtime and processed to turn into executable code) + 4. .NET for Android runtime and support libraries + 5. .NET for Android data payload libraries + +Most of those libraries have fairly obvious purpose and layout, this document +focuses on `.NET for Android` data payload libraries. + +# `.NET for Android` data payload libraries + +## Android packaging introduction + +Android allows applications to ship ABI-specific code inside the APK/AAB archives in +order to enable applications which need some sort of native code, while otherwise written +in a managed language like C#, Java or Kotlin. These libraries must be compiled to target +the platforms supported by Android and they must somehow co-exist in the same APK/AAB +archive (they always have the same name, just target a different platform/ABI). The way +chosen by Android to implement it is to place the per-ABI libraries in the `lib/{ABI}/` +directory of the archive. + +All of the libraries placed in the `lib/{ABI}` directories are expected to be ELF shared +library images, as required by the Android Linux kernel. + +## .NET for Android runtime, libraries and data + +`.NET for Android` runtime is composed of two libraries, one being the pre-compiled runtime +itself (`libmonodroid.so` in the APK) and another library being built together with the +application, containing application-specific dynamically generated code (`libxamarin-app.so` +in the APK). These two libraries together contain all the code and data to make the application +run properly on all the supported targets. + +In addition to the above, `.NET for Android` ships a number of managed assemblies. For a number +of years (starting with `Mono for Android`, through `Xamarin.Android`), all the assemblies had +been completely platform agnostic and, thus, were shipped in a custom directory in the APK archive +named `assemblies/`. However, at some point during transition to `dotnet/runtime` and its BCL, a +handful of managed libraries became platform specific and, thus, had to be shipped in a way that took +the platform requirement into account. As all those libraries shared the same name across platforms, +we had to find a way to package them so that they wouldn't conflict with each other. Thus the +`assemblies/` directory gained a subdirectory per ABI, which contained the platform specific assemblies. +Later on, the same was implemented in [assembly stores](AssemblyStores.md) - they would contain both kinds +of managed assemblies. + +The downside of packaging all the assemblies (or assembly stores) in the `assemblies/` directory was that +all the platforms would get copies of platform specific assemblies for the other supported ABIs, thus wasting +storage on the end user devices. + +Introduction of platform specific assemblies posed another problem. We discovered that in some instances, the +dotnet linker/trimmer would generate assemblies that might fail on certain platforms without us having any +prior warning. The solution to this was to make **all** the assemblies platform specific, making sure that +whatever the trimmer did, we'd always have the correct assembly loaded on the right platform. + +Making all assemblies platform specific, however, poses a problem of APK/AAB size - all of the assemblies would +exist in X copies and we couldn't allow such a big increase of archive size. Thus, all the assemblies (and also +assembly stores as well as a runtime configuration blob file) were moved to the `lib/{ABI}/` directories and +"masqueraded" as ELF shared libraries, by giving them the `lib*.so` names. However, the files were still managed +assemblies, not valid ELF images. + +Earlier this year, however, Google [announced](https://android-developers.googleblog.com/2024/08/adding-16-kb-page-size-to-android.html) that +Android 15 will enable shared libraries aligned to 16k instead of the "traditional" 4k and, at some point, the alignment +will become a requirement for submission to the Play Store. This made us suspect that the libraries in `lib/{ABI}/` will +be actually verified to be valid ELF images at some point and we decided to proactively turn our data files shipped in +those directories into actual ELF shared libraries. The way it is done is described in the following section. + +## Data payload stub library + +ELF binaries consist of a number of sections, which contain code, data (read-only and read-write), debug symbols etc. +However, the ELF specification doesn't dictate names of any of those sections and, thus, developers are free to lay out +ELF binaries any way they see fit, as long as the binary conforms to the ELF specification and the operating system +requirements. This gave us the idea of placing our data files (assemblies, assembly stores, debug data, config files etc) +in a custom section inside the ELF image. The resulting file would pass any verification Android will perform at some +point and, at the same time, it won't slow down our operation because we can still load data directly from the shared +library (by using the `mmap(2)` Unix call) without having to load the ELF image into memory. + +To implement that, we added to our distribution a "stub" of a shared ELF library, which is essentially a small, valid +but otherwise empty ELF image. This stub is built together with the rest of the `.NET for Android` runtime and its +layout is discovered and remembered, so that at runtime we can quickly move to the location where our data lives and +load it as we see fit. The runtime `mmap`s the entire file, looks at the file header and finds the start of payload +section, then stores that location in a pointer for further use. + +The way the data is placed in the ELF image is by appending a new section, called `payload`, to the stub binary at +application build time. This is done by using the `llvm-objcopy` utility, which we ship, and then the result is +packaged into the `lib/{ABI}/` directory. The section is properly aligned, the entire file is a valid ELF image. + +One downside of this approach is that if one were to run the `llvm-strip` or `strip` utility on the resulting +shared libray, the `payload` section (as it uses a "non-standard" name) would be considered by the strip utility +to be unnecessary and summarily removed. + +### Layout of the payload library + +In order to examine content of our "payload" ELF shared library, one can run the `llvm-readelf` utility which is +shipped with the Android NDK (and also part of native developer tools on macOS and Linux distributions which have +the LLVM Clang toolchain installed), or the `readelf` utility which is part of GNU binutils. + +File used in the samples below is the `.NET for Android` assembly store, wrapped in an ELF image for the Arm64 +(`AArch64`) architecture. + +The first command verifies that the file is a valid ELF image and shows the header information, including the +target platform/abi/machine: + +```shell +$ llvm-readelf --file-header libassemblies.arm64-v8a.blob.so +ELF Header: + Magic: 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 + Class: ELF64 + Data: 2's complement, little endian + Version: 1 (current) + OS/ABI: UNIX - System V + ABI Version: 0 + Type: DYN (Shared object file) + Machine: AArch64 + Version: 0x1 + Entry point address: 0x0 + Start of program headers: 64 (bytes into file) + Start of section headers: 849480 (bytes into file) + Flags: 0x0 + Size of this header: 64 (bytes) + Size of program headers: 56 (bytes) + Number of program headers: 8 + Size of section headers: 64 (bytes) + Number of section headers: 11 + Section header string table index: 9 +``` + +The second command lists the sections contained within the ELF image, their alignment, sizes and offsets +into the file where the sections begin: + +```shell +$ llvm-readelf --section-headers libassemblies.arm64-v8a.blob.so +There are 11 section headers, starting at offset 0xcf648: + +Section Headers: + [Nr] Name Type Address Off Size ES Flg Lk Inf Al + [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 + [ 1] .note.gnu.build-id NOTE 0000000000000200 000200 000024 00 A 0 0 4 + [ 2] .dynsym DYNSYM 0000000000000228 000228 000030 18 A 5 1 8 + [ 3] .gnu.hash GNU_HASH 0000000000000258 000258 000020 00 A 2 0 8 + [ 4] .hash HASH 0000000000000278 000278 000018 04 A 2 0 4 + [ 5] .dynstr STRTAB 0000000000000290 000290 000032 00 A 0 0 1 + [ 6] .dynamic DYNAMIC 00000000000042c8 0002c8 0000b0 10 WA 5 0 8 + [ 7] .relro_padding NOBITS 0000000000004378 000378 000c88 00 WA 0 0 1 + [ 8] .data PROGBITS 0000000000008378 000378 000001 00 WA 0 0 1 + [ 9] .shstrtab STRTAB 0000000000000000 000379 00005e 00 0 0 1 + [10] payload PROGBITS 0000000000000000 004000 0cb647 00 0 0 16384 +Key to Flags: + W (write), A (alloc), X (execute), M (merge), S (strings), I (info), + L (link order), O (extra OS processing required), G (group), T (TLS), + C (compressed), x (unknown), o (OS specific), E (exclude), + R (retain), p (processor specific) +``` + +Of interest to us is the presence of the `payload` section, its starting offset (it will usually +be `0x4000`, that is 16k into the file but it might be a multiple of the value, if the stub ever +grows) and its size will, obviously, differ depending on the payload. + +The information above is sufficient to verify that the file is valid `.NET for Android` payload +shared library. + +In order to extract payload from the ELF image, one can use the following command: + +```shell +$ llvm-objcopy --dump-section=payload=payload.bin libassemblies.arm64-v8a.blob.so +$ ls -gG payload.bin +-rw-rw-r-- 1 833095 Sep 12 11:32 payload.bin +``` + +To verify the size is correct, we can convert the section size indicated in the section headers +output from hexadecimal to decimal: + +```shell +$ printf "%d\n" 0x0cb647 +833095 +``` + +In this case, the payload file is an assembly store, which should have its first 4 bytes read +`XABA`, we can verify this with the following command: + +```shell +$ hexdump -c -n 4 payload.bin +0000000 X A B A +0000004 +``` diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 6ccb3fa4847..1ab3bc20d1b 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -42,6 +42,7 @@ projects that use the Microsoft.Android framework in .NET 6+. <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libmono-android.release.so" /> <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-debug-app-helper.so" /> <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-native-tracing.so" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libarchive-dso-stub.so" /> <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libunwind_xamarin.a" /> diff --git a/build-tools/create-packs/SignList.xml b/build-tools/create-packs/SignList.xml index f61c00fd16b..a71ea2c7fcf 100644 --- a/build-tools/create-packs/SignList.xml +++ b/build-tools/create-packs/SignList.xml @@ -16,6 +16,7 @@ + diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index bb3e307a6c1..29813191f0f 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -207,6 +207,7 @@ <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\ld.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llc.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-mc.exe" /> + <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-objcopy.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-strip.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\aarch64-linux-android-as.cmd" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\aarch64-linux-android-ld.cmd" /> diff --git a/build-tools/installers/unix-binutils.projitems b/build-tools/installers/unix-binutils.projitems index 519c1daed2b..3e01812136f 100644 --- a/build-tools/installers/unix-binutils.projitems +++ b/build-tools/installers/unix-binutils.projitems @@ -19,6 +19,7 @@ <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\ld" /> <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llc" /> <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-mc" /> + <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-objcopy" /> <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-strip" /> <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\x86_64-linux-android-as" /> <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\x86_64-linux-android-ld" /> diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index 0d298316d7a..103d07403fc 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -15,7 +15,7 @@ namespace Xamarin.Android.Prepare // partial class Configurables { - const string BinutilsVersion = "L_18.1.6-8.0.0"; + const string BinutilsVersion = "L_18.1.6-8.0.0-1"; const string MicrosoftOpenJDK17Version = "17.0.12"; const string MicrosoftOpenJDK17Release = "17.0.12"; @@ -157,6 +157,7 @@ public static partial class Defaults new NDKTool (name: "ld"), new NDKTool (name: "llc"), new NDKTool (name: "llvm-mc"), + new NDKTool (name: "llvm-objcopy"), new NDKTool (name: "llvm-strip"), }; } diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index f3ed040b0d9..285c72dafb3 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -13,6 +13,7 @@ _ResolveAssemblies MSBuild target. + + + <_ResolvedArchiveDSOStub Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Filename)%(ResolvedFileToPublish.Extension)' == 'libarchive-dso-stub.so' " /> + + + + + + + /// Looks up a localized string similar to Command '{0}' failed.\n{1} + /// + public static string XA0142 { + get { + return ResourceManager.GetString("XA0142", resourceCulture); + } + } + + /// /// Looks up a localized string similar to There was a problem parsing {0}. This is likely due to incomplete or invalid XML. Exception: {1}. /// public static string XA1000 { @@ -1382,7 +1391,7 @@ public static string XA4249 { return ResourceManager.GetString("XA4249", resourceCulture); } } - + /// /// Looks up a localized string similar to Native library '{0}' will not be bundled because it has an unsupported ABI. Move this file to a directory with a valid Android ABI name such as 'libs/armeabi-v7a/'.. /// diff --git a/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx b/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx index 9f4a6fa5fbd..04864c1f9af 100644 --- a/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx +++ b/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx @@ -1072,4 +1072,8 @@ To use a custom JDK path for a command line build, set the 'JavaSdkDirectory' MS The following are literal names and should not be translated: Maven, group_id, artifact_id {0} - A Maven artifact specification + + Command '{0}' failed.\n{1} + '{0}' is a failed command name (potentially with path) followed by all the arguments passed to it. {1} is the combined output on the standard error and standard output streams. + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 9feb1e3d7a7..6c9aff630ab 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -371,6 +371,7 @@ public override bool RunTask () OutputFiles = outputFiles.Select (a => new TaskItem (a)).ToArray (); Log.LogDebugTaskItems (" [Output] OutputFiles :", OutputFiles); + DSOWrapperGenerator.CleanUp (BuildEngine4, Log); return !Log.HasLoggedErrors; } @@ -404,7 +405,8 @@ void AddRuntimeConfigBlob (ZipArchiveEx apk) // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we // like since we can finish scanning the zip central directory earlier at startup. string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so"); - AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath)); + string wrappedSourcePath = DSOWrapperGenerator.WrapIt (MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath), BuildEngine4, Log); + AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath)); } } } @@ -448,7 +450,8 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary assemblies) + void DoAddAssembliesFromArchCollection (AndroidTargetArch arch, Dictionary assemblies) { // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. @@ -492,7 +495,8 @@ void DoAddAssembliesFromArchCollection (Dictionary assemblies if (UseAssemblyStore) { storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assembly); } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath)); + string wrappedSourcePath = DSOWrapperGenerator.WrapIt (arch, sourcePath, Path.GetFileName (assemblyPath), BuildEngine4, Log); + AddFileToArchiveIfNewer (apk, wrappedSourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath)); } // Try to add config if exists @@ -502,7 +506,7 @@ void DoAddAssembliesFromArchCollection (Dictionary assemblies storeAssemblyInfo.ConfigFile = new FileInfo (config); } } else { - AddAssemblyConfigEntry (apk, assemblyDirectory, config); + AddAssemblyConfigEntry (apk, arch, assemblyDirectory, config); } // Try to add symbols if Debug @@ -519,9 +523,10 @@ void DoAddAssembliesFromArchCollection (Dictionary assemblies storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath); } else { string archiveSymbolsPath = assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols)); + string wrappedSymbolsPath = DSOWrapperGenerator.WrapIt (arch, symbolsPath, Path.GetFileName (archiveSymbolsPath), BuildEngine4, Log); AddFileToArchiveIfNewer ( apk, - symbolsPath, + wrappedSymbolsPath, archiveSymbolsPath, compressionMethod: GetCompressionMethod (archiveSymbolsPath) ); @@ -607,7 +612,7 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat return true; } - void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string configFile) + void AddAssemblyConfigEntry (ZipArchiveEx apk, AndroidTargetArch arch, string assemblyPath, string configFile) { string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile)); existingEntries.Remove (inArchivePath); @@ -623,13 +628,8 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi } Log.LogDebugMessage ($"Adding {configFile} as the archive file is out of date."); - using (var source = File.OpenRead (configFile)) { - var dest = new MemoryStream (); - source.CopyTo (dest); - dest.WriteByte (0); - dest.Position = 0; - apk.AddEntryAndFlush (inArchivePath, dest, compressionMethod); - } + string wrappedConfigFile = DSOWrapperGenerator.WrapIt (arch, configFile, Path.GetFileName (inArchivePath), BuildEngine4, Log); + apk.AddFileAndFlush (wrappedConfigFile, inArchivePath, compressionMethod); } /// diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareDSOWrapperState.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareDSOWrapperState.cs new file mode 100644 index 00000000000..fca949c7b6c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareDSOWrapperState.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Registers a state object used by the DSOWrapperGenerator class later on during +/// the build. This is to avoid having to pass parameters to some tasks (esp. BuildApk) +/// which do not necessarily need those parameters directly. Registering the state here +/// also avoids having to update monodroid whenever any required parameter is added to +/// BuildApk. +/// +public class PrepareDSOWrapperState : AndroidTask +{ + public override string TaskPrefix => "PDWS"; + + [Required] + public ITaskItem[] ArchiveDSOStubs { get; set; } + + [Required] + public string AndroidBinUtilsDirectory { get; set; } + + [Required] + public string BaseOutputDirectory { get; set; } + + public override bool RunTask () + { + var stubPaths = new Dictionary (); + + foreach (ITaskItem stubItem in ArchiveDSOStubs) { + string rid = stubItem.GetRequiredMetadata ("ArchiveDSOStub", "RuntimeIdentifier", Log); + AndroidTargetArch arch = MonoAndroidHelper.RidToArch (rid); + if (stubPaths.ContainsKey (arch)) { + throw new InvalidOperationException ($"Internal error: duplicate archive DSO stub architecture '{arch}' (RID: '{rid}')"); + } + + if (!File.Exists (stubItem.ItemSpec)) { + throw new InvalidOperationException ($"Internal error: archive DSO stub file '{stubItem.ItemSpec}' does not exist"); + } + + stubPaths.Add (arch, stubItem); + } + + var config = new DSOWrapperGenerator.Config (stubPaths, AndroidBinUtilsDirectory, BaseOutputDirectory); + BuildEngine4.RegisterTaskObjectAssemblyLocal (DSOWrapperGenerator.RegisteredConfigKey, config, RegisteredTaskObjectLifetime.Build); + + return !Log.HasLoggedErrors; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index e4dfa7b26de..73541abf637 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -64,8 +64,57 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, ret = ReadZipEntry (path, arch, uncompressIfNecessary); } - ret?.Seek (0, SeekOrigin.Begin); - return ret; + if (ret == null) { + return null; + } + + ret.Flush (); + ret.Seek (0, SeekOrigin.Begin); + (ulong elfPayloadOffset, ulong elfPayloadSize, ELFPayloadError error) = Xamarin.Android.AssemblyStore.Utils.FindELFPayloadSectionOffsetAndSize (ret); + + if (error != ELFPayloadError.None) { + string message = error switch { + ELFPayloadError.NotELF => $"Entry '{path}' is not a valid ELF binary", + ELFPayloadError.LoadFailed => $"Entry '{path}' could not be loaded", + ELFPayloadError.NotSharedLibrary => $"Entry '{path}' is not a shared ELF library", + ELFPayloadError.NotLittleEndian => $"Entry '{path}' is not a little-endian ELF image", + ELFPayloadError.NoPayloadSection => $"Entry '{path}' does not contain the 'payload' section", + _ => $"Unknown ELF payload section error for entry '{path}': {error}" + }; + Console.WriteLine (message); + } else { + Console.WriteLine ($"Extracted content from ELF image '{path}'"); + } + + if (elfPayloadOffset == 0) { + ret.Seek (0, SeekOrigin.Begin); + return ret; + } + + // Make a copy of JUST the payload section, so that it contains only the data the tests expect and support + var payload = new MemoryStream (); + var data = buffers.Rent (16384); + int toRead = data.Length; + int nRead = 0; + ulong remaining = elfPayloadSize; + + ret.Seek ((long)elfPayloadOffset, SeekOrigin.Begin); + while (remaining > 0 && (nRead = ret.Read (data, 0, toRead)) > 0) { + payload.Write (data, 0, nRead); + remaining -= (ulong)nRead; + + if (remaining < (ulong)data.Length) { + // Make sure the last chunk doesn't gobble in more than we need + toRead = (int)remaining; + } + } + buffers.Return (data); + + payload.Flush (); + ret.Dispose (); + + payload.Seek (0, SeekOrigin.Begin); + return payload; } Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index afb5d34690b..54d10a91441 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -8,43 +8,43 @@ "Size": 22488 }, "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { - "Size": 1114 + "Size": 18208 }, "lib/arm64-v8a/lib_Java.Interop.dll.so": { - "Size": 69279 + "Size": 86368 }, "lib/arm64-v8a/lib_Mono.Android.dll.so": { - "Size": 98660 + "Size": 115752 }, "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { - "Size": 5315 + "Size": 22408 }, "lib/arm64-v8a/lib_System.Console.dll.so": { - "Size": 7294 + "Size": 24384 }, "lib/arm64-v8a/lib_System.Linq.dll.so": { - "Size": 9390 + "Size": 26480 }, "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { - "Size": 616700 + "Size": 633792 }, "lib/arm64-v8a/lib_System.Runtime.dll.so": { - "Size": 2956 + "Size": 20048 }, "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { - "Size": 4502 + "Size": 21592 }, "lib/arm64-v8a/lib_UnnamedProject.dll.so": { - "Size": 2932 + "Size": 20024 }, "lib/arm64-v8a/libarc.bin.so": { - "Size": 1685 + "Size": 18776 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 484920 + "Size": 485800 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3196336 @@ -98,5 +98,5 @@ "Size": 1904 } }, - "PackageSize": 2783765 + "PackageSize": 2791957 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 59a284fc02a..43092050ff6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -29,223 +29,223 @@ "Size": 2396 }, "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { - "Size": 2363 + "Size": 19456 }, "lib/arm64-v8a/lib_FormsViewGroup.dll.so": { - "Size": 8330 + "Size": 25424 }, "lib/arm64-v8a/lib_Java.Interop.dll.so": { - "Size": 77673 + "Size": 94768 }, "lib/arm64-v8a/lib_Mono.Android.dll.so": { - "Size": 506473 + "Size": 523568 }, "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { - "Size": 5315 + "Size": 22408 }, "lib/arm64-v8a/lib_mscorlib.dll.so": { - "Size": 4344 + "Size": 21432 }, "lib/arm64-v8a/lib_netstandard.dll.so": { - "Size": 5985 + "Size": 23080 }, "lib/arm64-v8a/lib_System.Collections.Concurrent.dll.so": { - "Size": 12714 + "Size": 29808 }, "lib/arm64-v8a/lib_System.Collections.dll.so": { - "Size": 19198 + "Size": 36288 }, "lib/arm64-v8a/lib_System.Collections.NonGeneric.dll.so": { - "Size": 8668 + "Size": 25760 }, "lib/arm64-v8a/lib_System.Collections.Specialized.dll.so": { - "Size": 6756 + "Size": 23848 }, "lib/arm64-v8a/lib_System.ComponentModel.dll.so": { - "Size": 2496 + "Size": 19584 }, "lib/arm64-v8a/lib_System.ComponentModel.Primitives.dll.so": { - "Size": 4212 + "Size": 21304 }, "lib/arm64-v8a/lib_System.ComponentModel.TypeConverter.dll.so": { - "Size": 25362 + "Size": 42456 }, "lib/arm64-v8a/lib_System.Console.dll.so": { - "Size": 7331 + "Size": 24424 }, "lib/arm64-v8a/lib_System.Core.dll.so": { - "Size": 2365 + "Size": 19456 }, "lib/arm64-v8a/lib_System.Diagnostics.DiagnosticSource.dll.so": { - "Size": 11349 + "Size": 28440 }, "lib/arm64-v8a/lib_System.Diagnostics.TraceSource.dll.so": { - "Size": 7601 + "Size": 24696 }, "lib/arm64-v8a/lib_System.dll.so": { - "Size": 2765 + "Size": 19856 }, "lib/arm64-v8a/lib_System.Drawing.dll.so": { - "Size": 2342 + "Size": 19432 }, "lib/arm64-v8a/lib_System.Drawing.Primitives.dll.so": { - "Size": 12959 + "Size": 30048 }, "lib/arm64-v8a/lib_System.Formats.Asn1.dll.so": { - "Size": 32843 + "Size": 49936 }, "lib/arm64-v8a/lib_System.IO.Compression.Brotli.dll.so": { - "Size": 12393 + "Size": 29488 }, "lib/arm64-v8a/lib_System.IO.Compression.dll.so": { - "Size": 16696 + "Size": 33784 }, "lib/arm64-v8a/lib_System.IO.IsolatedStorage.dll.so": { - "Size": 11196 + "Size": 28288 }, "lib/arm64-v8a/lib_System.Linq.dll.so": { - "Size": 21645 + "Size": 38736 }, "lib/arm64-v8a/lib_System.Linq.Expressions.dll.so": { - "Size": 168722 + "Size": 185816 }, "lib/arm64-v8a/lib_System.Net.Http.dll.so": { - "Size": 72400 + "Size": 89488 }, "lib/arm64-v8a/lib_System.Net.Primitives.dll.so": { - "Size": 24022 + "Size": 41112 }, "lib/arm64-v8a/lib_System.Net.Requests.dll.so": { - "Size": 4461 + "Size": 21552 }, "lib/arm64-v8a/lib_System.ObjectModel.dll.so": { - "Size": 9981 + "Size": 27072 }, "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { - "Size": 939274 + "Size": 956368 }, "lib/arm64-v8a/lib_System.Private.DataContractSerialization.dll.so": { - "Size": 199596 + "Size": 216688 }, "lib/arm64-v8a/lib_System.Private.Uri.dll.so": { - "Size": 45095 + "Size": 62184 }, "lib/arm64-v8a/lib_System.Private.Xml.dll.so": { - "Size": 220007 + "Size": 237096 }, "lib/arm64-v8a/lib_System.Private.Xml.Linq.dll.so": { - "Size": 18497 + "Size": 35592 }, "lib/arm64-v8a/lib_System.Runtime.dll.so": { - "Size": 3111 + "Size": 20200 }, "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { - "Size": 4502 + "Size": 21592 }, "lib/arm64-v8a/lib_System.Runtime.Numerics.dll.so": { - "Size": 37312 + "Size": 54400 }, "lib/arm64-v8a/lib_System.Runtime.Serialization.dll.so": { - "Size": 2265 + "Size": 19360 }, "lib/arm64-v8a/lib_System.Runtime.Serialization.Formatters.dll.so": { - "Size": 3243 + "Size": 20336 }, "lib/arm64-v8a/lib_System.Runtime.Serialization.Primitives.dll.so": { - "Size": 4361 + "Size": 21456 }, "lib/arm64-v8a/lib_System.Security.Cryptography.dll.so": { - "Size": 63408 + "Size": 80496 }, "lib/arm64-v8a/lib_System.Text.RegularExpressions.dll.so": { - "Size": 166506 + "Size": 183600 }, "lib/arm64-v8a/lib_System.Xml.dll.so": { - "Size": 2167 + "Size": 19256 }, "lib/arm64-v8a/lib_System.Xml.Linq.dll.so": { - "Size": 2181 + "Size": 19272 }, "lib/arm64-v8a/lib_UnnamedProject.dll.so": { - "Size": 5007 + "Size": 22096 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Activity.dll.so": { - "Size": 17872 + "Size": 34960 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.AppCompat.AppCompatResources.dll.so": { - "Size": 7425 + "Size": 24520 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.AppCompat.dll.so": { - "Size": 146152 + "Size": 163240 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.CardView.dll.so": { - "Size": 7469 + "Size": 24560 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.CoordinatorLayout.dll.so": { - "Size": 18823 + "Size": 35912 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Core.dll.so": { - "Size": 134318 + "Size": 151408 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.CursorAdapter.dll.so": { - "Size": 10076 + "Size": 27168 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.DrawerLayout.dll.so": { - "Size": 16856 + "Size": 33944 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Fragment.dll.so": { - "Size": 55435 + "Size": 72528 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Legacy.Support.Core.UI.dll.so": { - "Size": 6806 + "Size": 23896 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.Common.dll.so": { - "Size": 7982 + "Size": 25072 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.LiveData.Core.dll.so": { - "Size": 7770 + "Size": 24864 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.ViewModel.dll.so": { - "Size": 8114 + "Size": 25208 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.Loader.dll.so": { - "Size": 14499 + "Size": 31592 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.RecyclerView.dll.so": { - "Size": 95162 + "Size": 112256 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.SavedState.dll.so": { - "Size": 6050 + "Size": 23144 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.SwipeRefreshLayout.dll.so": { - "Size": 14858 + "Size": 31952 }, "lib/arm64-v8a/lib_Xamarin.AndroidX.ViewPager.dll.so": { - "Size": 20961 + "Size": 38056 }, "lib/arm64-v8a/lib_Xamarin.Forms.Core.dll.so": { - "Size": 563905 + "Size": 581000 }, "lib/arm64-v8a/lib_Xamarin.Forms.Platform.Android.dll.so": { - "Size": 373374 + "Size": 390464 }, "lib/arm64-v8a/lib_Xamarin.Forms.Platform.dll.so": { - "Size": 18753 + "Size": 35848 }, "lib/arm64-v8a/lib_Xamarin.Forms.Xaml.dll.so": { - "Size": 63542 + "Size": 80632 }, "lib/arm64-v8a/lib_Xamarin.Google.Android.Material.dll.so": { - "Size": 67675 + "Size": 84768 }, "lib/arm64-v8a/libarc.bin.so": { - "Size": 1685 + "Size": 18776 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 484920 + "Size": 485800 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3196336 @@ -416,7 +416,7 @@ "Size": 6 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1223 + "Size": 1221 }, "META-INF/BNDLTOOL.SF": { "Size": 98577 @@ -2486,5 +2486,5 @@ "Size": 812848 } }, - "PackageSize": 10579211 + "PackageSize": 10628363 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs new file mode 100644 index 00000000000..cdd47c233b7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// +/// Puts passed files inside a real ELF shared library so that they +/// pass scrutiny when examined. The payload is placed inside its own +/// section of a file, so the entire file is a 100% valid ELF image. +/// +/// +/// +/// The generated files have their payload section positioned at the offset of +/// 16k (0x4000) from the beginning of file. It's done this way because it not +/// only gives us enough room for the stub part of the ELF image to precede that +/// offset, but it also complies with Google policy of aligning to 16k **and** +/// is still nicely aligned to a 4k boundary on 32-bit systems. This helps mmapping +/// the section on both 64-bit and 32-bit systems. +/// +/// +/// +/// The generated file **MUST NOT** be stripped with `llvm-strip` etc, +/// as it will remove the payload together with other sections it deems +/// unnecessary. +/// +class DSOWrapperGenerator +{ + internal const string RegisteredConfigKey = ".:!DSOWrapperGeneratorConfig!:."; + + internal class Config + { + public Dictionary DSOStubPaths { get; } + public string AndroidBinUtilsDirectory { get; } + public string BaseOutputDirectory { get; } + + public Config (Dictionary stubPaths, string androidBinUtilsDirectory, string baseOutputDirectory) + { + DSOStubPaths = stubPaths; + AndroidBinUtilsDirectory = androidBinUtilsDirectory; + BaseOutputDirectory = baseOutputDirectory; + } + }; + + // + // Must be the same avalue as ARCHIVE_DSO_STUB_PAYLOAD_SECTION_ALIGNMENT in src/native/CMakeLists.txt + // + const ulong PayloadSectionAlignment = 0x4000; + + static Config EnsureConfig (IBuildEngine4 buildEngine) + { + var config = buildEngine.GetRegisteredTaskObjectAssemblyLocal (RegisteredConfigKey, RegisteredTaskObjectLifetime.Build); + if (config == null) { + throw new InvalidOperationException ("Internal error: no registered config found"); + } + + return config; + } + + static string GetArchOutputPath (AndroidTargetArch targetArch, Config config) + { + return Path.Combine (config.BaseOutputDirectory, MonoAndroidHelper.ArchToRid (targetArch), "wrapped"); + } + + /// + /// Puts the indicated file () inside an ELF shared library and returns + /// path to the wrapped file. + /// + public static string WrapIt (AndroidTargetArch targetArch, string payloadFilePath, string outputFileName, IBuildEngine4 buildEngine, TaskLoggingHelper log) + { + log.LogDebugMessage ($"[{targetArch}] Putting '{payloadFilePath}' inside ELF shared library '{outputFileName}'"); + Config config = EnsureConfig (buildEngine); + string outputDir = GetArchOutputPath (targetArch, config); + Directory.CreateDirectory (outputDir); + + string outputFile = Path.Combine (outputDir, outputFileName); + log.LogDebugMessage ($" output file path: {outputFile}"); + + if (!config.DSOStubPaths.TryGetValue (targetArch, out ITaskItem? stubItem)) { + throw new InvalidOperationException ($"Internal error: archive DSO stub location not known for architecture '{targetArch}'"); + } + + File.Copy (stubItem.ItemSpec, outputFile, overwrite: true); + + string quotedOutputFile = MonoAndroidHelper.QuoteFileNameArgument (outputFile); + string objcopy = Path.Combine (config.AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (config.AndroidBinUtilsDirectory, "llvm-objcopy")); + var args = new List { + "--add-section", + $"payload={MonoAndroidHelper.QuoteFileNameArgument (payloadFilePath)}", + quotedOutputFile, + }; + + int ret = MonoAndroidHelper.RunProcess (objcopy, String.Join (" ", args), log); + if (ret != 0) { + return outputFile; + } + + args.Clear (); + args.Add ("--set-section-flags"); + args.Add ("payload=readonly,data"); + args.Add ($"--set-section-alignment payload={PayloadSectionAlignment}"); + args.Add (quotedOutputFile); + ret = MonoAndroidHelper.RunProcess (objcopy, String.Join (" ", args), log); + + return outputFile; + } + + /// + /// Call when all packaging is done. The method will remove all the wrapper shared libraries that were previously + /// created by this class. The reason to do so is to ensure that we don't package any "stale" content and those + /// wrapper files aren't part of any dependency chain so it's hard to check their up to date state. + /// + public static void CleanUp (IBuildEngine4 buildEngine, TaskLoggingHelper log) + { + Config config = EnsureConfig (buildEngine); + + foreach (var kvp in config.DSOStubPaths) { + string outputDir = GetArchOutputPath (kvp.Key, config); + if (!Directory.Exists (outputDir)) { + continue; + } + + Directory.Delete (outputDir, recursive: true); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 9fe90178ea4..38335fcebb8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -6,6 +6,7 @@ using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Text; +using System.Threading; using Xamarin.Android.Tools; using Xamarin.Tools.Zip; @@ -50,6 +51,67 @@ void AppendLines (string prefix, List lines, StringBuilder sb) } } + public static int RunProcess (string command, string arguments, TaskLoggingHelper log, DataReceivedEventHandler? onOutput = null, DataReceivedEventHandler? onError = null) + { + var stdout_completed = new ManualResetEvent (false); + var stderr_completed = new ManualResetEvent (false); + var psi = new ProcessStartInfo () { + FileName = command, + Arguments = arguments, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + }; + + var stdoutLines = new List (); + var stderrLines = new List (); + + log.LogDebugMessage ($"Running process: {psi.FileName} {psi.Arguments}"); + using var proc = new Process (); + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) { + onOutput?.Invoke (s, e); + stdoutLines.Add (e.Data); + } else + stdout_completed.Set (); + }; + + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) { + onError?.Invoke (s, e); + stderrLines.Add (e.Data); + } else + stderr_completed.Set (); + }; + + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + proc.WaitForExit (); + + if (psi.RedirectStandardError) { + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (psi.RedirectStandardOutput) { + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (proc.ExitCode != 0) { + var sb = MergeStdoutAndStderrMessages (stdoutLines, stderrLines); + log.LogCodedError ("XA0142", Properties.Resources.XA0142, $"{psi.FileName} {psi.Arguments}", sb.ToString ()); + } + + try { + return proc.ExitCode; + } finally { + proc.Close (); + } + } + public static int RunProcess (string name, string args, DataReceivedEventHandler onOutput, DataReceivedEventHandler onError, Dictionary environmentVariables = null) { var psi = new ProcessStartInfo (name, args) { @@ -719,5 +781,12 @@ static uint ZipAlignmentToMaskOrPageSize (int alignment, bool needMask) _ => throw new InvalidOperationException ($"Internal error: unsupported zip page alignment value {alignment}") }; } + + public static string QuoteFileNameArgument (string fileName) + { + var builder = new CommandLineBuilder (); + builder.AppendFileNameIfNotNull (fileName); + return builder.ToString (); + } } } diff --git a/src/native/CMakeLists.txt b/src/native/CMakeLists.txt index 5af5b4b92f2..b0cbcc0ae4b 100644 --- a/src/native/CMakeLists.txt +++ b/src/native/CMakeLists.txt @@ -42,6 +42,7 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) include("${CMAKE_ANDROID_NDK}/build/cmake/abis.cmake") +include("${CMAKE_SOURCE_DIR}/cmake/ArchiveDSOStub.cmake") if(CMAKE_BUILD_TYPE STREQUAL Debug) set(DEBUG_BUILD True) @@ -69,6 +70,10 @@ if(ENABLE_CLANG_ASAN AND ENABLE_CLANG_UBSAN) endif() if(ENABLE_CLANG_ASAN OR ENABLE_CLANG_UBSAN) + if(BUILD_ARCHIVE_DSO_STUB) + message(FATAL_ERROR "ASAN/UBSAN builds aren't supported by the archive DSO target") + endif() + set(STRIP_DEBUG_DEFAULT OFF) set(ANALYZERS_ENABLED ON) else() @@ -191,12 +196,14 @@ macro(xa_add_compile_definitions TARGET) endif() endmacro() -if(DEBUG_BUILD AND NOT DISABLE_DEBUG) - add_compile_definitions(DEBUG) -endif() +if(NOT BUILD_ARCHIVE_DSO_STUB) + if(DEBUG_BUILD AND NOT DISABLE_DEBUG) + add_compile_definitions(DEBUG) + endif() -if(NOT DEBUG_BUILD) - add_compile_definitions(RELEASE NDEBUG) + if(NOT DEBUG_BUILD) + add_compile_definitions(RELEASE NDEBUG) + endif() endif() # @@ -326,10 +333,15 @@ set(POTENTIAL_COMMON_COMPILER_ARGS -funswitch-loops -Wa,-noexecstack -fPIC - -g -O2 ) +if(NOT BUILD_ARCHIVE_DSO_STUB) + list(APPEND POTENTIAL_COMMON_COMPILER_ARGS + -g + ) +endif() + set(POTENTIAL_COMMON_LINKER_ARGS -fstack-protector-strong LINKER:-fstrict-return @@ -451,11 +463,13 @@ xa_check_c_linker_args(XA_COMMON_C_LINKER_ARGS "${POTENTIAL_XA_COMMON_LINKER_ARG xa_check_c_linker_args(XA_C_DSO_LINKER_ARGS "${POTENTIAL_XA_DSO_LINKER_ARGS}") xa_check_cxx_linker_args(XA_CXX_DSO_LINKER_ARGS "${POTENTIAL_XA_DSO_LINKER_ARGS}") -add_compile_options("$<$:${COMMON_CXX_ARGS}>") -add_compile_options("$<$:${COMMON_C_ARGS}>") +if(NOT BUILD_ARCHIVE_DSO_STUB) + add_compile_options("$<$:${COMMON_CXX_ARGS}>") + add_compile_options("$<$:${COMMON_C_ARGS}>") -add_link_options("$<$:${COMMON_CXX_LINKER_ARGS}>") -add_link_options("$<$:${COMMON_C_LINKER_ARGS}>") + add_link_options("$<$:${COMMON_CXX_LINKER_ARGS}>") + add_link_options("$<$:${COMMON_C_LINKER_ARGS}>") +endif() # # Helper macros @@ -470,25 +484,29 @@ macro(set_static_library_suffix TARGET_NAME) endif() endmacro() -add_subdirectory(libunwind) -add_subdirectory(lz4) -add_subdirectory(libstub) -add_subdirectory(shared) -add_subdirectory(java-interop) -add_subdirectory(xamarin-app-stub) -add_subdirectory(runtime-base) -add_subdirectory(tracing) -add_subdirectory(pinvoke-override) - -if(DEBUG_BUILD) - add_subdirectory(xamarin-debug-app-helper) -endif() +if(BUILD_ARCHIVE_DSO_STUB) + add_subdirectory(archive-dso-stub) +else() + add_subdirectory(libunwind) + add_subdirectory(lz4) + add_subdirectory(libstub) + add_subdirectory(shared) + add_subdirectory(java-interop) + add_subdirectory(xamarin-app-stub) + add_subdirectory(runtime-base) + add_subdirectory(tracing) + add_subdirectory(pinvoke-override) + + if(DEBUG_BUILD) + add_subdirectory(xamarin-debug-app-helper) + endif() -add_subdirectory(monodroid) + add_subdirectory(monodroid) -add_custom_target(run_static_analysis - COMMAND ${ANDROID_TOOLCHAIN_ROOT}/bin/clang-check -analyze -p="${CMAKE_CURRENT_BINARY_DIR}" ${CLANG_CHECK_SOURCES} > ${CMAKE_SOURCE_DIR}/static-analysis.${ANDROID_ABI}.${CMAKE_BUILD_TYPE}.txt 2>&1 - COMMAND_EXPAND_LISTS - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - USES_TERMINAL -) + add_custom_target(run_static_analysis + COMMAND ${ANDROID_TOOLCHAIN_ROOT}/bin/clang-check -analyze -p="${CMAKE_CURRENT_BINARY_DIR}" ${CLANG_CHECK_SOURCES} > ${CMAKE_SOURCE_DIR}/static-analysis.${ANDROID_ABI}.${CMAKE_BUILD_TYPE}.txt 2>&1 + COMMAND_EXPAND_LISTS + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + USES_TERMINAL + ) +endif() diff --git a/src/native/archive-dso-stub/CMakeLists.txt b/src/native/archive-dso-stub/CMakeLists.txt new file mode 100644 index 00000000000..d21669498d0 --- /dev/null +++ b/src/native/archive-dso-stub/CMakeLists.txt @@ -0,0 +1,36 @@ +include("${CMAKE_SOURCE_DIR}/cmake/ArchiveDSOStub.cmake") + +set(LIB_NAME archive-dso-stub) + +set(LIB_SOURCES + stub.cc +) + +add_library( + ${LIB_NAME} + SHARED + ${LIB_SOURCES} +) + +set(ARCHIVE_DSO_STUB_LIB_NAME "lib${LIB_NAME}.so" PARENT_SCOPE) + +target_compile_options( + ${LIB_NAME} + PRIVATE + ${XA_DEFAULT_SYMBOL_VISIBILITY} + ${XA_COMMON_CXX_ARGS} + -nostdlib -fno-exceptions -fno-rtti +) + +target_link_options( + ${LIB_NAME} + PRIVATE + ${XA_COMMON_CXX_LINKER_ARGS} + -nostdlib -fno-exceptions -fno-rtti -s +) + +add_custom_command( + TARGET ${LIB_NAME} + POST_BUILD + COMMAND ${CMAKE_STRIP} "$" +) diff --git a/src/native/archive-dso-stub/stub.cc b/src/native/archive-dso-stub/stub.cc new file mode 100644 index 00000000000..7e3d5e0e978 --- /dev/null +++ b/src/native/archive-dso-stub/stub.cc @@ -0,0 +1,7 @@ +// +// This is here to clearly indicate that a `lib/{ABI}/lib*.so` file is really a +// data file for .NET for Android and not a "regular" library. Might be useful +// when analyzing APK/AABs. +// +[[gnu::visibility("default")]] +bool dotnet_for_android_data_payload = true; diff --git a/src/native/cmake/ArchiveDSOStub.cmake b/src/native/cmake/ArchiveDSOStub.cmake new file mode 100644 index 00000000000..6c43377007f --- /dev/null +++ b/src/native/cmake/ArchiveDSOStub.cmake @@ -0,0 +1,2 @@ +set(ARCHIVE_DSO_STUB_LIB_NAME "archive-dso-stub") +set(ARCHIVE_DSO_STUB_LIB_FILE_NAME "lib${ARCHIVE_DSO_STUB_LIB_NAME}.so") diff --git a/src/native/monodroid/CMakeLists.txt b/src/native/monodroid/CMakeLists.txt index 09d4b16c4dd..65dd350931a 100644 --- a/src/native/monodroid/CMakeLists.txt +++ b/src/native/monodroid/CMakeLists.txt @@ -1,5 +1,53 @@ option(ENABLE_TIMING "Build with timing support" OFF) +# First generate some code + +# +# Must be the same value as DSOWrapperGenerator.PayloadSectionAlignment in src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs +# +set(ARCHIVE_DSO_STUB_PAYLOAD_SECTION_ALIGNMENT 0x4000) + +file(COPY "${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/${ARCHIVE_DSO_STUB_LIB_FILE_NAME}" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +set(ARCHIVE_DSO_STUB_LIB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${ARCHIVE_DSO_STUB_LIB_FILE_NAME}") + +# Emulate what we do when embedding something inside the ELF file +set(PAYLOAD_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt) + +execute_process( + COMMAND ${CMAKE_OBJCOPY} --add-section payload=${PAYLOAD_PATH} ${ARCHIVE_DSO_STUB_LIB_PATH} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ECHO_ERROR_VARIABLE + COMMAND_ERROR_IS_FATAL ANY +) + +execute_process( + COMMAND ${CMAKE_OBJCOPY} --set-section-flags payload=readonly,data --set-section-alignment payload=${ARCHIVE_DSO_STUB_PAYLOAD_SECTION_ALIGNMENT} ${ARCHIVE_DSO_STUB_LIB_PATH} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ECHO_ERROR_VARIABLE + COMMAND_ERROR_IS_FATAL ANY +) + +execute_process( + COMMAND ${CMAKE_READELF} --file-header --section-headers ${ARCHIVE_DSO_STUB_LIB_PATH} --elf-output-style=JSON + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + OUTPUT_VARIABLE ARCHIVE_DSO_STUB_HEADER_JSON + ECHO_ERROR_VARIABLE + COMMAND_ERROR_IS_FATAL ANY +) +string(JSON SECTION_HEADER_ENTRY_SIZE GET "${ARCHIVE_DSO_STUB_HEADER_JSON}" 0 "ElfHeader" "SectionHeaderEntrySize") +string(JSON SECTION_HEADER_ENTRY_COUNT GET "${ARCHIVE_DSO_STUB_HEADER_JSON}" 0 "ElfHeader" "SectionHeaderCount") + +math(EXPR PAYLOAD_SECTION_INDEX "${SECTION_HEADER_ENTRY_COUNT} - 1") +string(JSON PAYLOAD_SECTION_OFFSET GET "${ARCHIVE_DSO_STUB_HEADER_JSON}" 0 "Sections" ${PAYLOAD_SECTION_INDEX} "Section" "Offset") +file(REMOVE ${ARCHIVE_DSO_STUB_LIB_PATH}) + +message(STATUS "Archive DSO stub header entry size: ${SECTION_HEADER_ENTRY_SIZE}; section count: ${SECTION_HEADER_ENTRY_COUNT}; payload offset: ${PAYLOAD_SECTION_OFFSET}") +configure_file( + archive-dso-stub-config.hh.in + ${CMAKE_CURRENT_BINARY_DIR}/include/archive-dso-stub-config.hh + USE_SOURCE_PERMISSIONS +) + # Needed modules include(CheckIncludeFile) diff --git a/src/native/monodroid/archive-dso-stub-config.hh.in b/src/native/monodroid/archive-dso-stub-config.hh.in new file mode 100644 index 00000000000..b0aa77eb3e6 --- /dev/null +++ b/src/native/monodroid/archive-dso-stub-config.hh.in @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace xamarin::android { + struct ArchiveDSOStubConfig + { + static inline constexpr size_t PayloadSectionAlignment = @ARCHIVE_DSO_STUB_PAYLOAD_SECTION_ALIGNMENT@; + static inline constexpr size_t SectionHeaderEntrySize = @SECTION_HEADER_ENTRY_SIZE@; + static inline constexpr size_t SectionHeaderEntryCount = @SECTION_HEADER_ENTRY_COUNT@; + static inline constexpr uint32_t PayloadSectionOffset = @PAYLOAD_SECTION_OFFSET@; + + // We know that payload section is the last one in the binary, this is an index into + // the section header table. + static inline constexpr size_t PayloadSectionIndex = SectionHeaderEntryCount - 1; + }; +} diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index a078cf0c1cd..317b6cb9011 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -53,8 +53,8 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector if (close_fd) { close (fd); } - auto header = static_cast(assembly_store_map.area); + + auto [payload_start, payload_size] = get_wrapper_dso_payload_pointer_and_size (assembly_store_map, entry_name.get ()); + log_debug (LOG_ASSEMBLY, "Adjusted assembly store pointer: %p; size: %zu", payload_start, payload_size); + auto header = static_cast(payload_start); if (header->magic != ASSEMBLY_STORE_MAGIC) { log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid .NET for Android assembly store file", entry_name.get ()); @@ -200,7 +203,7 @@ EmbeddedAssemblies::map_assembly_store (dynamic_local_string constexpr size_t header_size = sizeof(AssemblyStoreHeader); - assembly_store.data_start = static_cast(assembly_store_map.area); + assembly_store.data_start = static_cast(payload_start); assembly_store.assembly_count = header->entry_count; assembly_store.index_entry_count = header->index_entry_count; assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); diff --git a/src/native/monodroid/embedded-assemblies.cc b/src/native/monodroid/embedded-assemblies.cc index 57733c6349e..5ecaa8c8623 100644 --- a/src/native/monodroid/embedded-assemblies.cc +++ b/src/native/monodroid/embedded-assemblies.cc @@ -161,7 +161,7 @@ EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexc int fd; bool close_fd; if (!AndroidSystem::is_embedded_dso_mode_enabled ()) { - log_debug (LOG_ASSEMBLY, "Mapping a runtime file from a filesystem"); + log_debug (LOG_ASSEMBLY, "Mapping a runtime file from filesystem"); close_fd = true; // file.file_fd refers to the directory where our files live @@ -180,14 +180,19 @@ EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexc close (fd); } + auto [payload_data, payload_size] = get_wrapper_dso_payload_pointer_and_size (map_info, file.name); + + // `data_size` might have been ELF wrapper file size, we must store payload size here (the actual DLL size) + file.data_size = static_cast(payload_size); + if (MonodroidState::is_startup_in_progress ()) { - file.data = static_cast(map_info.area); + file.data = static_cast(payload_data); } else { uint8_t *expected_null = nullptr; bool already_mapped = !__atomic_compare_exchange ( /* ptr */ &file.data, /* expected */ &expected_null, - /* desired */ reinterpret_cast(&map_info.area), + /* desired */ reinterpret_cast(&payload_data), /* weak */ false, /* success_memorder */ __ATOMIC_ACQUIRE, /* failure_memorder */ __ATOMIC_RELAXED @@ -1339,7 +1344,7 @@ EmbeddedAssemblies::register_from_filesystem (const char *lib_dir_path,bool look } runtime_config_blob_mmap = md_mmap_apk_file (fd.value (), 0, file_size.value (), cur->d_name); - runtime_config_blob_found = true; + store_mapped_runtime_config_data (runtime_config_blob_mmap, cur->d_name); continue; } diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 4e26e933eb7..70384ffbcc8 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -11,12 +11,15 @@ #include #include +#include #include +#include #include #include #include +#include "archive-dso-stub-config.hh" #include "strings.hh" #include "xamarin-app.hh" #include "cpp-util.hh" @@ -101,6 +104,7 @@ namespace xamarin::android::internal { static constexpr size_t assembly_store_file_path_size = calc_size(apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); static constexpr auto assembly_store_file_path = concat_string_views (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); + static constexpr size_t dso_size_overhead = ArchiveDSOStubConfig::PayloadSectionOffset + (ArchiveDSOStubConfig::SectionHeaderEntryCount * ArchiveDSOStubConfig::SectionHeaderEntrySize); public: /* filename is e.g. System.dll, System.dll.mdb, System.pdb */ @@ -155,10 +159,21 @@ namespace xamarin::android::internal { void get_runtime_config_blob (const char *& area, uint32_t& size) const { - area = static_cast(runtime_config_blob_mmap.area); + area = static_cast(runtime_config_data); - abort_unless (runtime_config_blob_mmap.size < std::numeric_limits::max (), "Runtime config binary blob size exceeds %u bytes", std::numeric_limits::max ()); - size = static_cast(runtime_config_blob_mmap.size); + abort_unless (runtime_config_data_size < std::numeric_limits::max (), "Runtime config binary blob size exceeds %u bytes", std::numeric_limits::max ()); + size = static_cast(runtime_config_data_size); + } + + void unmap_runtime_config_blob () + { + if (runtime_config_blob_mmap.area == nullptr) { + return; + } + + munmap (runtime_config_blob_mmap.area, runtime_config_blob_mmap.size); + runtime_config_blob_mmap.area = nullptr; + runtime_config_blob_mmap.size = 0; } bool have_runtime_config_blob () const noexcept @@ -262,6 +277,47 @@ namespace xamarin::android::internal { bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); + [[gnu::always_inline]] + static std::tuple get_wrapper_dso_payload_pointer_and_size (md_mmap_info const& map_info, const char *file_name) noexcept + { + using Elf_Header = std::conditional_t; + using Elf_SHeader = std::conditional_t; + + const void* const mapped_elf = map_info.area; + auto elf_bytes = static_cast(mapped_elf); + auto elf_header = reinterpret_cast(mapped_elf); + + if constexpr (SharedConstants::debug_build) { + // In debug mode we might be dealing with plain data, without DSO wrapper + if (elf_header->e_ident[EI_MAG0] != ELFMAG0 || + elf_header->e_ident[EI_MAG1] != ELFMAG1 || + elf_header->e_ident[EI_MAG2] != ELFMAG2 || + elf_header->e_ident[EI_MAG3] != ELFMAG3) { + log_debug (LOG_ASSEMBLY, "Not an ELF image: %s", file_name); + // Not an ELF image, just return what we mmapped before + return { map_info.area, map_info.size }; + } + } + + auto section_header = reinterpret_cast(elf_bytes + elf_header->e_shoff); + Elf_SHeader const& payload_hdr = section_header[ArchiveDSOStubConfig::PayloadSectionIndex]; + + return { + const_cast(reinterpret_cast (elf_bytes + ArchiveDSOStubConfig::PayloadSectionOffset)), + payload_hdr.sh_size + }; + } + + [[gnu::always_inline]] + void store_mapped_runtime_config_data (md_mmap_info const& map_info, const char *file_name) noexcept + { + auto [payload_start, payload_size] = get_wrapper_dso_payload_pointer_and_size (map_info, file_name); + log_debug (LOG_ASSEMBLY, "Runtime config: payload pointer %p ; size %zu", payload_start, payload_size); + runtime_config_data = payload_start; + runtime_config_data_size = payload_size; + runtime_config_blob_found = true; + } + std::tuple get_assemblies_prefix_and_length () const noexcept { if (assemblies_prefix_override != nullptr) { @@ -382,6 +438,8 @@ namespace xamarin::android::internal { const char *assemblies_prefix_override = nullptr; md_mmap_info runtime_config_blob_mmap{}; + void *runtime_config_data = nullptr; + size_t runtime_config_data_size = 0; bool runtime_config_blob_found = false; uint32_t number_of_mapped_assembly_stores = 0; uint32_t number_of_zip_dso_entries = 0; diff --git a/src/native/monodroid/monodroid-glue.cc b/src/native/monodroid/monodroid-glue.cc index 1c5aed8025c..3faa74af385 100644 --- a/src/native/monodroid/monodroid-glue.cc +++ b/src/native/monodroid/monodroid-glue.cc @@ -700,11 +700,7 @@ MonodroidRuntime::mono_runtime_init ([[maybe_unused]] JNIEnv *env, [[maybe_unuse void MonodroidRuntime::cleanup_runtime_config (MonovmRuntimeConfigArguments *args, [[maybe_unused]] void *user_data) { - if (args == nullptr || args->kind != 1 || args->runtimeconfig.data.data == nullptr) { - return; - } - - munmap (static_cast(const_cast(args->runtimeconfig.data.data)), args->runtimeconfig.data.data_len); + embeddedAssemblies.unmap_runtime_config_blob (); } MonoDomain* diff --git a/src/native/native.targets b/src/native/native.targets index ab4539bd4fa..91ca4b82996 100644 --- a/src/native/native.targets +++ b/src/native/native.targets @@ -2,7 +2,7 @@ + DependsOnTargets="_ConfigureAndBuildArchiveDSOStub;_ConfigureRuntimes;_BuildAndroidRuntimes;_BuildAndroidAnalyzerRuntimes;_CopyToPackDirs"> @@ -51,6 +51,39 @@ + + + <_ArchiveDSOInput Include="archive-dso-stub\CMakeLists.txt" /> + + <_ArchiveDSOOutput Include="@(AndroidSupportedTargetJitAbi->'$(IntermediateOutputPath)\%(AndroidRID)-archive-dso-stub\CMakeCache.txt')" /> + <_ArchiveDSOOutput Include="@(AndroidSupportedTargetJitAbi->'$(OutputPath)\%(AndroidRID)\libarchive-dso-stub.so')" /> + + <_ArchiveOutputDirToCreate Include="$(IntermediateOutputPath)%(AndroidSupportedTargetJitAbi.AndroidRID)-archive-dso-stub" /> + + + + + + <_ConfigureArchiveDSOStubCommands Include="@(AndroidSupportedTargetJitAbi)"> + $(CmakePath) + --preset default-release-%(AndroidSupportedTargetJitAbi.Identity) -DBUILD_ARCHIVE_DSO_STUB=ON -DSTRIP_DEBUG=ON "$(MSBuildThisFileDirectory)" + $(IntermediateOutputPath)%(AndroidSupportedTargetJitAbi.AndroidRID)-archive-dso-stub + + + + + + + + + + + + <_MonoDroidSources Include="android-dso-stub\*.cc;libstub\*.hh" /> <_MonoDroidSources Include="libstub\*.cc;libstub\*.hh" /> <_MonoDroidSources Include="monodroid\*.cc;monodroid\*.hh" /> <_MonoDroidSources Include="runtime-base\*.cc;runtime-base\*.hh" /> @@ -183,6 +217,9 @@ + diff --git a/src/native/runtime-base/shared-constants.hh b/src/native/runtime-base/shared-constants.hh index d2c0cfcd08d..d21c99fa79f 100644 --- a/src/native/runtime-base/shared-constants.hh +++ b/src/native/runtime-base/shared-constants.hh @@ -124,6 +124,13 @@ namespace xamarin::android::internal static constexpr std::string_view xamarin_native_tracing_name { "libxamarin-native-tracing.so" }; static constexpr hash_t xamarin_native_tracing_name_hash = xxhash::hash (xamarin_native_tracing_name); + + static constexpr bool is_64_bit_target = __SIZEOF_POINTER__ == 8; +#if defined(DEBUG) + static constexpr bool debug_build = true; +#else + static constexpr bool debug_build = false; +#endif }; } #endif // __SHARED_CONSTANTS_HH diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs index c99a23d0636..52e6f20fbf8 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs @@ -61,6 +61,7 @@ public static (IList? explorers, string? errorMessage) Op return (null, $"File '{inputFile}' is a ZIP archive, but not an Android one."); case FileFormat.AssemblyStore: + case FileFormat.ELF: return (new List { new AssemblyStoreExplorer (info)}, null); case FileFormat.Aab: diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs index cc39baa6ecc..982b62645e9 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs @@ -58,10 +58,12 @@ protected AssemblyStoreReader (Stream store, string path) protected abstract bool IsSupported (); protected abstract void Prepare (); + protected abstract ulong GetStoreStartDataOffset (); public Stream ReadEntryImageData (AssemblyStoreItem entry, bool uncompressIfNeeded = false) { - StoreStream.Seek (entry.DataOffset, SeekOrigin.Begin); + ulong startOffset = GetStoreStartDataOffset (); + StoreStream.Seek ((uint)startOffset + entry.DataOffset, SeekOrigin.Begin); var stream = new MemoryStream (); if (uncompressIfNeeded) { @@ -78,6 +80,7 @@ public Stream ReadEntryImageData (AssemblyStoreItem entry, bool uncompressIfNeed remainingToRead -= (long)nread; } stream.Flush (); + stream.Seek (0, SeekOrigin.Begin); return stream; } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs b/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs new file mode 100644 index 00000000000..932f151c80f --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs @@ -0,0 +1,11 @@ +namespace Xamarin.Android.AssemblyStore; + +enum ELFPayloadError +{ + None, + NotELF, + LoadFailed, + NotSharedLibrary, + NotLittleEndian, + NoPayloadSection, +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs b/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs index 4a02e0ae0c3..7cca3f8d0a1 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs @@ -6,6 +6,7 @@ enum FileFormat AabBase, Apk, AssemblyStore, + ELF, Zip, Unknown, } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs index d907721c088..f60fa3a067b 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs @@ -31,4 +31,6 @@ protected override bool IsSupported () protected override void Prepare () { } + + protected override ulong GetStoreStartDataOffset () => 0; } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index f434139387a..a43960102e2 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -31,6 +31,7 @@ partial class StoreReader_V2 : AssemblyStoreReader readonly HashSet supportedVersions; Header? header; + ulong elfOffset = 0; static StoreReader_V2 () { @@ -83,12 +84,34 @@ public StoreReader_V2 (Stream store, string path) static string GetBlobName (string abi) => $"libassemblies.{abi}.blob.so"; + protected override ulong GetStoreStartDataOffset () => elfOffset; + protected override bool IsSupported () { StoreStream.Seek (0, SeekOrigin.Begin); using var reader = CreateReader (); uint magic = reader.ReadUInt32 (); + if (magic == Utils.ELF_MAGIC) { + ELFPayloadError error; + (elfOffset, _, error) = Utils.FindELFPayloadSectionOffsetAndSize (StoreStream); + + if (error != ELFPayloadError.None) { + string message = error switch { + ELFPayloadError.NotELF => $"Store '{StorePath}' is not a valid ELF binary", + ELFPayloadError.LoadFailed => $"Store '{StorePath}' could not be loaded", + ELFPayloadError.NotSharedLibrary => $"Store '{StorePath}' is not a shared ELF library", + ELFPayloadError.NotLittleEndian => $"Store '{StorePath}' is not a little-endian ELF image", + ELFPayloadError.NoPayloadSection => $"Store '{StorePath}' does not contain the 'payload' section", + _ => $"Unknown ELF payload section error for store '{StorePath}': {error}" + }; + Log.Debug (message); + } else if (elfOffset >= 0) { + StoreStream.Seek ((long)elfOffset, SeekOrigin.Begin); + magic = reader.ReadUInt32 (); + } + } + if (magic != Utils.ASSEMBLY_STORE_MAGIC) { Log.Debug ($"Store '{StorePath}' has invalid header magic number."); return false; @@ -126,7 +149,7 @@ protected override void Prepare () AssemblyCount = header.entry_count; IndexEntryCount = header.index_entry_count; - StoreStream.Seek (Header.NativeSize, SeekOrigin.Begin); + StoreStream.Seek ((long)elfOffset + Header.NativeSize, SeekOrigin.Begin); using var reader = CreateReader (); var index = new List (); diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs index d36d86fbd71..284f501c9ac 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs @@ -2,6 +2,8 @@ using System.IO; using System.Buffers; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -23,9 +25,74 @@ static class Utils public const uint ZIP_MAGIC = 0x4034b50; public const uint ASSEMBLY_STORE_MAGIC = 0x41424158; + public const uint ELF_MAGIC = 0x464c457f; public static readonly ArrayPool BytePool = ArrayPool.Shared; + public static (ulong offset, ulong size, ELFPayloadError error) FindELFPayloadSectionOffsetAndSize (Stream stream) + { + stream.Seek (0, SeekOrigin.Begin); + Class elfClass = ELFReader.CheckELFType (stream); + if (elfClass == Class.NotELF) { + return ReturnError (null, ELFPayloadError.NotELF); + } + + if (!ELFReader.TryLoad (stream, shouldOwnStream: false, out IELF? elf)) { + return ReturnError (elf, ELFPayloadError.LoadFailed); + } + + if (elf.Type != FileType.SharedObject) { + return ReturnError (elf, ELFPayloadError.NotSharedLibrary); + } + + if (elf.Endianess != ELFSharp.Endianess.LittleEndian) { + return ReturnError (elf, ELFPayloadError.NotLittleEndian); + } + + if (!elf.TryGetSection ("payload", out ISection? payloadSection)) { + return ReturnError (elf, ELFPayloadError.NoPayloadSection); + } + + bool is64 = elf.Machine switch { + Machine.ARM => false, + Machine.Intel386 => false, + + Machine.AArch64 => true, + Machine.AMD64 => true, + + _ => throw new NotSupportedException ($"Unsupported ELF architecture '{elf.Machine}'") + }; + + ulong offset; + ulong size; + + if (is64) { + (offset, size) = GetOffsetAndSize64 ((Section)payloadSection); + } else { + (offset, size) = GetOffsetAndSize32 ((Section)payloadSection); + } + + elf.Dispose (); + return (offset, size, ELFPayloadError.None); + + (ulong offset, ulong size) GetOffsetAndSize64 (Section payload) + { + return (payload.Offset, payload.Size); + } + + (ulong offset, ulong size) GetOffsetAndSize32 (Section payload) + { + return ((ulong)payload.Offset, (ulong)payload.Size); + } + + (ulong offset, ulong size, ELFPayloadError error) ReturnError (IELF? elf, ELFPayloadError error) + { + elf?.Dispose (); + + return (0, 0, error); + } + } + public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) { if (String.IsNullOrEmpty (path)) { @@ -42,6 +109,7 @@ public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) // ATM, all formats we recognize have 4-byte magic at the start FileFormat format = reader.ReadUInt32 () switch { Utils.ZIP_MAGIC => FileFormat.Zip, + Utils.ELF_MAGIC => FileFormat.ELF, Utils.ASSEMBLY_STORE_MAGIC => FileFormat.AssemblyStore, _ => FileFormat.Unknown }; diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index 81520d99671..ec4c90fecb1 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -19,6 +19,7 @@ +