From d57dfffca47f8bbae256b1bcc4886bc4c41a6921 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 4 Jun 2020 22:00:28 +0200 Subject: [PATCH] [tools] TypeMap dumper utility Utility to read and dump to files/screen the typemap contents found in `libxamarin-android.so` --- Directory.Build.props | 2 + Xamarin.Android.sln | 9 +- .../ProcessPlotCSVFile.csproj | 3 +- .../xaprepare/xaprepare/xaprepare.csproj | 2 +- src/monodroid/jni/application_dso_stub.cc | 2 + src/monodroid/jni/xamarin-app.hh | 3 + tools/jit-times/jit-times.csproj | 2 +- tools/scripts/tmt | 10 + tools/tmt/AnELF.cs | 322 ++++++++++++ tools/tmt/AndroidArch.cs | 17 + tools/tmt/ApkManagedTypeResolver.cs | 99 ++++ tools/tmt/Directory.Build.targets | 6 + tools/tmt/ELF32.cs | 88 ++++ tools/tmt/ELF64.cs | 80 +++ tools/tmt/FastDevTypeMap.cs | 423 +++++++++++++++ tools/tmt/FilesystemManagedTypeResolver.cs | 44 ++ tools/tmt/ITypemap.cs | 16 + tools/tmt/Loader.cs | 322 ++++++++++++ tools/tmt/Log.cs | 52 ++ tools/tmt/ManagedTypeResolver.cs | 107 ++++ tools/tmt/Map.cs | 22 + tools/tmt/MapArchitecture.cs | 11 + tools/tmt/MapEntry.cs | 14 + tools/tmt/MapJavaType.cs | 14 + tools/tmt/MapKind.cs | 8 + tools/tmt/MapManagedType.cs | 29 ++ tools/tmt/README.md | 418 +++++++++++++++ tools/tmt/Report.cs | 351 +++++++++++++ tools/tmt/Utilities.cs | 39 ++ tools/tmt/XamarinAppDSO.cs | 150 ++++++ tools/tmt/XamarinAppDebugDSO.cs | 255 ++++++++++ tools/tmt/XamarinAppReleaseDSO.cs | 480 ++++++++++++++++++ tools/tmt/main.cs | 180 +++++++ tools/tmt/tmt.csproj | 23 + 34 files changed, 3599 insertions(+), 4 deletions(-) create mode 100755 tools/scripts/tmt create mode 100644 tools/tmt/AnELF.cs create mode 100644 tools/tmt/AndroidArch.cs create mode 100644 tools/tmt/ApkManagedTypeResolver.cs create mode 100644 tools/tmt/Directory.Build.targets create mode 100644 tools/tmt/ELF32.cs create mode 100644 tools/tmt/ELF64.cs create mode 100644 tools/tmt/FastDevTypeMap.cs create mode 100644 tools/tmt/FilesystemManagedTypeResolver.cs create mode 100644 tools/tmt/ITypemap.cs create mode 100644 tools/tmt/Loader.cs create mode 100644 tools/tmt/Log.cs create mode 100644 tools/tmt/ManagedTypeResolver.cs create mode 100644 tools/tmt/Map.cs create mode 100644 tools/tmt/MapArchitecture.cs create mode 100644 tools/tmt/MapEntry.cs create mode 100644 tools/tmt/MapJavaType.cs create mode 100644 tools/tmt/MapKind.cs create mode 100644 tools/tmt/MapManagedType.cs create mode 100644 tools/tmt/README.md create mode 100644 tools/tmt/Report.cs create mode 100644 tools/tmt/Utilities.cs create mode 100644 tools/tmt/XamarinAppDSO.cs create mode 100644 tools/tmt/XamarinAppDebugDSO.cs create mode 100644 tools/tmt/XamarinAppReleaseDSO.cs create mode 100644 tools/tmt/main.cs create mode 100644 tools/tmt/tmt.csproj diff --git a/Directory.Build.props b/Directory.Build.props index b2149d14161..48e20b0c5ca 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -29,6 +29,8 @@ 0.4.1 0.11.2 5.4.0 + 1.1.11 + 6.6.0.161 diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 2b4d6c3c192..9ba70f24857 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -146,7 +146,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "apksigner", "src\apksigner\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "java-source-utils", "external\Java.Interop\tools\java-source-utils\java-source-utils.csproj", "{37FCD325-1077-4603-98E7-4509CAD648D6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "decompress-assemblies", "tools\decompress-assemblies\decompress-assemblies.csproj", "{88B746FF-8D6E-464D-9D66-FF2ECCF148E0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "tools\decompress-assemblies\decompress-assemblies.csproj", "{88B746FF-8D6E-464D-9D66-FF2ECCF148E0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution @@ -408,6 +410,10 @@ Global {88B746FF-8D6E-464D-9D66-FF2ECCF148E0}.Debug|AnyCPU.Build.0 = Debug|Any CPU {88B746FF-8D6E-464D-9D66-FF2ECCF148E0}.Release|AnyCPU.ActiveCfg = Release|Any CPU {88B746FF-8D6E-464D-9D66-FF2ECCF148E0}.Release|AnyCPU.Build.0 = Release|Any CPU + {1A273ED2-AE84-48E9-9C23-E978C2D0CB34}.Debug|AnyCPU.ActiveCfg = Debug|anycpu + {1A273ED2-AE84-48E9-9C23-E978C2D0CB34}.Debug|AnyCPU.Build.0 = Debug|anycpu + {1A273ED2-AE84-48E9-9C23-E978C2D0CB34}.Release|AnyCPU.ActiveCfg = Release|anycpu + {1A273ED2-AE84-48E9-9C23-E978C2D0CB34}.Release|AnyCPU.Build.0 = Release|anycpu EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -476,6 +482,7 @@ Global {9A9EF774-6EA6-414F-9D2F-DCD66C56B92A} = {04E3E11E-B47D-4599-8AFC-50515A95E715} {37FCD325-1077-4603-98E7-4509CAD648D6} = {864062D3-A415-4A6F-9324-5820237BA058} {88B746FF-8D6E-464D-9D66-FF2ECCF148E0} = {864062D3-A415-4A6F-9324-5820237BA058} + {1A273ED2-AE84-48E9-9C23-E978C2D0CB34} = {864062D3-A415-4A6F-9324-5820237BA058} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {53A1F287-EFB2-4D97-A4BB-4A5E145613F6} diff --git a/build-tools/plots-to-appinsights/ProcessPlotCSVFile.csproj b/build-tools/plots-to-appinsights/ProcessPlotCSVFile.csproj index 02445bcf896..6e9e5916ce9 100644 --- a/build-tools/plots-to-appinsights/ProcessPlotCSVFile.csproj +++ b/build-tools/plots-to-appinsights/ProcessPlotCSVFile.csproj @@ -1,4 +1,5 @@ + Exe @@ -9,7 +10,7 @@ - + diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj index 7e866f6c00f..eff149b6032 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.csproj +++ b/build-tools/xaprepare/xaprepare/xaprepare.csproj @@ -55,7 +55,7 @@ - + diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 088197db324..a949c02dffb 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -5,6 +5,8 @@ // This file MUST have "valid" values everywhere - the DSO it is compiled into is loaded by the // designer on desktop. +uint64_t format_tag = FORMAT_TAG; + #if defined (DEBUG) || !defined (ANDROID) static TypeMapEntry java_to_managed[] = {}; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index fcdaf34a10b..b6303dfddca 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -8,6 +8,7 @@ #include "monodroid.h" +static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian @@ -111,6 +112,8 @@ struct ApplicationConfig const char *android_package_name; }; +MONO_API uint64_t format_tag; + #if defined (DEBUG) || !defined (ANDROID) MONO_API const TypeMap type_map; // MUST match src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs #else diff --git a/tools/jit-times/jit-times.csproj b/tools/jit-times/jit-times.csproj index c4173950203..be61d248de0 100644 --- a/tools/jit-times/jit-times.csproj +++ b/tools/jit-times/jit-times.csproj @@ -10,6 +10,6 @@ $(XAInstallPrefix)xbuild\Xamarin\Android\ - + diff --git a/tools/scripts/tmt b/tools/scripts/tmt new file mode 100755 index 00000000000..cfc361b5c17 --- /dev/null +++ b/tools/scripts/tmt @@ -0,0 +1,10 @@ +#!/bin/bash +truepath=$(readlink "$0" || echo "$0") +mydir=$(dirname ${truepath}) +binariesdir="${mydir}/typemap-tool" + +if [ -x "${binariesdir}/tmt" ]; then + exec "${binariesdir}/tmt" "$@" +else + exec dotnet "${binariesdir}/tmt.dll" +fi diff --git a/tools/tmt/AnELF.cs b/tools/tmt/AnELF.cs new file mode 100644 index 00000000000..f953a533806 --- /dev/null +++ b/tools/tmt/AnELF.cs @@ -0,0 +1,322 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; + +using ELFSharp; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace tmt +{ + abstract class AnELF + { + protected static readonly byte[] EmptyArray = new byte[0]; + + const string DynsymSectionName = ".dynsym"; + const string SymtabSectionName = ".symtab"; + const string RodataSectionName = ".rodata"; + + ISymbolTable dynamicSymbolsSection; + ISection rodataSection; + ISymbolTable? symbolsSection; + string filePath; + IELF elf; + Stream elfStream; + + protected ISymbolTable DynSymSection => dynamicSymbolsSection; + protected ISymbolTable? SymSection => symbolsSection; + protected ISection RodataSection => rodataSection; + protected IELF AnyELF => elf; + protected Stream ELFStream => elfStream; + + public string FilePath => filePath; + public MapArchitecture MapArchitecture => GetMapArchitecture (); + + public abstract bool Is64Bit { get; } + public abstract string Bitness { get; } + + protected AnELF (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + { + this.filePath = filePath; + this.elf = elf; + elfStream = stream; + dynamicSymbolsSection = dynsymSection; + this.rodataSection = rodataSection; + symbolsSection = symSection; + } + + protected ISymbolEntry? GetSymbol (string symbolName) + { + ISymbolEntry? symbol = null; + + if (symbolsSection != null) + symbol = GetSymbol (symbolsSection, symbolName); + + if (symbol == null) + symbol = GetSymbol (dynamicSymbolsSection, symbolName); + + return symbol; + } + + protected static ISymbolEntry? GetSymbol (ISymbolTable symtab, string symbolName) + { + return symtab.Entries.Where (entry => String.Compare (entry.Name, symbolName, StringComparison.Ordinal) == 0).FirstOrDefault (); + } + + protected static SymbolEntry? GetSymbol (SymbolTable symtab, T symbolValue) where T: struct + { + return symtab.Entries.Where (entry => entry.Value.Equals (symbolValue)).FirstOrDefault (); + } + + public bool HasSymbol (string symbolName) + { + return GetSymbol (symbolName) != null; + } + + public byte[] GetData (string symbolName) + { + Log.Debug ($"Looking for symbol: {symbolName}"); + ISymbolEntry? symbol = GetSymbol (symbolName); + if (symbol == null) + return EmptyArray; + + if (Is64Bit) { + var symbol64 = symbol as SymbolEntry; + if (symbol64 == null) + throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 64-bit symbol"); + return GetData (symbol64); + } + + var symbol32 = symbol as SymbolEntry; + if (symbol32 == null) + throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 32-bit symbol"); + + return GetData (symbol32); + } + + public abstract byte[] GetData (ulong symbolValue, ulong size); + + public string GetASCIIZ (ulong symbolValue) + { + return GetASCIIZ (GetData (symbolValue, 0), 0); + } + + public string GetASCIIZ (byte[] data, ulong offset) + { + if (offset >= (ulong)data.LongLength) + throw new InvalidOperationException ("Not enough data to retrieve an ASCIIZ string"); + + int count = data.Length; + + for (ulong i = offset; i < (ulong)data.LongLength; i++) { + if (data[i] == 0) { + count = (int)(i - offset); + break; + } + } + + return Encoding.ASCII.GetString (data, (int)offset, count); + } + + protected virtual byte[] GetData (SymbolEntry symbol) + { + throw new NotSupportedException (); + } + + protected virtual byte[] GetData (SymbolEntry symbol) + { + throw new NotSupportedException (); + } + + protected byte[] GetData (ISymbolEntry symbol, ulong size, ulong offset) + { + return GetData (symbol.PointedSection, size, offset); + } + + protected byte[] GetData (ISection section, ulong size, ulong offset) + { + Log.Debug ($"AnELF.GetData: section == {section.Name}; size == {size}; offset == {offset:X}"); + byte[] data = section.GetContents (); + + Log.Debug ($" data length: {data.Length} (long: {data.LongLength})"); + Log.Debug ($" offset: {offset}; size: {size}"); + if ((ulong)data.LongLength < (offset + size)) { + return EmptyArray; + } + + if (size == 0) + size = (ulong)data.Length - offset; + + var ret = new byte[size]; + checked { + Array.Copy (data, (int)offset, ret, 0, (int)size); + } + + return ret; + } + + public uint GetUInt32 (string symbolName) + { + return GetUInt32 (GetData (symbolName), 0, symbolName); + } + + public uint GetUInt32 (ulong symbolValue) + { + return GetUInt32 (GetData (symbolValue, 4), 0, symbolValue.ToString ()); + } + + protected uint GetUInt32 (byte[] data, ulong offset, string symbolName) + { + if (data.Length < 4) { + throw new InvalidOperationException ($"Data not big enough to retrieve a 32-bit integer from it (need 4, got {data.Length})"); + } + + return BitConverter.ToUInt32 (GetIntegerData (4, data, offset, symbolName), 0); + } + + public ulong GetUInt64 (string symbolName) + { + return GetUInt64 (GetData (symbolName), 0, symbolName); + } + + public ulong GetUInt64 (ulong symbolValue) + { + return GetUInt64 (GetData (symbolValue, 8), 0, symbolValue.ToString ()); + } + + protected ulong GetUInt64 (byte[] data, ulong offset, string symbolName) + { + if (data.Length < 8) + throw new InvalidOperationException ("Data not big enough to retrieve a 64-bit integer from it"); + return BitConverter.ToUInt64 (GetIntegerData (8, data, offset, symbolName), 0); + } + + byte[] GetIntegerData (uint size, byte[] data, ulong offset, string symbolName) + { + if ((ulong)data.LongLength < (offset + size)) { + string bits = size == 4 ? "32" : "64"; + throw new InvalidOperationException ($"Unable to read UInt{bits} value for symbol '{symbolName}': data not long enough"); + } + + byte[] ret = new byte[size]; + Array.Copy (data, (int)offset, ret, 0, ret.Length); + Endianess myEndianness = BitConverter.IsLittleEndian ? Endianess.LittleEndian : Endianess.BigEndian; + if (AnyELF.Endianess != myEndianness) { + Array.Reverse (ret); + } + + return ret; + } + + public static bool TryLoad (Stream stream, string filePath, out AnELF? anElf) + { + anElf = null; + Class elfClass = ELFReader.CheckELFType (stream); + if (elfClass == Class.NotELF) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not an ELF binary"); + return false; + } + + IELF elf = ELFReader.Load (stream, shouldOwnStream: false); + + if (elf.Type != FileType.SharedObject) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not a shared library"); + return false; + } + + if (elf.Endianess != Endianess.LittleEndian) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not a little-endian binary"); + return false; + } + + bool is64; + switch (elf.Machine) { + case Machine.ARM: + case Machine.Intel386: + is64 = false; + + break; + + case Machine.AArch64: + case Machine.AMD64: + is64 = true; + + break; + + default: + Log.Warning ($"{filePath} is for an unsupported machine type {elf.Machine}"); + return false; + } + + ISymbolTable? symtab = GetSymbolTable (elf, DynsymSectionName); + if (symtab == null) { + Log.Warning ($"{filePath} does not contain dynamic symbol section '{DynsymSectionName}'"); + return false; + } + ISymbolTable dynsym = symtab; + + ISection? sec = GetSection (elf, RodataSectionName); + if (sec == null) { + Log.Warning ("${filePath} does not contain read-only data section ('{RodataSectionName}')"); + return false; + } + ISection rodata = sec; + + ISymbolTable? sym = GetSymbolTable (elf, SymtabSectionName); + + if (is64) { + anElf = new ELF64 (stream, filePath, elf, dynsym, rodata, sym); + } else { + anElf = new ELF32 (stream, filePath, elf, dynsym, rodata, sym); + } + + Log.Debug ($"AnELF.TryLoad: {filePath} is a {anElf.Bitness}-bit ELF binary ({elf.Machine})"); + return true; + } + + protected static ISymbolTable? GetSymbolTable (IELF elf, string sectionName) + { + ISection? section = GetSection (elf, sectionName); + if (section == null) { + return null; + } + + var symtab = section as ISymbolTable; + if (symtab == null) { + return null; + } + + return symtab; + } + + protected static ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } + + MapArchitecture GetMapArchitecture () + { + switch (AnyELF.Machine) { + case Machine.ARM: + return MapArchitecture.ARM; + + case Machine.Intel386: + return MapArchitecture.X86; + + case Machine.AArch64: + return MapArchitecture.ARM64; + + case Machine.AMD64: + return MapArchitecture.X86_64; + + default: + throw new InvalidOperationException ($"Unsupported ELF machine type {AnyELF.Machine}"); + } + } + } +} diff --git a/tools/tmt/AndroidArch.cs b/tools/tmt/AndroidArch.cs new file mode 100644 index 00000000000..fde4758bed3 --- /dev/null +++ b/tools/tmt/AndroidArch.cs @@ -0,0 +1,17 @@ +using System; + +namespace tmt +{ + [Flags] + enum AndroidArch + { + None = 0x00, + + ARM = 0x01, + ARM64 = 0x02, + X86 = 0x04, + X86_64 = 0x08, + + All = ARM | ARM64 | X86 | X86_64, + } +} diff --git a/tools/tmt/ApkManagedTypeResolver.cs b/tools/tmt/ApkManagedTypeResolver.cs new file mode 100644 index 00000000000..1860bd2a5a3 --- /dev/null +++ b/tools/tmt/ApkManagedTypeResolver.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using K4os.Compression.LZ4; +using Mono.Cecil; +using Xamarin.Tools.Zip; + +namespace tmt +{ + class ApkManagedTypeResolver : ManagedTypeResolver + { + const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + + Dictionary assemblies; + ZipArchive apk; + + public ApkManagedTypeResolver (ZipArchive apk, string assemblyEntryPrefix) + { + this.apk = apk; + assemblies = new Dictionary (StringComparer.Ordinal); + + foreach (ZipEntry entry in apk) { + if (!entry.FullName.StartsWith (assemblyEntryPrefix, StringComparison.Ordinal)) { + continue; + } + + if (!entry.FullName.EndsWith (".dll", StringComparison.Ordinal)) { + continue; + } + + assemblies.Add (Path.GetFileNameWithoutExtension (entry.FullName), entry); + assemblies.Add (entry.FullName, entry); + } + } + + protected override string? FindAssembly (string assemblyName) + { + if (assemblies.Count == 0) { + return null; + } + + if (!assemblies.TryGetValue (assemblyName, out ZipEntry? entry) || entry == null) { + return null; + } + + return entry.FullName; + } + + protected override AssemblyDefinition ReadAssembly (string assemblyPath) + { + if (!assemblies.TryGetValue (assemblyPath, out ZipEntry? entry) || entry == null) { + // Should "never" happen - if the assembly wasn't there, FindAssembly should have returned `null` + throw new InvalidOperationException ($"Should not happen: assembly {assemblyPath} not found in the APK archive."); + } + + byte[]? assemblyBytes = null; + var stream = new MemoryStream (); + entry.Extract (stream); + stream.Seek (0, SeekOrigin.Begin); + + // + // LZ4 compressed assembly header format: + // uint magic; // 0x5A4C4158; 'XALZ', little-endian + // uint descriptor_index; // Index into an internal assembly descriptor table + // uint uncompressed_length; // Size of assembly, uncompressed + // + using (var reader = new BinaryReader (stream)) { + uint magic = reader.ReadUInt32 (); + if (magic == CompressedDataMagic) { + reader.ReadUInt32 (); // descriptor index, ignore + uint decompressedLength = reader.ReadUInt32 (); + + int inputLength = (int)(stream.Length - 12); + byte[] sourceBytes = Utilities.BytePool.Rent (inputLength); + reader.Read (sourceBytes, 0, inputLength); + + assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength); + int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength); + if (decoded != (int)decompressedLength) { + throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})"); + } + Utilities.BytePool.Return (sourceBytes); + } + } + + if (assemblyBytes != null) { + stream.Close (); + stream.Dispose (); + stream = new MemoryStream (); + stream.Write (assemblyBytes, 0, assemblyBytes.Length); + Utilities.BytePool.Return (assemblyBytes); + stream.Seek (0, SeekOrigin.Begin); + } + + return AssemblyDefinition.ReadAssembly (stream); + } + } +} diff --git a/tools/tmt/Directory.Build.targets b/tools/tmt/Directory.Build.targets new file mode 100644 index 00000000000..e5976c4e009 --- /dev/null +++ b/tools/tmt/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/tmt/ELF32.cs b/tools/tmt/ELF32.cs new file mode 100644 index 00000000000..64004277397 --- /dev/null +++ b/tools/tmt/ELF32.cs @@ -0,0 +1,88 @@ +using System; +using System.IO; + +using ELFSharp; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace tmt +{ + class ELF32 : AnELF + { + public override bool Is64Bit => false; + public override string Bitness => "32"; + + SymbolTable DynamicSymbols => (SymbolTable)DynSymSection; + SymbolTable? Symbols => (SymbolTable?)SymSection; + Section Rodata => (Section)RodataSection; + ELF ELF => (ELF)AnyELF; + + public ELF32 (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + : base (stream, filePath, elf, dynsymSection, rodataSection, symSection) + {} + + public override byte[] GetData (ulong symbolValue, ulong size = 0) + { + checked { + return GetData ((uint)symbolValue, size); + } + } + + byte[] GetData (uint symbolValue, ulong size) + { + Log.Debug ($"ELF64.GetData: Looking for symbol value {symbolValue:X08}"); + + SymbolEntry? symbol = GetSymbol (DynamicSymbols, symbolValue); + if (symbol == null && Symbols != null) { + symbol = GetSymbol (Symbols, symbolValue); + } + + if (symbol != null) { + Log.Debug ($"ELF64.GetData: found in section {symbol.PointedSection.Name}"); + return GetData (symbol); + } + + Section section = FindSectionForValue (symbolValue); + + Log.Debug ($"ELF64.GetData: found in section {section} {section.Name}"); + return GetData (section, size, OffsetInSection (section, symbolValue)); + } + + protected override byte[] GetData (SymbolEntry symbol) + { + ulong offset = symbol.Value - symbol.PointedSection.LoadAddress; + return GetData (symbol, symbol.Size, offset); + } + + Section FindSectionForValue (uint symbolValue) + { + Log.Debug ($"FindSectionForValue ({symbolValue:X08})"); + int nsections = ELF.Sections.Count; + + for (int i = nsections - 1; i >= 0; i--) { + Section section = ELF.GetSection (i); + if (section.Type != SectionType.ProgBits) + continue; + + if (SectionInRange (section, symbolValue)) + return section; + } + + throw new InvalidOperationException ($"Section matching symbol value {symbolValue:X08} cannot be found"); + } + + bool SectionInRange (Section section, uint symbolValue) + { + Log.Debug ($"SectionInRange ({section.Name}, {symbolValue:X08})"); + Log.Debug ($" address == {section.LoadAddress:X08}; size == {section.Size}; last address = {section.LoadAddress + section.Size:X08}"); + Log.Debug ($" symbolValue >= section.LoadAddress? {symbolValue >= section.LoadAddress}"); + Log.Debug ($" (section.LoadAddress + section.Size) >= symbolValue? {(section.LoadAddress + section.Size) >= symbolValue}"); + return symbolValue >= section.LoadAddress && (section.LoadAddress + section.Size) >= symbolValue; + } + + ulong OffsetInSection (Section section, uint symbolValue) + { + return symbolValue - section.LoadAddress; + } + } +} diff --git a/tools/tmt/ELF64.cs b/tools/tmt/ELF64.cs new file mode 100644 index 00000000000..abab8970b32 --- /dev/null +++ b/tools/tmt/ELF64.cs @@ -0,0 +1,80 @@ +using System; +using System.IO; + +using ELFSharp; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace tmt +{ + class ELF64 : AnELF + { + public override bool Is64Bit => true; + public override string Bitness => "64"; + + SymbolTable DynamicSymbols => (SymbolTable)DynSymSection; + SymbolTable? Symbols => (SymbolTable?)SymSection; + Section Rodata => (Section)RodataSection; + ELF ELF => (ELF)AnyELF; + + public ELF64 (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + : base (stream, filePath, elf, dynsymSection, rodataSection, symSection) + {} + + public override byte[] GetData (ulong symbolValue, ulong size = 0) + { + Log.Debug ($"ELF64.GetData: Looking for symbol value {symbolValue:X08}"); + + SymbolEntry? symbol = GetSymbol (DynamicSymbols, symbolValue); + if (symbol == null && Symbols != null) { + symbol = GetSymbol (Symbols, symbolValue); + } + + if (symbol != null) { + Log.Debug ($"ELF64.GetData: found in section {symbol.PointedSection.Name}"); + return GetData (symbol); + } + + Section section = FindSectionForValue (symbolValue); + + Log.Debug ($"ELF64.GetData: found in section {section} {section.Name}"); + return GetData (section, size, OffsetInSection (section, symbolValue)); + } + + protected override byte[] GetData (SymbolEntry symbol) + { + return GetData (symbol, symbol.Size, OffsetInSection (symbol.PointedSection, symbol.Value)); + } + + Section FindSectionForValue (ulong symbolValue) + { + Log.Debug ($"FindSectionForValue ({symbolValue:X08})"); + int nsections = ELF.Sections.Count; + + for (int i = nsections - 1; i >= 0; i--) { + Section section = ELF.GetSection (i); + if (section.Type != SectionType.ProgBits) + continue; + + if (SectionInRange (section, symbolValue)) + return section; + } + + throw new InvalidOperationException ($"Section matching symbol value {symbolValue:X08} cannot be found"); + } + + bool SectionInRange (Section section, ulong symbolValue) + { + Log.Debug ($"SectionInRange ({section.Name}, {symbolValue:X08})"); + Log.Debug ($" address == {section.LoadAddress:X08}; size == {section.Size}; last address = {section.LoadAddress + section.Size:X08}"); + Log.Debug ($" symbolValue >= section.LoadAddress? {symbolValue >= section.LoadAddress}"); + Log.Debug ($" (section.LoadAddress + section.Size) >= symbolValue? {(section.LoadAddress + section.Size) >= symbolValue}"); + return symbolValue >= section.LoadAddress && (section.LoadAddress + section.Size) >= symbolValue; + } + + ulong OffsetInSection (Section section, ulong symbolValue) + { + return symbolValue - section.LoadAddress; + } + } +} diff --git a/tools/tmt/FastDevTypeMap.cs b/tools/tmt/FastDevTypeMap.cs new file mode 100644 index 00000000000..9d7ae542b42 --- /dev/null +++ b/tools/tmt/FastDevTypeMap.cs @@ -0,0 +1,423 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace tmt +{ + class FastDevTypeMap : ITypemap + { + FastDevTypeMap_Version? fdtm; + + FastDevTypeMap_Version FDTM => fdtm ?? throw new InvalidOperationException ("Format implementation not found"); + + public MapArchitecture MapArchitecture => MapArchitecture.FastDev; + public string Description => fdtm?.Description ?? "FastDev typemap"; + public string FormatVersion => fdtm?.FormatVersion ?? "0"; + public Map Map => FDTM.Map; + public string FullPath => FDTM.FullPath; + + public bool CanLoad (Stream stream, string filePath) + { + FastDevTypeMap_Version? reader = new FastDevTypeMap_V2 (stream, filePath); + if (!reader.CanLoad (stream, filePath)) { + reader = null; + } + + if (reader == null) { + Log.Error ($"{filePath} format is not supported by this version of TMT"); + return false; + } + + fdtm = reader; + return true; + } + + public bool Load (string outputDirectory, bool generateFiles) + { + return FDTM.Load (outputDirectory, generateFiles); + } + } + + abstract class FastDevTypeMap_Version : ITypemap + { + // Corresponds to the `MODULE_INDEX_MAGIC` constant in src/monodroid/xamarin-app.hh + protected const UInt32 ModuleIndexMagic = 0x49544158; // 'XATI', little-endian + + // Corresponds to the `MODULE_MAGIC_NAMES` constant in src/monodroid/xamarin-app.hh + protected const UInt32 ModuleNamesMagic = 0x53544158; // 'XATS', little-endian + + public MapArchitecture MapArchitecture => MapArchitecture.FastDev; + public string Description => "FastDev typemap"; + public string FormatVersion => SupportedFormat.ToString (); + public string FullPath { get; } + + protected Stream Input { get; } + protected UInt32 Magic { get; private set; } = 0; + + protected abstract UInt32 SupportedFormat { get; } + public abstract Map Map { get; } + + public abstract bool Load (string outputDirectory, bool generateFiles); + + protected FastDevTypeMap_Version (Stream stream, string fullPath) + { + Input = stream; + FullPath = fullPath; + } + + public bool CanLoad (Stream stream, string filePath) + { + UInt32 expectedMagic = 0; + if (!GetExpectedMagic (filePath, ref expectedMagic)) { + return false; + } + + stream.Seek (0, SeekOrigin.Begin); + using (var br = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true)) { + if (br.ReadUInt32 () != expectedMagic) { + return false; + } + + if (br.ReadUInt32 () != SupportedFormat) { + return false; + } + + Magic = expectedMagic; + } + + return true; + } + + protected bool GetExpectedMagic (string filePath, ref UInt32 magic) + { + string ext = Path.GetExtension (filePath); + + if (String.Compare (ext, ".index", StringComparison.OrdinalIgnoreCase) == 0) { + magic = ModuleIndexMagic; + } else if (String.Compare (ext, ".typemap", StringComparison.OrdinalIgnoreCase) == 0) { + magic = ModuleNamesMagic; + } else { + return false; + } + + return true; + } + + protected bool EnsureValidModule (BinaryReader br, string filePath, bool primaryFile) + { + UInt32 magic = br.ReadUInt32 (); + if (magic != ModuleNamesMagic) { + string message = $"FastDev module file '{filePath}' is not valid (wrong magic number: expected 0x{ModuleNamesMagic:x}, got 0x{magic:x})"; + if (primaryFile) { + throw new InvalidOperationException (message); // Should "never" happen + } + Log.Warning (message); + return false; + } + + UInt32 formatVersion = br.ReadUInt32 (); + if (formatVersion != SupportedFormat) { + string message = $"FastDev module file '{filePath}' uses unsupported format version ({formatVersion})"; + if (primaryFile) { + throw new InvalidOperationException (message); // Should "never" happen + } + Log.Warning (message); + return false; + } + + return true; + } + } + + class FastDevTypeMap_V2 : FastDevTypeMap_Version + { + const uint InvalidJavaToManagedMappingIndex = UInt32.MaxValue; + + sealed class Entry + { + public string TypeName = String.Empty; + public string AssemblyName = String.Empty; + public string SourceFile = String.Empty; + public UInt32 MappedTypeIndex; + } + + Map? map; + List managedToJava; + List javaToManaged; + HashSet managedToJavaUsed; + HashSet javaToManagedUsed; + + public override Map Map => map ?? throw new InvalidOperationException ("Data hasn't been loaded yet"); + protected override UInt32 SupportedFormat => 2; + + public FastDevTypeMap_V2 (Stream stream, string fullPath) + : base (stream, fullPath) + { + managedToJava = new List (); + javaToManaged = new List (); + + managedToJavaUsed = new HashSet (StringComparer.Ordinal); + javaToManagedUsed = new HashSet (StringComparer.Ordinal); + } + + public override bool Load (string outputDirectory, bool generateFiles) + { + try { + if (!DoLoad (outputDirectory, generateFiles)) { + return false; + } + + map = new Map (MapKind.Debug, MapArchitecture.FastDev, managedToJava, javaToManaged, FormatVersion); + return true; + } catch (Exception ex) { + Log.ExceptionError ($"{Description}: failed to load maps", ex); + return false; + } + } + + void AddToMap (List javaToManagedSource, List managedToJavaSource) + { + foreach (Entry entry in javaToManagedSource) { + Entry managedEntry; + + if (entry.MappedTypeIndex == InvalidJavaToManagedMappingIndex) { + Log.Debug ($"Java-to-managed: entry {entry.TypeName} marked as to be ignored"); + managedEntry = new Entry { + AssemblyName = entry.AssemblyName, + TypeName = "[Ignored]", + }; + } else { + managedEntry = managedToJavaSource[(int)entry.MappedTypeIndex]; + } + + if (AlreadyUsed (entry, managedEntry, javaToManagedUsed)) { + Log.Debug ($"Java-to-managed: skipping duplicate {entry.TypeName} -> {managedEntry.TypeName}"); + continue; + } + + CreateAndAdd (entry, managedEntry, javaToManaged); + } + + foreach (Entry entry in managedToJavaSource) { + Entry javaEntry = javaToManagedSource[(int)entry.MappedTypeIndex]; + if (AlreadyUsed (entry, javaEntry, managedToJavaUsed)) { + Log.Info ($"Managed-to-java: skipping duplicate {entry.TypeName} -> {javaEntry.TypeName}"); + continue; + } + + CreateAndAdd (javaEntry, entry, managedToJava); + } + + bool AlreadyUsed (Entry from, Entry to, HashSet cache) + { + string key = $"{from.TypeName}/{to.TypeName}"; + if (cache.Contains (key)) { + return true; + } + + cache.Add (key); + return false; + } + + void CreateAndAdd (Entry java, Entry managed, List list) + { + list.Add ( + new MapEntry ( + new MapManagedType (managed.TypeName, managed.SourceFile), + new MapJavaType (java.TypeName, java.SourceFile) + ) + ); + } + } + + bool DoLoad (string outputDirectory, bool generateFiles) + { + switch (Magic) { + case ModuleIndexMagic: + return LoadIndex (); + + case ModuleNamesMagic: + return LoadModule (Input, FullPath, primaryFile: true); + + default: + Log.Error ($"File {FullPath} has an usupported magic number ({Magic})"); + return false; + } + } + + int FindFirstNUL (byte[] data) + { + for (int i = 0; i < data.Length; i++) { + if (data [i] == 0) { + return i; + } + } + + return -1; + } + + bool GetNonEmptyString (byte[] data, string filePath, int idx, ref string result) + { + int nulPosition = FindFirstNUL (data); + + if (nulPosition < 0) { + Log.Error ($"{filePath} entry at index {idx} is malformed - no terminating NUL"); + return false; + } + + if (nulPosition == 0) { + Log.Error ($"{filePath} entry at index {idx} is malformed - empty file name"); + return false; + } + + result = Encoding.UTF8.GetString (data, 0, nulPosition); + return true; + } + + bool LoadIndex () + { + string dir = Path.GetDirectoryName (FullPath) ?? String.Empty; + + Input.Seek (0, SeekOrigin.Begin); + using (var br = new BinaryReader (Input, Encoding.UTF8, leaveOpen: true)) { + br.ReadUInt32 (); // magic + br.ReadUInt32 (); // format version + + UInt32 entryCount = br.ReadUInt32 (); + if (entryCount == 0) { + Log.Error ($"FastDev index {FullPath} has no entries"); + return false; + } + + UInt32 filenameWidth = br.ReadUInt32 (); + if (filenameWidth == 0) { + Log.Error ($"FastDev index indicates file name width is 0"); + return false; + } + + bool somethingFailed = false; + for (UInt32 idx = 0; idx < entryCount; idx++) { + string name = String.Empty; + + if (!GetNonEmptyString (br.ReadBytes ((int)filenameWidth), FullPath, (int)idx, ref name)) { + somethingFailed = true; + continue; + } + + if (!LoadModule (Path.Combine (dir, name), primaryFile: false)) { + somethingFailed = true; + } + } + + return !somethingFailed; + } + } + + bool LoadModule (string filePath, bool primaryFile) + { + if (!File.Exists (filePath)) { + Log.Error ($"Module {filePath} not found"); + return false; + } + + using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read)) { + return LoadModule (fs, filePath, primaryFile); + } + } + + bool LoadModule (Stream stream, string filePath, bool primaryFile) + { + Log.Debug ($"Loading FastDev typemap from module: {filePath}"); + stream.Seek (0, SeekOrigin.Begin); + + string assemblyName = String.Empty; + var javaToManaged = new List (); + var managedToJava = new List (); + using (var br = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true)) { + if (!EnsureValidModule (br, filePath, primaryFile)) { + return false; + } + + UInt32 entryCount = br.ReadUInt32 (); + if (entryCount == 0) { + Log.Warning ($"FastDev typemap module file {filePath} has no entries"); + return false; + } + + UInt32 javaNameWidth = br.ReadUInt32 (); + if (javaNameWidth == 0) { + Log.Error ($"FastDev module {filePath} indicates Java type name width is 0"); + return false; + } + + UInt32 managedNameWidth = br.ReadUInt32 (); + if (managedNameWidth == 0) { + Log.Error ($"FastDev module {filePath} indicates managed type name width is 0"); + return false; + } + + UInt32 assemblyNameSize = br.ReadUInt32 (); + if (assemblyNameSize == 0) { + Log.Error ($"FastDev module {filePath} indicates assembly name length is 0"); + return false; + } + + byte[] data = Utilities.BytePool.Rent ((int)assemblyNameSize); + int read = br.Read (data, 0, (int)assemblyNameSize); + if (read != (int)assemblyNameSize) { + Log.Error ($"FastDev typemap module file {filePath} is too short: not enough bytes to read assembly name"); + Utilities.BytePool.Return (data); + return false; + } + assemblyName = Encoding.UTF8.GetString (data, 0, read); + Utilities.BytePool.Return (data); + + if (!LoadModuleTable (br, assemblyName, filePath, javaToManaged, entryCount, javaNameWidth)) { + Log.Error ($"Failed to read java-to-managed table from module {filePath}"); + return false; + } + + if (!LoadModuleTable (br, assemblyName, filePath, managedToJava, entryCount, managedNameWidth)) { + Log.Error ($"Failed to read managed-to-java table from module {filePath}"); + return false; + } + + AddToMap (javaToManaged, managedToJava); + } + + return true; + } + + bool LoadModuleTable (BinaryReader br, string assemblyName, string filePath, List table, UInt32 entryCount, UInt32 nameWidth) + { + byte[] data = Utilities.BytePool.Rent ((int)nameWidth); + bool somethingFailed = false; + for (UInt32 i = 0; i < entryCount; i++) { + int read = br.Read (data, 0, (int)nameWidth); + if (read != (int)nameWidth) { + Log.Error ($"Error reading module {filePath} at index {i}: missing {nameWidth - read} bytes"); + Utilities.BytePool.Return (data); + return false; + } + + string typeName = String.Empty; + if (!GetNonEmptyString (data, filePath, (int)i, ref typeName)) { + somethingFailed = true; + } + + UInt32 mappedTypeIndex = br.ReadUInt32 (); + table.Add ( + new Entry { + AssemblyName = assemblyName, + TypeName = typeName, + MappedTypeIndex = mappedTypeIndex, + } + ); + } + + Utilities.BytePool.Return (data); + return !somethingFailed; + } + } +} diff --git a/tools/tmt/FilesystemManagedTypeResolver.cs b/tools/tmt/FilesystemManagedTypeResolver.cs new file mode 100644 index 00000000000..eaa75c02cd0 --- /dev/null +++ b/tools/tmt/FilesystemManagedTypeResolver.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.IO; + +using Mono.Cecil; + +namespace tmt +{ + class FilesystemManagedTypeResolver : ManagedTypeResolver + { + List searchPaths; + bool havePaths; + + public FilesystemManagedTypeResolver (List searchPaths) + { + this.searchPaths = searchPaths; + havePaths = searchPaths.Count > 0; + } + + protected override string? FindAssembly (string assemblyName) + { + if (!havePaths) { + return null; + } + + string? assemblyPath = null; + string assemblyFileName = $"{assemblyName}.dll"; + + foreach (string dir in searchPaths) { + string path = Path.Combine (dir, assemblyFileName); + if (File.Exists (path)) { + assemblyPath = path; + break; + } + } + + return assemblyPath; + } + + protected override AssemblyDefinition ReadAssembly (string assemblyPath) + { + return AssemblyDefinition.ReadAssembly (assemblyPath); + } + } +} diff --git a/tools/tmt/ITypemap.cs b/tools/tmt/ITypemap.cs new file mode 100644 index 00000000000..8847fc0f8a9 --- /dev/null +++ b/tools/tmt/ITypemap.cs @@ -0,0 +1,16 @@ +using System.IO; + +namespace tmt +{ + interface ITypemap + { + string Description { get; } + string FormatVersion { get; } + string FullPath { get; } + Map Map { get; } + MapArchitecture MapArchitecture { get; } + + bool CanLoad (Stream stream, string filePath); + bool Load (string outputDirectory, bool generateFiles); + } +} diff --git a/tools/tmt/Loader.cs b/tools/tmt/Loader.cs new file mode 100644 index 00000000000..990570b5f13 --- /dev/null +++ b/tools/tmt/Loader.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Tools.Zip; + +namespace tmt +{ + class Loader + { + const string xamarinApp = "libxamarin-app.so"; + const string sharedLibsDirName = "app_shared_libraries"; + const string typemapIndexFileName = "typemap.index"; + + static readonly string TypemapIndexRelPath = Path.Combine ("android", "typemaps", typemapIndexFileName); + static readonly List archDirs = new List { + "arm64-v8a", + "armeabi-v7a", + "x86", + "x86_64", + }; + + static readonly char dirSep = Path.DirectorySeparatorChar; + + AndroidArch archFilter; + bool loadOnlyFirst; + + public Loader (AndroidArch archFilter, bool loadOnlyFirst) + { + this.archFilter = archFilter; + this.loadOnlyFirst = loadOnlyFirst; + } + + public List TryLoad (string path) + { + if (File.Exists (path)) { + return LoadFromFile (path); + } + + if (!Directory.Exists (path)) { + Log.Info ($"Directory or file '{path}' not found."); + return new List (); + } + + string fsPath = Path.Combine (path, typemapIndexFileName); + if (File.Exists (fsPath)) { + return LoadFromFile (fsPath); + } + + fsPath = Path.Combine (path, xamarinApp); + if (File.Exists (fsPath)) { + return LoadFromFile (fsPath); + } + + if (PathEndsWith (path, sharedLibsDirName)) { + return LoadFromSharedLibsDir (path); + } + + // typemap.index must come first, since we always create `libxamarin-app.so`, but it will be empty if + // FastDev is used, which would cause an error + fsPath = Path.Combine (path, TypemapIndexRelPath); + if (File.Exists (fsPath)) { + return LoadFromFile (fsPath); + } + + fsPath = Path.Combine (path, sharedLibsDirName); + if (Directory.Exists (fsPath)) { + return LoadFromSharedLibsDir (fsPath); + } + + if (PathEndsWith (path, "obj")) { + return LoadFromObjDir (path); + } + + fsPath = Path.Combine (path, "obj"); + if (Directory.Exists (fsPath)) { + return LoadFromObjDir (fsPath); + } + + Log.Warning ($"Cannot determine what to load from {path}"); + return new List (); + } + + List LoadFromObjDir (string objDirPath) + { + string fsPath; + + // The reason for two loops here is that we cannot guarantee the order in which subdirectories will be + // reported and we must always load FastDev typemap before we attempt to load one from `libxamarin-app.so`, + // since the latter would be empty and error out in case FastDev is used + foreach (string subdir in Directory.EnumerateDirectories (objDirPath)) { + fsPath = Path.Combine (subdir, TypemapIndexRelPath); + if (File.Exists (fsPath)) { + // We don't need to search more - typemap.index exists only in the obj/Debug directory + return LoadFromFile (fsPath); + } + } + + var ret = new List (); + foreach (string subdir in Directory.EnumerateDirectories (objDirPath)) { + fsPath = Path.Combine (subdir, sharedLibsDirName); + if (!Directory.Exists (fsPath)) { + continue; + } + + ret.AddRange (LoadFromSharedLibsDir (fsPath)); + } + + return ret; + } + + List LoadFromSharedLibsDir (string dirPath) + { + var ret = new List (); + foreach (string archDir in archDirs) { + string fsPath = Path.Combine (dirPath, archDir, xamarinApp); + if (File.Exists (fsPath)) { + ITypemap? dsoTypemap = LoadDSOFromFilesystem (fsPath); + if (dsoTypemap != null) { + ret.Add (dsoTypemap); + if (loadOnlyFirst) { + break; + } + } + } + } + + return ret; + } + + List LoadFromFile (string filePath) + { + if (String.Compare (Path.GetFileName (filePath), "typemap.index", StringComparison.OrdinalIgnoreCase) == 0) { + return CreateListAndReturn (LoadFromFastDevTypemap (filePath)); + } + + string ext = Path.GetExtension (filePath); + if (String.Compare (ext, ".typemap", StringComparison.OrdinalIgnoreCase) == 0) { + return CreateListAndReturn (LoadFromFastDevTypemap (filePath)); + } + + if (String.Compare (ext, ".so", StringComparison.OrdinalIgnoreCase) == 0) { + return CreateListAndReturn (LoadDSOFromFilesystem (filePath)); + } + + if (String.Compare (ext, ".apk", StringComparison.OrdinalIgnoreCase) == 0) { + return LoadFromAPK (filePath, "assemblies/"); + } + + if (String.Compare (ext, ".aab", StringComparison.OrdinalIgnoreCase) == 0) { + return LoadFromAPK (filePath, "base/root/assemblies/"); + } + + Log.Info ($"Unsupported file extension '{ext}', unable to load"); + return CreateListAndReturn (null); + + List CreateListAndReturn (ITypemap? tm) + { + if (tm == null) { + return new List (); + } + + return new List { + tm + }; + } + } + + ITypemap? LoadFromFastDevTypemap (string filePath) + { + ITypemap tm = new FastDevTypeMap (); + + var fs = File.Open (filePath, FileMode.Open, FileAccess.Read); + try { + if (tm.CanLoad (fs, filePath)) { + return tm; + } + } catch { + DisposeStream (fs); + throw; + } + + DisposeStream (fs); + return null; + + void DisposeStream (FileStream fs) + { + fs.Close (); + fs.Dispose (); + } + } + + List LoadFromAPK (string filePath, string assemblyEntryPrefix) + { + const string xamarinAppEntryTail = "/" + xamarinApp; + + var ret = new List (); + ZipArchive zip = ZipArchive.Open (filePath, FileMode.Open); + var managedResolver = new ApkManagedTypeResolver (zip, assemblyEntryPrefix); + foreach (ZipEntry entry in zip) { + if (!entry.FullName.EndsWith (xamarinAppEntryTail, StringComparison.Ordinal)) { + continue; + } + + var stream = new MemoryStream (); + entry.Extract (stream); + + ITypemap? tm = LoadDSO (stream, $"{filePath}!{entry.FullName}", managedResolver); + if (tm != null) { + ret.Add (tm); + if (loadOnlyFirst) { + break; + } + } + } + + return ret; + } + + ITypemap? LoadDSOFromFilesystem (string filePath) + { + filePath = Path.GetFullPath (filePath); + string? fileDir = Path.GetDirectoryName (filePath); + if (String.IsNullOrEmpty (fileDir)) { + fileDir = "./"; + } + fileDir = Path.GetFullPath (fileDir); + + string objConfigurationDir = Path.GetFullPath (Path.Combine (fileDir, "..", "..")); + if (!IsObjConfigurationDir (objConfigurationDir)) { + // We might be loading the DSO from a location outside the project tree, there will likely be no + // managed assemblies. + objConfigurationDir = String.Empty; + } + + var fs = File.Open (filePath, FileMode.Open, FileAccess.Read); + var searchPaths = new List (); + + if (objConfigurationDir.Length > 0) { + // Assemblies are kept in: + // obj/$CONFIGURATION/android/assets and + // obj/$CONFIGURATION/android/assets/shrunk + // with the latter being the preferred location + // + string assetsDir = Path.Combine (objConfigurationDir, "android", "assets"); + AddIfExists (searchPaths, Path.Combine (assetsDir, "shrunk")); + AddIfExists (searchPaths, assetsDir); + } + + return LoadDSO (fs, filePath, new FilesystemManagedTypeResolver (searchPaths)); + } + + ITypemap? LoadDSO (Stream stream, string filePath, ManagedTypeResolver managedResolver) + { + ITypemap tm = new XamarinAppDebugDSO (managedResolver, filePath); + if (tm.CanLoad (stream, filePath)) { + return ReturnIfCorrectArch (tm); + } + + tm = new XamarinAppReleaseDSO (managedResolver, filePath); + if (tm.CanLoad (stream, filePath)) { + return ReturnIfCorrectArch (tm); + } + + return null; + + ITypemap? ReturnIfCorrectArch (ITypemap tm) + { + return IsMatchingArch (tm) ? tm : null; + } + } + + bool IsMatchingArch (ITypemap tm) + { + bool matches; + + switch (tm.MapArchitecture) { + case MapArchitecture.ARM: + matches = (archFilter & AndroidArch.ARM) == AndroidArch.ARM; + break; + + case MapArchitecture.ARM64: + matches = (archFilter & AndroidArch.ARM64) == AndroidArch.ARM64; + break; + + case MapArchitecture.X86: + matches = (archFilter & AndroidArch.X86) == AndroidArch.X86; + break; + + case MapArchitecture.X86_64: + matches = (archFilter & AndroidArch.X86_64) == AndroidArch.X86_64; + break; + + case MapArchitecture.FastDev: + return true; + + default: + throw new NotSupportedException ($"Type map architecture {tm.Map.Architecture} is not supported"); + } + + return matches; + } + + bool IsObjConfigurationDir (string path) + { + return Directory.Exists (Path.Combine (path, sharedLibsDirName)); + } + + void AddIfExists (List searchPaths, string dirPath) + { + if (!Directory.Exists (dirPath)) + return; + + searchPaths.Add (Path.GetFullPath (dirPath)); + } + + bool PathEndsWith (string path, string tail) + { + return path.EndsWith ($"{dirSep}{tail}", StringComparison.OrdinalIgnoreCase) || path.EndsWith ($"{dirSep}{tail}{dirSep}", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/tools/tmt/Log.cs b/tools/tmt/Log.cs new file mode 100644 index 00000000000..e1f8ecedeb3 --- /dev/null +++ b/tools/tmt/Log.cs @@ -0,0 +1,52 @@ +using System; + +namespace tmt +{ + static class Log + { + static bool showDebug = false; + + public static void SetVerbose (bool verbose) + { + showDebug = verbose; + } + + public static void Error (string message = "") + { + if (message.Length > 0) { + Console.Error.Write ("Error: "); + } + Console.Error.WriteLine (message); + } + + public static void Warning (string message = "") + { + if (message.Length > 0) { + Console.Error.Write ("Warning: "); + } + + Console.Error.WriteLine (message); + } + + public static void Info (string message = "") + { + Console.WriteLine (message); + } + + public static void Debug (string message = "") + { + if (!showDebug) { + return; + } + + Console.WriteLine (message); + } + + public static void ExceptionError (string message, Exception ex) + { + Log.Error (message); + Log.Error ("Exception was thrown:"); + Log.Error (ex.ToString ()); + } + } +} diff --git a/tools/tmt/ManagedTypeResolver.cs b/tools/tmt/ManagedTypeResolver.cs new file mode 100644 index 00000000000..cc372f0b1f3 --- /dev/null +++ b/tools/tmt/ManagedTypeResolver.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Mono.Cecil; + +namespace tmt +{ + abstract class ManagedTypeResolver + { + protected HashSet AlreadyWarned { get; } = new HashSet (StringComparer.OrdinalIgnoreCase); + protected HashSet AlreadyLoaded { get; } = new HashSet (StringComparer.OrdinalIgnoreCase); + Dictionary types = new Dictionary (StringComparer.Ordinal); + + protected Dictionary AssemblyPaths { get; } = new Dictionary (StringComparer.OrdinalIgnoreCase); + + protected ManagedTypeResolver () + {} + + public string Lookup (string assemblyName, Guid mvid, uint tokenID) + { + if (tokenID == 0) { + return "[name unknown]"; + } + + string? assemblyPath = null; + if (!AssemblyPaths.TryGetValue (assemblyName, out assemblyPath)) { + assemblyPath = FindAssembly (assemblyName); + if (!String.IsNullOrEmpty (assemblyPath)) { + AssemblyPaths.Add (assemblyName, assemblyPath); + } + } + + if (!String.IsNullOrEmpty (assemblyPath)) { + if (TryLookup (assemblyPath, mvid, tokenID, out string typeName)) { + return typeName; + } + } else if (!AlreadyWarned.Contains (assemblyName)) { + Log.Warning ($"Assembly {assemblyName} cannot be found"); + AlreadyWarned.Add (assemblyName); + } + + return $"[token id: {tokenID}]"; + } + + protected bool TryLookup (string assemblyPath, Guid mvid, uint tokenID, out string typeName) + { + LoadAssembly (assemblyPath); + + if (tokenID == 0) { + typeName = String.Empty; + return false; + } + + string key = GetTypeKey (assemblyPath, mvid, tokenID); + typeName = String.Empty; + if (types.TryGetValue (key, out string? name)) { + typeName = name ?? String.Empty; + return true; + } + + if (!AlreadyWarned.Contains (key)) { + Log.Warning ($"Type with token ID {tokenID} ({tokenID:X08}) not found in {assemblyPath}"); + AlreadyWarned.Add (key); + } + + return false; + } + + protected abstract AssemblyDefinition ReadAssembly (string assemblyPath); + protected abstract string? FindAssembly (string assemblyName); + + void LoadAssembly (string assemblyPath) + { + if (AlreadyLoaded.Contains (assemblyPath)) { + return; + } + + Log.Debug ($"Loading '{assemblyPath}'"); + AssemblyDefinition asm = ReadAssembly (assemblyPath); + foreach (ModuleDefinition module in asm.Modules) { + foreach (TypeDefinition type in module.Types) { + LoadType (type); + } + } + AlreadyLoaded.Add (assemblyPath); + + void LoadType (TypeDefinition type) + { + Log.Debug ($" {type.Name} (token as int: {type.MetadataToken.ToUInt32()}; RID: {type.MetadataToken.RID}; type: {type.MetadataToken.TokenType})"); + types.Add (GetTypeKey (assemblyPath, type.Module.Mvid, type.MetadataToken.ToUInt32 ()), type.FullName); + if (type.NestedTypes.Count == 0) { + return; + } + + foreach (TypeDefinition nestedType in type.NestedTypes) { + LoadType (nestedType); + } + } + } + + string GetTypeKey (string assemblyPath, Guid mvid, uint tokenID) + { + return $"{assemblyPath.ToLowerInvariant ()}{mvid}{tokenID}"; + } + } +} diff --git a/tools/tmt/Map.cs b/tools/tmt/Map.cs new file mode 100644 index 00000000000..3df9c68de5a --- /dev/null +++ b/tools/tmt/Map.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; + +namespace tmt +{ + class Map + { + public MapKind Kind { get; } + public MapArchitecture Architecture { get; } + public List ManagedToJava { get; } + public List JavaToManaged { get; } + public string FormatVersion { get; } + + public Map (MapKind kind, MapArchitecture architecture, List managedToJava, List javaToManaged, string formatVersion) + { + Kind = kind; + Architecture = architecture; + ManagedToJava = managedToJava; + JavaToManaged = javaToManaged; + FormatVersion = formatVersion; + } + } +} diff --git a/tools/tmt/MapArchitecture.cs b/tools/tmt/MapArchitecture.cs new file mode 100644 index 00000000000..371bc80be88 --- /dev/null +++ b/tools/tmt/MapArchitecture.cs @@ -0,0 +1,11 @@ +namespace tmt +{ + enum MapArchitecture + { + FastDev, + X86, + X86_64, + ARM, + ARM64, + } +} diff --git a/tools/tmt/MapEntry.cs b/tools/tmt/MapEntry.cs new file mode 100644 index 00000000000..dfb7013eae8 --- /dev/null +++ b/tools/tmt/MapEntry.cs @@ -0,0 +1,14 @@ +namespace tmt +{ + class MapEntry + { + public MapManagedType ManagedType { get; } + public MapJavaType JavaType { get; } + + public MapEntry (MapManagedType managedType, MapJavaType javaType) + { + ManagedType = managedType; + JavaType = javaType; + } + } +} diff --git a/tools/tmt/MapJavaType.cs b/tools/tmt/MapJavaType.cs new file mode 100644 index 00000000000..70579672ae6 --- /dev/null +++ b/tools/tmt/MapJavaType.cs @@ -0,0 +1,14 @@ +namespace tmt +{ + class MapJavaType + { + public string Name { get; } + public string SourceFile { get; } + + public MapJavaType (string name, string sourceFile = "") + { + Name = name; + SourceFile = sourceFile; + } + } +} diff --git a/tools/tmt/MapKind.cs b/tools/tmt/MapKind.cs new file mode 100644 index 00000000000..2eaa38a7fa8 --- /dev/null +++ b/tools/tmt/MapKind.cs @@ -0,0 +1,8 @@ +namespace tmt +{ + enum MapKind + { + Debug, + Release + } +} diff --git a/tools/tmt/MapManagedType.cs b/tools/tmt/MapManagedType.cs new file mode 100644 index 00000000000..99d54746a4c --- /dev/null +++ b/tools/tmt/MapManagedType.cs @@ -0,0 +1,29 @@ +using System; + +namespace tmt +{ + class MapManagedType + { + public Guid MVID { get; } = Guid.Empty; + public uint TokenID { get; } = 0; + public string TypeName { get; set; } = String.Empty; + public string AssemblyName { get; set; } = String.Empty; + public bool IsDuplicate { get; set; } + public bool IsGeneric { get; set; } + public string SourceFile { get; } = String.Empty; + + public MapManagedType (Guid mvid, uint tokenID, string assemblyName, string sourceFile) + { + MVID = mvid; + TokenID = tokenID; + AssemblyName = assemblyName; + SourceFile = sourceFile; + } + + public MapManagedType (string name, string sourceFile = "") + { + TypeName = name; + SourceFile = sourceFile; + } + } +} diff --git a/tools/tmt/README.md b/tools/tmt/README.md new file mode 100644 index 00000000000..4739332e153 --- /dev/null +++ b/tools/tmt/README.md @@ -0,0 +1,418 @@ + +**Table of Contents** + +- [Introduction](#introduction) + - [Reporting](#reporting) + - [Filtering](#filtering) + - [Type name filtering](#type-name-filtering) + - [Architecture filtering](#architecture-filtering) +- [Output format](#output-format) + - [Debug maps](#debug-maps) + - [Java to Managed](#java-to-managed) + - [Managed to Java](#managed-to-java) + - [Release maps](#release-maps) + - [Raw output](#raw-output) + - [Java to Managed](#java-to-managed-1) + - [Managed to Java](#managed-to-java-1) + - [Processed output](#processed-output) + - [Java to Managed](#java-to-managed-2) + - [Managed to Java](#managed-to-java-2) +- [Command line options](#command-line-options) + - [Sample invocations](#sample-invocations) + - [Filter by type name](#filter-by-type-name) + - [Dump all typemaps in a project directory](#dump-all-typemaps-in-a-project-directory) + - [Dump all type maps from an apk](#dump-all-type-maps-from-an-apk) + - [Dump only x86_64 typemap from an apk](#dump-only-x86_64-typemap-from-an-apk) + - [Dump typemap from a single shared library](#dump-typemap-from-a-single-shared-library) + + + +# Introduction + +This is a utility program to read Xamarin.Android application type +maps compiled into a shared library named `libxamarin-app.so`. This +library is compiled for each native architecture separately and +packaged together with the application into the APK archive. + +Type maps are used on run time to map Managed types to Java types and +vice versa. This process differs between `Release` and `Debug` +builds, but in both cases the maps are stored as binary data in the +native shared library. This fact makes it hard to determine which +Managed/Java type maps to its counterpart (especially in the `Release` +mode where Managed type names aren't stored at all - the types are +instead identified by their assembly module's GUID and the type token ID). + +This utility can read both `Release` and `Debug` maps and is able to +find `libxamarin-app.so` when pointed to one of the following +locations: + + * Application APK/AAB archives + * Application project top directory + * `obj` subdirectory + * `obj/$CONFIGURATION` subdirectory + * Any directory which contains `libxamarin-app.so` + * Path to a concrete `libxamarin-app.so` file + +If the utility reads `Release` map from a `libxamarin-app.so` in a +directory that is not one of the above locations, it will not be able +to obtain Managed type and assembly names. + +When passed a path to APK/AAB archive, the utility will by default +load all the `libxamarin-app.so` shared libraries found in the +archive - one for each enabled target architecture. This can be +disabled by using the `-a` option (see [Filtering](#architecture-filtering) below) + +The only required parameter is a path a location containing the type +maps, as defined above. + +Optionally, the location path may be followed by a single parameter +specifying a .NET regular expression to match Managed and Java types +in order to filter the map (see [Filtering](#type-name-filtering) below) + +## Reporting + +By default, the utility generates files with full contents of type +maps read from the shared library. The output differs depending on +whether the `Release` or `Debug` configuration is found in the shared +library. + +For `Release`, two files for each native shared library are generated: + + * Raw output (.raw) + Contains uninterpreted raw data from the map. This is the most + technical, detailed format. It performs no type name lookup, + contains information about managed assemblies, their modules, + identifiers etc. + * Processed output (.txt) + Contains a comma-separated value collection of type mappings, with + most of the technical details of the raw format removed, managed + type names looked up (if possible). There are two modes for this + format - full and abbreviated. They differ in that the latter + does not contain information about Managed type module MVID or its + token ID. + +For `Debug`, only one file is generated with the same information as +the `Processed output` format above. + +In both cases, file names are constructed following the pattern below: + + typemap-vX-CONFIGURATION-ARCHITECTURE-KIND.{raw,txt} + +Where + + * `X` stands for the type map format version + * `CONFIGURATION` is either `Debug` or `Release` + * `ARCHITECTURE` is one of: `ARM`, `ARM64`, `X86`, `X86_64` + * `KIND` is either `managed` or `java` + +If regular expression type name filtering is not enabled, the utility +will not print the map contents to the console. This is because even +small applications can contain thousands of entries and the whole map +content is better written to a file instead of producing voluminous +output on the screen. + +## Filtering + +### Type name filtering + +Passing a regular expression to the utility causes it to match type +names (both Managed and Java for **each** loaded entry, regardless on +whether the current map being processed is java-to-managed or +managed-to-java) against it and report the matches to the console. + +Also, in this case the files are not generated by the default, which +can be changed by using the `-g` option. This is because filtering +mode is mostly useful for "quick look" at the maps and the full output +will likely be of no interest in this instance. + +### Architecture filtering + +By default the utility reads `libxamarin-app.so` for any architecture +it finds in the location indicated on command line. The `-a` option +can be used to limit the set of architectures by specifying a +comma-separated list of architecture names from the following set +(architecture names are case-insensitive): `ARM`, `ARM64`, `X86` and +`X86_64` + +# Output format + +## Debug maps + +### Java to Managed +``` +Java-Type-Name Managed-Type-Name Is-Duplicate-Type-Entry? +android/accessibilityservice/AccessibilityButtonController Android.AccessibilityServices.AccessibilityButtonController, Mono.Android +android/accessibilityservice/AccessibilityButtonController$AccessibilityButtonCallback Android.AccessibilityServices.AccessibilityButtonController+AccessibilityButtonCallback, Mono.Android duplicate entry +android/accessibilityservice/AccessibilityGestureEvent Android.AccessibilityServices.AccessibilityGestureEvent, Mono.Android +``` + +### Managed to Java + +``` +Managed-Type-Name Java-Type-Name Is-Duplicate-Type-Entry? +Android.AccessibilityServices.AccessibilityButtonController, Mono.Android android/accessibilityservice/AccessibilityButtonController +Android.AccessibilityServices.AccessibilityButtonController+AccessibilityButtonCallback, Mono.Android android/accessibilityservice/AccessibilityButtonController$AccessibilityButtonCallback +Android.AccessibilityServices.AccessibilityButtonController+AccessibilityButtonCallbackInvoker, Mono.Android android/accessibilityservice/AccessibilityButtonController$AccessibilityButtonCallback +``` + +## Release maps + +### Raw output + +Raw output dumps the typemap data without interpretation - it doesn't +perform `MVID+TokenID` translation into fully qualified type name. + +#### Java to Managed + +``` +MANAGED_MODULE_INDEX TYPE_TOKEN_DECIMAL (TYPE_TOKEN_HEXADECIMAL) JAVA_TYPE_NAME + 1 33554623 (020000BF) android/app/Activity + 1 33554624 (020000C0) android/app/Application + 1 33554628 (020000C4) android/app/backup/BackupAgent + 1 33554630 (020000C6) android/app/backup/BackupDataInput +``` + +#### Managed to Java + +``` +TYPE_TOKEN_DECIMAL (TYPE_TOKEN_HEXADECIMAL) JAVA_MAP_INDEX + +Module 0000: HelloLibrary (MVID: 863f5810-e772-4a82-ab83-eb8530dc5d8c; entries: 2; duplicates: 0) + map: + 33554434 (02000002) 157 + 33554435 (02000003) 71 + no duplicates + +Module 0001: Mono.Android (MVID: ec958fab-bf1d-48b4-b93b-ea43dd50f503; entries: 155; duplicates: 25) + map: + 33554508 (0200004C) 147 + 33554510 (0200004E) 149 +``` + +### Processed output + +Dumps the typemap after translating the raw information above into +human-readable fully qualified type names. + +#### Java to Managed +``` +Java-Type-Name Managed-Type-Name Is-Generic-Type? MVID Token-ID +android/app/Activity Android.App.Activity, Mono.Android ec958fab-bf1d-48b4-b93b-ea43dd50f503 33554623 (0x020000BF) +android/app/Application Android.App.Application, Mono.Android ec958fab-bf1d-48b4-b93b-ea43dd50f503 33554624 (0x020000C0) +android/app/backup/BackupAgent Android.App.Backup.BackupAgent, Mono.Android ec958fab-bf1d-48b4-b93b-ea43dd50f503 33554628 (0x020000C4) +mono/android/runtime/JavaArray [name unknown], Mono.Android generic, ignored ec958fab-bf1d-48b4-b93b-ea43dd50f503 0 (0x00000000) +``` + +The last entry above is for a generic Managed type, which typemap +marks with an invalid token ID of `0` and, thus, its name cannot be +looked up and in its place `[name unknown]` is printed. The entry +still serves a purpose by showing the associated Java type, the +assembly and module UUID (MVID). + +#### Managed to Java + +``` +Managed-Type-Name Java-Type-Name Is-Generic-Type? Is-Duplicate-Type-Entry? MVID Token-ID +Mono.Samples.Hello.LibraryActivity, HelloLibrary mono/samples/hello/LibraryActivity 863f5810-e772-4a82-ab83-eb8530dc5d8c 33554434 (0x02000002) +Mono.Samples.Hello.MyBackupAgent, HelloLibrary crc64fb76f5491eb54ddd/MyBackupAgent 863f5810-e772-4a82-ab83-eb8530dc5d8c 33554435 (0x02000003) +HelloWorld.MainActivity, HelloWorld example/MainActivity 8f7ca9c7-fa38-4784-b1de-d110106a1734 33554434 (0x02000002) +Android.App.Activity, Mono.Android android/app/Activity ec958fab-bf1d-48b4-b93b-ea43dd50f503 33554623 (0x020000BF) +``` + +# Command line options + +``` +Usage: tmt [OPTIONS] [FILTER_REGEX] + +OPTIONS are: + + -j, --only-java Process only the java-to-managed map + -m, --only-managed Process only the managed-to-java map + -s, --short-report Omit some map details from the report (e.g. MVID + and TokenID from managed-to-java map) + -o, --output-directory=DIR Write the report files in the DIR directory + instead of the current one + -a, --arch=ARCH_LIST Limit reporting only to the specified + architectures. ARCH_LIST is a comma-separated + list of architectures (one of, case-insensitive: + ARM, ARM64, X86, X86_64) + -1, --only-first Process only the first shared library from the APK/ + AAB archive or a project directory. Architecture + filter is ignored in this case. This is the + default action when regex filtering is used, + otherwise it defaults to `false` + -g, --generate-files Generate report files. If regex filtering is used, + this setting defaults to `false`, it is `true` + otherwise. + + -v, --verbose Show debug messages +``` + +## Sample invocations + +### Filter by type name + +In this mode no files are produced, the matched entries are printed to +the console + +```shell +$ bin/Debug/bin/tmt samples/HelloWorld '.*Java.Lang.Object' +samples/HelloWorld/obj/Release/app_shared_libraries/arm64-v8a/libxamarin-app.so: + File Type: Xamarin App Release DSO + Format version: 1 + Map kind: Release + Map architecture: ARM64 + Managed to Java entries: 183 + Java to Managed entries: 158 (without duplicates) + + Matching entries (Java to Managed): + java/lang/Object -> Java.Lang.Object, Mono.Android; MVID: ec958fab-bf1d-48b4-b93b-ea43dd50f503; Token ID: 33554768 (0x02000150) + + Matching entries (Managed to Java): + Java.Lang.Object; MVID: ec958fab-bf1d-48b4-b93b-ea43dd50f503; Token ID: 33554768 (0x02000150) -> java/lang/Object +``` + +### Dump all typemaps in a project directory +```shell +# +# Alternative invocations with the same result: +# +# bin/Debug/bin/tmt/tmt samples/HelloWorld/obj +# bin/Debug/bin/tmt/tmt samples/HelloWorld/obj/Debug +# +$ bin/Debug/bin/tmt samples/HelloWorld/ +samples/HelloWorld/obj/Debug/app_shared_libraries/arm64-v8a/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: ARM64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-ARM64-java.txt + Managed to Java output: typemap-v1-Debug-ARM64-managed.txt + +samples/HelloWorld/obj/Debug/app_shared_libraries/armeabi-v7a/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: ARM + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-ARM-java.txt + Managed to Java output: typemap-v1-Debug-ARM-managed.txt + +samples/HelloWorld/obj/Debug/app_shared_libraries/x86/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: X86 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-X86-java.txt + Managed to Java output: typemap-v1-Debug-X86-managed.txt + +samples/HelloWorld/obj/Debug/app_shared_libraries/x86_64/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: X86_64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-X86_64-java.txt + Managed to Java output: typemap-v1-Debug-X86_64-managed.txt +``` + +### Dump all type maps from an apk + +```shell +$ bin/Debug/bin/tmt samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk +samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk!lib/arm64-v8a/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: ARM64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-ARM64-java.txt + Managed to Java output: typemap-v1-Debug-ARM64-managed.txt + +samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk!lib/armeabi-v7a/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: ARM + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-ARM-java.txt + Managed to Java output: typemap-v1-Debug-ARM-managed.txt + +samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk!lib/x86/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: X86 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-X86-java.txt + Managed to Java output: typemap-v1-Debug-X86-managed.txt + +samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk!lib/x86_64/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: X86_64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-X86_64-java.txt + Managed to Java output: typemap-v1-Debug-X86_64-managed.txt +``` + +### Dump only x86_64 typemap from an apk + +```shell +$ bin/Debug/bin/tmt -a x86_64 samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk +samples/HelloWorld/bin/Debug/com.xamarin.android.helloworld-Signed.apk!lib/x86_64/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: X86_64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-X86_64-java.txt + Managed to Java output: typemap-v1-Debug-X86_64-managed.txt +``` + +### Dump typemap from a single shared library + +```shell +$ bin/Debug/bin/tmt samples/HelloWorld/obj/Debug/app_shared_libraries/arm64-v8a/libxamarin-app.so +samples/HelloWorld/obj/Debug/app_shared_libraries/arm64-v8a/libxamarin-app.so: + Loading Java to Managed map: 6062 entries, please wait... + Loading Managed to Java map: 6062 entries, please wait... + File Type: Xamarin App Debug DSO + Format version: 1 + Map kind: Debug + Map architecture: ARM64 + Managed to Java entries: 6062 + Java to Managed entries: 5376 (without duplicates) + Java to Managed output: typemap-v1-Debug-ARM64-java.txt + Managed to Java output: typemap-v1-Debug-ARM64-managed.txt +``` diff --git a/tools/tmt/Report.cs b/tools/tmt/Report.cs new file mode 100644 index 00000000000..85cbdc34da0 --- /dev/null +++ b/tools/tmt/Report.cs @@ -0,0 +1,351 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace tmt +{ + class Report + { + const string FileFieldSeparator = "\t"; + const string ManagedTypeColumnHeader = "Managed-Type-Name"; + const string JavaTypeColumnHeader = "Java-Type-Name"; + const string DuplicateColumnHeader = "Is-Duplicate-Type-Entry?"; + const string GenericColumnHeader = "Is-Generic-Type?"; + const string MVIDColumnHeader = "MVID"; + const string TokenIDColumnHeader = "Token-ID"; + + string outputDirectory; + Regex? filterRegex; + bool full; + bool onlyJava; + bool onlyManaged; + bool generateFiles; + + public Report (string outputDirectory, string filterRegex, bool full, bool onlyJava, bool onlyManaged, bool generateFiles) + { + this.outputDirectory = outputDirectory; + this.full = full; + this.onlyJava = onlyJava; + this.onlyManaged = onlyManaged; + this.generateFiles = generateFiles; + + if (filterRegex.Length > 0) { + this.filterRegex = new Regex (filterRegex, RegexOptions.Compiled); + } + } + + public void Generate (ITypemap typemap) + { + string baseOutputFile = generateFiles ? Utilities.GetOutputFileBaseName (outputDirectory, typemap.FormatVersion, typemap.Map.Kind, typemap.Map.Architecture) : String.Empty; + Action fileGenerator; + Action consoleGenerator; + bool filtering = filterRegex != null; + + if (!onlyManaged) { + typemap.Map.JavaToManaged.Sort ((MapEntry left, MapEntry right) => left.JavaType.Name.CompareTo (right.JavaType.Name)); + if (typemap.Map.Kind == MapKind.Release) { + fileGenerator = FileGenerateJavaToManagedRelease; + consoleGenerator = ConsoleGenerateJavaToManagedRelease; + } else { + fileGenerator = FileGenerateJavaToManagedDebug; + consoleGenerator = ConsoleGenerateJavaToManagedDebug; + } + + Generate ( + filtering ? "Java to Managed" : "Java to Managed output file", + Utilities.GetJavaOutputFileName (baseOutputFile, "txt"), + typemap.Map.JavaToManaged, + full, + fileGenerator, + consoleGenerator + ); + } + + if (!onlyJava) { + typemap.Map.ManagedToJava.Sort ((MapEntry left, MapEntry right) => { + int result = String.Compare (left.ManagedType.AssemblyName, right.ManagedType.AssemblyName, StringComparison.OrdinalIgnoreCase); + if (result != 0) + return result; + + return left.ManagedType.TypeName.CompareTo (right.ManagedType.TypeName); + }); + + if (typemap.Map.Kind == MapKind.Release) { + fileGenerator = FileGenerateManagedToJavaRelease; + consoleGenerator = ConsoleGenerateManagedToJavaRelease; + } else { + fileGenerator = FileGenerateManagedToJavaDebug; + consoleGenerator = ConsoleGenerateManagedToJavaDebug; + } + + Generate ( + filtering ? "Managed to Java" : "Managed to Java output file", + Utilities.GetManagedOutputFileName (baseOutputFile, "txt"), + typemap.Map.ManagedToJava, + full, + fileGenerator, + consoleGenerator + ); + } + } + + void Generate (string name, string outputFile, List typemap, bool full, Action fileGenerator, Action consoleGenerator) + { + if (generateFiles) { + Log.Info ($" {name}: {outputFile}"); + Utilities.CreateFileDirectory (outputFile); + } + + StreamWriter? sw = null; + if (generateFiles) { + sw = new StreamWriter (outputFile, false, new UTF8Encoding (false)); + } + + bool firstMatch = true; + bool first = true; + try { + foreach (MapEntry entry in typemap) { + if (generateFiles) { + fileGenerator (sw!, entry, full, first); + if (first) { + first = false; + } + } + + if (filterRegex == null || !EntryMatches (entry, filterRegex)) { + continue; + } + + if (firstMatch) { + Log.Info (); + Log.Info ($" Matching entries ({name}):"); + firstMatch = false; + } + + consoleGenerator (entry, full); + } + } finally { + if (sw != null) { + sw.Flush (); + sw.Close (); + sw.Dispose (); + } + } + } + + string GetTokenID (MapEntry entry) + { + return $"{entry.ManagedType.TokenID} (0x{entry.ManagedType.TokenID:X08})"; + } + + string GetManagedTypeNameRelease (MapEntry entry) + { + return $"{entry.ManagedType.TypeName}, {entry.ManagedType.AssemblyName}"; + } + + string GetManagedTypeNameDebug (MapEntry entry) + { + return entry.ManagedType.TypeName; + } + + const string IgnoredGeneric = "generic, ignored"; + const string Duplicate = "duplicate entry"; + + void FileGenerateJavaToManagedDebug (StreamWriter sw, MapEntry entry, bool full, bool firstEntry) + { + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{JavaTypeColumnHeader}{sep}{ManagedTypeColumnHeader}{sep}{DuplicateColumnHeader}"); + } + + WriteLineToFile (sw, + entry.JavaType.Name, + GetManagedTypeNameDebug (entry), + entry.ManagedType.IsDuplicate ? Duplicate : String.Empty + ); + } + + void ConsoleGenerateJavaToManagedDebug (MapEntry entry, bool full) + { + Log.Info ($" {entry.JavaType.Name} -> {entry.ManagedType.TypeName}"); + } + + void FileGenerateJavaToManagedRelease (StreamWriter sw, MapEntry entry, bool full, bool firstEntry) + { + string managedTypeName = GetManagedTypeNameRelease (entry); + string generic = entry.ManagedType.IsGeneric ? IgnoredGeneric : String.Empty; + + if (!full) { + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{JavaTypeColumnHeader}{sep}{ManagedTypeColumnHeader}{sep}{GenericColumnHeader}"); + } + + WriteLineToFile ( + sw, + entry.JavaType.Name, + managedTypeName, + generic + ); + return; + } + + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{JavaTypeColumnHeader}{sep}{ManagedTypeColumnHeader}{sep}{GenericColumnHeader}{sep}{MVIDColumnHeader}{sep}{TokenIDColumnHeader}"); + } + WriteLineToFile ( + sw, + entry.JavaType.Name, + managedTypeName, + generic, + entry.ManagedType.MVID.ToString (), + TokenIdToString (entry) + ); + } + + void ConsoleGenerateJavaToManagedRelease (MapEntry entry, bool full) + { + string managedTypeName = GetManagedTypeNameRelease (entry); + if (!full) { + string generic; + + if (entry.ManagedType.IsGeneric) { + generic = $" ({IgnoredGeneric})"; + } else { + generic = String.Empty; + } + + Log.Info ($" {entry.JavaType.Name} -> {managedTypeName}{generic}"); + return; + } + + Log.Info ($" {entry.JavaType.Name} -> {managedTypeName}; MVID: {entry.ManagedType.MVID}; Token ID: {TokenIdToString (entry)}"); + } + + string TokenIdToString (MapEntry entry) + { + if (entry.ManagedType.IsGeneric) { + return "0 (0x00000000)"; + } else { + return GetTokenID (entry); + } + } + + void FileGenerateManagedToJavaDebug (StreamWriter sw, MapEntry entry, bool full, bool firstEntry) + { + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{ManagedTypeColumnHeader}{sep}{JavaTypeColumnHeader}{sep}{DuplicateColumnHeader}"); + } + + WriteLineToFile (sw, + GetManagedTypeNameDebug (entry), + entry.JavaType.Name, + entry.ManagedType.IsDuplicate ? Duplicate : String.Empty + ); + } + + void ConsoleGenerateManagedToJavaDebug (MapEntry entry, bool full) + { + Log.Info ($" {GetManagedTypeNameDebug (entry)} -> {entry.JavaType.Name}{GetAdditionalInfo (entry)}"); + } + + void FileGenerateManagedToJavaRelease (StreamWriter sw, MapEntry entry, bool full, bool firstEntry) + { + string managedTypeName = GetManagedTypeNameRelease (entry); + string duplicate = entry.ManagedType.IsDuplicate ? Duplicate : " "; + string generic = entry.ManagedType.IsGeneric ? IgnoredGeneric : " "; + + if (full) { + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{ManagedTypeColumnHeader}{sep}{JavaTypeColumnHeader}{sep}{GenericColumnHeader}{sep}{DuplicateColumnHeader}{sep}{MVIDColumnHeader}{sep}{TokenIDColumnHeader}"); + } + WriteLineToFile ( + sw, + managedTypeName, + entry.JavaType.Name, + generic, + duplicate, + entry.ManagedType.MVID.ToString (), + GetTokenID (entry) + ); + } else { + if (firstEntry) { + string sep = FileFieldSeparator; + sw.WriteLine ($"{ManagedTypeColumnHeader}{sep}{JavaTypeColumnHeader}{sep}{GenericColumnHeader}{sep}{DuplicateColumnHeader}"); + } + WriteLineToFile ( + sw, + managedTypeName, + entry.JavaType.Name, + generic, + duplicate + ); + } + } + + void ConsoleGenerateManagedToJavaRelease (MapEntry entry, bool full) + { + if (!full) { + Log.Info ($" {GetManagedTypeNameDebug (entry)} -> {entry.JavaType.Name}{GetAdditionalInfo (entry)}"); + } else { + Log.Info ($" {GetManagedTypeNameDebug (entry)}; MVID: {entry.ManagedType.MVID}; Token ID: {TokenIdToString (entry)} -> {entry.JavaType.Name}{GetAdditionalInfo (entry)}"); + } + } + + string GetAdditionalInfo (MapEntry entry) + { + var status = new List (); + if (entry.ManagedType.IsGeneric) { + status.Add (IgnoredGeneric); + } + + if (entry.ManagedType.IsDuplicate) { + status.Add (Duplicate); + } + + if (status.Count > 0) { + return " (" + String.Join ("; ", status) + ")"; + } + + return String.Empty; + } + + bool EntryMatches (MapEntry entry, Regex regex) + { + Match match = regex.Match (entry.JavaType.Name); + if (match.Success) { + return true; + } + + string managedName; + if (entry.ManagedType.AssemblyName.Length > 0) { + managedName = $"{entry.ManagedType.TypeName}, {entry.ManagedType.AssemblyName}"; + } else { + managedName = entry.ManagedType.TypeName; + } + match = regex.Match (managedName); + + return match.Success; + } + + void WriteLineToFile (StreamWriter sw, params string[] fields) + { + if (fields.Length == 0) { + sw.WriteLine (); + return; + } + + for (int i = 0; i < fields.Length; i++) { + if (i > 0) + sw.Write (FileFieldSeparator); + sw.Write (fields [i]); + } + sw.WriteLine (); + } + } +} diff --git a/tools/tmt/Utilities.cs b/tools/tmt/Utilities.cs new file mode 100644 index 00000000000..e892defa451 --- /dev/null +++ b/tools/tmt/Utilities.cs @@ -0,0 +1,39 @@ +using System; +using System.Buffers; +using System.IO; + +namespace tmt +{ + static class Utilities + { + public static readonly ArrayPool BytePool = ArrayPool.Shared; + + public static string GetOutputFileBaseName (string outputDirectory, string formatVersion, MapKind kind, MapArchitecture architecture) + { + string ret = $"typemap-v{formatVersion}-{kind}-{architecture}"; + if (outputDirectory.Length == 0) { + return ret; + } + + return Path.Combine (outputDirectory, ret); + } + + public static string GetManagedOutputFileName (string baseFileName, string extension) + { + return $"{baseFileName}-managed.{extension}"; + } + + public static string GetJavaOutputFileName (string baseFileName, string extension) + { + return $"{baseFileName}-java.{extension}"; + } + + public static void CreateFileDirectory (string filePath) + { + string fileDir = Path.GetDirectoryName (filePath) ?? String.Empty; + if (fileDir.Length > 0) { + Directory.CreateDirectory (fileDir); + } + } + } +} diff --git a/tools/tmt/XamarinAppDSO.cs b/tools/tmt/XamarinAppDSO.cs new file mode 100644 index 00000000000..61e5a794767 --- /dev/null +++ b/tools/tmt/XamarinAppDSO.cs @@ -0,0 +1,150 @@ +using System; +using System.IO; + +namespace tmt +{ + abstract class XamarinAppDSO : ITypemap + { + // Corresponds to the `FORMAT_TAG` constant in src/monodroid/xamarin-app.hh + protected const ulong FormatTag_V1 = 0x015E6972616D58; + + protected const string FormatTag = "format_tag"; + + AnELF? elf; + + protected AnELF ELF => elf ?? throw new InvalidOperationException ("ELF image not loaded"); + protected bool Is64Bit => ELF.Is64Bit; + protected ManagedTypeResolver ManagedResolver { get; } + + public MapArchitecture MapArchitecture => ELF.MapArchitecture; + public string FullPath { get; } = String.Empty; + public abstract string Description { get; } + public abstract string FormatVersion { get; } + public abstract Map Map { get; } + + protected XamarinAppDSO (ManagedTypeResolver managedResolver, string fullPath) + { + ManagedResolver = managedResolver; + FullPath = fullPath; + } + + protected XamarinAppDSO (ManagedTypeResolver managedResolver, AnELF elf) + : this (managedResolver, Path.GetFullPath (elf.FilePath)) + { + this.elf = elf; + } + + public bool CanLoad (Stream stream, string filePath) + { + stream.Seek (0, SeekOrigin.Begin); + if (!AnELF.TryLoad (stream, filePath, out AnELF? elf) || elf == null) { + Log.Debug ($"AnELF.TryLoad failed (elf == {elf})"); + return false; + } + + if (!CanLoad (elf)) { + Log.Debug ($"Cannot load {elf} in {this}"); + return false; + } + + this.elf = elf; + return true; + } + + public abstract bool Load (string outputDirectory, bool generateFiles); + public abstract bool CanLoad (AnELF elf); + + protected bool HasSymbol (AnELF elf, string symbolName) + { + bool ret = elf.HasSymbol (symbolName); + if (!ret) + Log.Debug ($"{elf.FilePath} is missing symbol '{symbolName}'"); + return ret; + } + + protected uint ReadUInt32 (byte[] data, ref ulong offset, bool packed = false) + { + const ulong DataSize = 4; + + if ((ulong)data.Length < (offset + DataSize)) + throw new InvalidOperationException ("Not enough data to read a 32-bit integer"); + + uint ret = BitConverter.ToUInt32 (data, (int)offset); + offset += packed ? DataSize : GetPaddedSize (offset); + + return ret; + } + + protected ulong ReadUInt64 (byte[] data, ref ulong offset, bool packed = false) + { + const ulong DataSize = 8; + + if ((ulong)data.Length < (offset + DataSize)) + throw new InvalidOperationException ("Not enough data to read a 64-bit integer"); + + ulong ret = BitConverter.ToUInt64 (data, (int)offset); + offset += packed ? DataSize : GetPaddedSize (offset); + + return ret; + } + + protected ulong ReadPointer (byte[] data, ref ulong offset, bool packed = false) + { + ulong ret; + + if (Is64Bit) { + ret = ReadUInt64 (data, ref offset, packed); + } else { + ret = (ulong)ReadUInt32 (data, ref offset, packed); + } + + return ret; + } + + protected ulong GetPaddedSize (ulong sizeSoFar) + { + ulong typeSize = GetTypeSize (); + + ulong modulo; + if (Is64Bit) { + modulo = typeSize < 8 ? 4u : 8u; + } else { + modulo = 4u; + } + + ulong alignment = sizeSoFar % modulo; + if (alignment == 0) + return typeSize; + + return typeSize + (modulo - alignment); + } + + ulong GetTypeSize () + { + Type type = typeof(S); + + if (type == typeof(string)) { + // We treat `string` as a generic pointer + return Is64Bit ? 8u : 4u; + } + + if (type == typeof(byte)) { + return 1u; + } + + if (type == typeof(bool)) { + return 1u; + } + + if (type == typeof(Int32) || type == typeof(UInt32)) { + return 4u; + } + + if (type == typeof(Int64) || type == typeof(UInt64)) { + return 8u; + } + + throw new InvalidOperationException ($"Unable to map managed type {type} to native assembler type"); + } + } +} diff --git a/tools/tmt/XamarinAppDebugDSO.cs b/tools/tmt/XamarinAppDebugDSO.cs new file mode 100644 index 00000000000..c653cd0881d --- /dev/null +++ b/tools/tmt/XamarinAppDebugDSO.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; + +namespace tmt +{ + class XamarinAppDebugDSO : XamarinAppDSO + { + XamarinAppDebugDSO_Version? xapp; + + public override string FormatVersion => xapp?.FormatVersion ?? "0"; + public override string Description => xapp?.Description ?? "Xamarin App Debug DSO Forwarder"; + public override Map Map => XAPP.Map; + + XamarinAppDebugDSO_Version XAPP => xapp ?? throw new InvalidOperationException ("Format implementation not found"); + + public XamarinAppDebugDSO (ManagedTypeResolver managedResolver, string fullPath) + : base (managedResolver, fullPath) + {} + + public override bool CanLoad (AnELF elf) + { + xapp = null; + ulong format_tag = 0; + if (elf.HasSymbol (FormatTag)) + format_tag = elf.GetUInt64 (FormatTag); + + XamarinAppDebugDSO_Version? reader = null; + switch (format_tag) { + case 0: + case FormatTag_V1: + format_tag = 1; + reader = new XamarinAppDebugDSO_V1 (ManagedResolver, elf); + break; + + default: + Log.Error ($"{elf.FilePath} format ({format_tag}) is not supported by this version of TMT"); + return false; + } + + if (reader == null || !reader.CanLoad (elf)) { + return false; + } + + xapp = reader; + return true; + } + + public override bool Load (string outputDirectory, bool generateFiles) + { + return XAPP.Load (outputDirectory, generateFiles); + } + } + + abstract class XamarinAppDebugDSO_Version : XamarinAppDSO + { + public override string Description => "Xamarin App Debug DSO"; + + protected XamarinAppDebugDSO_Version (ManagedTypeResolver managedResolver, AnELF elf) + : base (managedResolver, elf) + {} + + protected Map MakeMap (List managedToJava, List javaToManaged) + { + return new Map (MapKind.Debug, ELF.MapArchitecture, managedToJava, javaToManaged, FormatVersion); + } + + public override bool Load (string outputDirectory, bool generateFiles) + { + if (!LoadMaps ()) { + return false; + } + + return Convert (); + } + + protected abstract bool LoadMaps (); + protected abstract bool Convert (); + } + + class XamarinAppDebugDSO_V1 : XamarinAppDebugDSO_Version + { + sealed class MappedType + { + public string TargetType; + public ulong DuplicateCount = 0; + + public MappedType (string targetType) + { + TargetType = targetType; + } + } + + const string TypeMapSymbolName = "type_map"; + + Map? map; + SortedDictionary javaToManaged = new SortedDictionary (StringComparer.Ordinal); + SortedDictionary managedToJava = new SortedDictionary (StringComparer.Ordinal); + + public override string FormatVersion => "1"; + public override Map Map => map ?? throw new InvalidOperationException ("Data hasn't been loaded yet"); + + public XamarinAppDebugDSO_V1 (ManagedTypeResolver managedResolver, AnELF elf) + : base (managedResolver, elf) + {} + + + public override bool CanLoad (AnELF elf) + { + return HasSymbol (elf, TypeMapSymbolName); + } + + protected override bool Convert () + { + try { + DoConvert (); + } catch (Exception ex) { + Log.ExceptionError ($"{Description}: failed to convert loaded maps to common format", ex); + return false; + } + return true; + } + + void DoConvert () + { + var managed = new List (); + foreach (var kvp in managedToJava) { + string managedName = kvp.Key; + MappedType javaType = kvp.Value; + + managed.Add ( + new MapEntry ( + new MapManagedType (managedName) { IsDuplicate = javaType.DuplicateCount > 0 }, + new MapJavaType (javaType.TargetType) + ) + ); + } + + var java = new List (); + foreach (var kvp in javaToManaged) { + string javaName = kvp.Key; + MappedType managedType = kvp.Value; + + java.Add ( + new MapEntry ( + new MapManagedType (managedType.TargetType) { IsDuplicate = managedType.DuplicateCount > 0 }, + new MapJavaType (javaName) + ) + ); + } + + map = MakeMap (managed, java); + } + + protected override bool LoadMaps () + { + try { + return DoLoadMaps (); + } catch (Exception ex) { + Log.ExceptionError ($"{Description}: failed to load maps", ex); + return false; + } + } + + bool DoLoadMaps () + { + // MUST be kept in sync with: src/monodroid/jni/xamarin-app.hh (struct TypeMap) + ulong size = 0; + size += GetPaddedSize (size); // entry_count + size += GetPaddedSize (size); // assembly_name (pointer) + size += GetPaddedSize (size); // data (pointer) + size += GetPaddedSize (size); // java_to_managed (pointer) + size += GetPaddedSize (size); // managed_to_java (pointer) + + string filePath = ELF.FilePath; + byte[] mapData = ELF.GetData (TypeMapSymbolName); + if (mapData.Length == 0) { + Log.Error ($"{filePath} doesn't have a valid '{TypeMapSymbolName}' symbol"); + return false; + } + + if ((ulong)mapData.Length != size) { + Log.Error ($"Symbol '{TypeMapSymbolName}' in {filePath} has invalid size. Expected {size}, got {mapData.Length}"); + return false; + } + + ulong offset = 0; + uint entry_count = ReadUInt32 (mapData, ref offset); + + ReadPointer (mapData, ref offset); // assembly_name, unused in Debug mode + ReadPointer (mapData, ref offset); // data, unused in Debug mode + + ulong pointer = ReadPointer (mapData, ref offset); // java_to_managed + LoadMap ("Java to Managed", pointer, entry_count, AddJavaToManaged); + + pointer = ReadPointer (mapData, ref offset); // managed_to_java + LoadMap ("Managed to Java", pointer, entry_count, AddManagedToJava); + + return true; + } + + void AddManagedToJava (string mapFrom, string mapTo) + { + if (managedToJava.TryGetValue (mapFrom, out MappedType? entry)) { + entry.DuplicateCount++; + return; + } + + managedToJava.Add (mapFrom, new MappedType (mapTo)); + } + + void AddJavaToManaged (string mapFrom, string mapTo) + { + if (javaToManaged.TryGetValue (mapFrom, out MappedType? entry)) { + entry.DuplicateCount++; + return; + } + + javaToManaged.Add (mapFrom, new MappedType (mapTo)); + } + + void LoadMap (string name, ulong pointer, uint entry_count, Action addToMap) + { + string entries = entry_count == 1 ? "entry" : "entries"; + Log.Info ($" Loading {name} map: {entry_count} {entries}, please wait..."); + + ulong size = 0; + size += GetPaddedSize (size); // from + size += GetPaddedSize (size); // to + + ulong mapSize = entry_count * size; + byte[] data = ELF.GetData (pointer, mapSize); + + ulong offset = 0; + string mapFrom; + string mapTo; + for (uint i = 0; i < entry_count; i++) { + pointer = ReadPointer (data, ref offset); + if (pointer != 0) { + mapFrom = ELF.GetASCIIZ (pointer); + } else { + mapFrom = $"#{i}"; + } + + pointer = ReadPointer (data, ref offset); + if (pointer != 0) { + mapTo = ELF.GetASCIIZ (pointer); + } else { + mapTo = $"#{i}"; + } + + addToMap (mapFrom, mapTo); + } + } + } +} diff --git a/tools/tmt/XamarinAppReleaseDSO.cs b/tools/tmt/XamarinAppReleaseDSO.cs new file mode 100644 index 00000000000..535c0adf120 --- /dev/null +++ b/tools/tmt/XamarinAppReleaseDSO.cs @@ -0,0 +1,480 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace tmt +{ + class XamarinAppReleaseDSO : XamarinAppDSO + { + XamarinAppReleaseDSO_Version? xapp; + + public override string FormatVersion => xapp?.FormatVersion ?? "0"; + public override string Description => xapp?.Description ?? "Xamarin App Release DSO Forwarder"; + public override Map Map => XAPP.Map; + + XamarinAppReleaseDSO_Version XAPP => xapp ?? throw new InvalidOperationException ("Format implementation not found"); + + public XamarinAppReleaseDSO (ManagedTypeResolver managedResolver, string fullPath) + : base (managedResolver, fullPath) + {} + + protected XamarinAppReleaseDSO (ManagedTypeResolver managedResolver, AnELF elf) + : base (managedResolver, elf) + {} + + public override bool CanLoad (AnELF elf) + { + Log.Debug ($"Checking if {elf.FilePath} is a Release DSO"); + + xapp = null; + ulong format_tag = 0; + if (elf.HasSymbol (FormatTag)) + format_tag = elf.GetUInt64 (FormatTag); + + XamarinAppReleaseDSO_Version? reader = null; + switch (format_tag) { + case 0: + case FormatTag_V1: + format_tag = 1; + reader = new XamarinAppReleaseDSO_V1 (ManagedResolver, elf); + break; + + default: + Log.Error ($"{elf.FilePath} format ({format_tag}) is not supported by this version of TMT"); + return false; + } + + if (reader == null || !reader.CanLoad (elf)) { + return false; + } + + xapp = reader; + return true; + } + + public override bool Load (string outputDirectory, bool generateFiles) + { + return XAPP.Load (outputDirectory, generateFiles); + } + } + + abstract class XamarinAppReleaseDSO_Version : XamarinAppDSO + { + public override string Description => "Xamarin App Release DSO"; + + protected XamarinAppReleaseDSO_Version (ManagedTypeResolver managedResolver, AnELF elf) + : base (managedResolver, elf) + {} + + protected Map MakeMap (List managedToJava, List javaToManaged) + { + return new Map (MapKind.Release, ELF.MapArchitecture, managedToJava, javaToManaged, FormatVersion); + } + + public override bool Load (string outputDirectory, bool generateFiles) + { + if (!LoadMaps ()) { + return false; + } + + if (generateFiles) { + string rawOutputFile = Utilities.GetOutputFileBaseName (outputDirectory, FormatVersion, MapKind.Release, ELF.MapArchitecture); + SaveRaw (rawOutputFile, "raw"); + } + + return Convert (); + } + + protected abstract bool LoadMaps (); + protected abstract bool Convert (); + protected abstract void SaveRaw (string baseOutputFilePath, string extension); + } + + class XamarinAppReleaseDSO_V1 : XamarinAppReleaseDSO_Version + { + // Field names correspond to: src/monodroid/jni/xamarin-app.hh (struct TypeMapModuleEntry) + sealed class TypeMapModuleEntry + { + public uint type_token_id; + public uint java_map_index; + } + + // Field names correspond to: src/monodroid/jni/xamarin-app.hh (struct TypeMapModule) + sealed class TypeMapModule + { + public Guid module_uuid; + public uint entry_count; + public uint duplicate_count; + public List? map; + public List? duplicate_map; + public string assembly_name = String.Empty; + + // These three aren't used, listed for completeness + public readonly object? image = null; + public readonly uint java_name_width = 0; + public readonly byte[]? java_map = null; + } + + // Field names correspond to: src/monodroid/jni/xamarin-app.hh (struct TypeMapJava) + sealed class TypeMapJava + { + public uint module_index; + public uint type_token_id; + public string java_name = String.Empty; + } + + const string MapModulesSymbolName = "map_modules"; + const string ModuleCountSymbolName = "map_module_count"; + const string JavaTypeCountSymbolName = "java_type_count"; + const string JavaNameWidthSymbolName = "java_name_width"; + const string MapJavaSymbolName = "map_java"; + + Map? map; + List? modules; + List? javaTypes; + + public override string FormatVersion => "1"; + public override Map Map => map ?? throw new InvalidOperationException ("Data hasn't been loaded yet"); + + public XamarinAppReleaseDSO_V1 (ManagedTypeResolver managedResolver, AnELF elf) + : base (managedResolver, elf) + {} + + public override bool CanLoad (AnELF elf) + { + return + HasSymbol (elf, MapModulesSymbolName) && + HasSymbol (elf, ModuleCountSymbolName) && + HasSymbol (elf, JavaTypeCountSymbolName) && + HasSymbol (elf, JavaNameWidthSymbolName) && + HasSymbol (elf, MapJavaSymbolName); + } + + protected override bool LoadMaps () + { + try { + string filePath = ELF.FilePath; + modules = LoadMapModules (filePath); + javaTypes = LoadJavaTypes (filePath); + return true; + } catch (Exception ex) { + Log.ExceptionError ($"{Description}: failed to load maps", ex); + return false; + } + } + + protected override void SaveRaw (string baseOutputFilePath, string extension) + { + const string indent = "\t"; + + if (modules == null || javaTypes == null) { + Log.Warning ($"{Description}: cannot save raw report, no data"); + return; + } + + string outputFilePath = Utilities.GetManagedOutputFileName (baseOutputFilePath, extension); + Utilities.CreateFileDirectory (outputFilePath); + using (var sw = new StreamWriter (outputFilePath, false, new UTF8Encoding (false))) { + sw.WriteLine ("TYPE_TOKEN_DECIMAL (TYPE_TOKEN_HEXADECIMAL)\tJAVA_MAP_INDEX"); + uint index = 0; + foreach (TypeMapModule module in modules) { + sw.WriteLine (); + sw.WriteLine ($"Module {index++:D04}: {module.assembly_name} (MVID: {module.module_uuid}; entries: {module.entry_count}; duplicates: {module.duplicate_count})"); + if (module.map == null) { + sw.WriteLine ($"{indent}no map"); + } else { + WriteManagedMap ("map", module.map, sw); + } + + if (module.duplicate_map == null) { + if (module.duplicate_count > 0) + sw.WriteLine ($"{indent}no duplicate map, but there should be {module.duplicate_count} entries"); + else + sw.WriteLine ($"{indent}no duplicates"); + continue; + } + + WriteManagedMap ("duplicate map", module.duplicate_map, sw); + } + sw.Flush (); + } + + outputFilePath = Utilities.GetJavaOutputFileName (baseOutputFilePath, extension); + using (var sw = new StreamWriter (outputFilePath, false, new UTF8Encoding (false))) { + sw.WriteLine ("MANAGED_MODULE_INDEX\tTYPE_TOKEN_DECIMAL (TYPE_TOKEN_HEXADECIMAL)\tJAVA_TYPE_NAME"); + + foreach (TypeMapJava tmj in javaTypes) { + sw.WriteLine ($"{indent}{tmj.module_index}\t{tmj.type_token_id:D08} ({tmj.type_token_id:X08})\t{tmj.java_name}"); + } + sw.Flush (); + } + + void WriteManagedMap (string name, List map, StreamWriter sw) + { + sw.WriteLine ($"{indent}{name}:"); + foreach (TypeMapModuleEntry entry in map) { + sw.WriteLine ($"{indent}{indent}{entry.type_token_id} ({entry.type_token_id:X08})\t{entry.java_map_index}"); + } + } + } + + MapManagedType MakeManagedType (Guid mvid, uint tokenID, string assemblyName, string filePath, bool isGeneric, bool isDuplicate) + { + return new MapManagedType (mvid, tokenID, assemblyName, filePath) { + IsGeneric = isGeneric, + IsDuplicate = isDuplicate, + TypeName = ManagedResolver.Lookup (assemblyName, mvid, tokenID) + }; + } + + protected override bool Convert () + { + try { + DoConvert (); + } catch (Exception ex) { + Log.ExceptionError ($"{Description}: failed to convert loaded maps to common format", ex); + return false; + } + return true; + } + + bool DoConvert () + { + if (modules == null || javaTypes == null) { + Log.Warning ($"{Description}: cannot convert maps, no data"); + return false; + } + + string filePath = ELF.FilePath; + var managedToJava = new List (); + uint index = 0; + + bool somethingFailed = false; + foreach (TypeMapModule m in modules) { + ConvertManagedMap (m, EnsureMap (m), isDuplicate: false); + if (m.duplicate_map != null) { + if (!ConvertManagedMap (m, m.duplicate_map, isDuplicate: true)) { + somethingFailed = true; + } + } + + index++; + } + + if (somethingFailed) { + return false; + } + + index = 0; + var javaToManaged = new List (); + foreach (TypeMapJava tmj in javaTypes) { + (bool success, TypeMapModule? module, bool isGeneric, bool isDuplicate) = FindManagedType (tmj.module_index, tmj.type_token_id); + if (!success) { + somethingFailed = true; + continue; + } + + if (module == null) { + throw new InvalidOperationException ("module must not be null here"); + } + + javaToManaged.Add ( + new MapEntry ( + MakeManagedType (module.module_uuid, tmj.type_token_id, module.assembly_name, filePath, isGeneric, isDuplicate), + new MapJavaType (tmj.java_name, filePath) + ) + ); + index++; + } + + map = MakeMap (managedToJava, javaToManaged); + return true; + + bool ConvertManagedMap (TypeMapModule module, List map, bool isDuplicate) + { + foreach (TypeMapModuleEntry entry in map) { + TypeMapJava java; + + if ((uint)javaTypes.Count <= entry.java_map_index) { + Log.Error ($"Managed type {entry.type_token_id} in module {module.assembly_name} ({module.module_uuid}) has invalid Java map index {entry.java_map_index}"); + return false; + } + java = javaTypes[(int)entry.java_map_index]; + managedToJava.Add ( + new MapEntry ( + MakeManagedType (module.module_uuid, entry.type_token_id, module.assembly_name, filePath, isGeneric: false, isDuplicate: isDuplicate), + new MapJavaType (java.java_name, filePath) + ) + ); + } + return true; + } + + (bool success, TypeMapModule? module, bool isGeneric, bool isDuplicate) FindManagedType (uint moduleIndex, uint tokenID) + { + if (moduleIndex >= (uint)modules.Count) { + Log.Error ($"Invalid module index {moduleIndex} for type token ID {tokenID} at Java map index {index}"); + return (false, null, false, false); + } + + TypeMapModule m = modules[(int)moduleIndex]; + if (tokenID == 0) { + return (true, m, true, false); + } + + foreach (TypeMapModuleEntry entry in EnsureMap (m)) { + if (entry.type_token_id == tokenID) { + return (true, m, false, false); + } + } + + if (m.duplicate_map != null) { + foreach (TypeMapModuleEntry entry in m.duplicate_map) { + if (entry.type_token_id == tokenID) { + return (true, m, false, true); + } + } + } + + Log.Error ($"Module {m.assembly_name} ({m.module_uuid}) at index {moduleIndex} doesn't contain an entry for managed type with token ID {tokenID}"); + return (false, null, false, false); + } + + List EnsureMap (TypeMapModule m) + { + if (m.map == null) + throw new InvalidOperationException ($"Module {m.module_uuid} ({m.assembly_name}) has no map?"); + return m.map; + } + } + + List LoadJavaTypes (string filePath) + { + ulong javaTypeCount = (ulong)ELF.GetUInt32 (JavaTypeCountSymbolName); + ulong javaNameWidth = (ulong)ELF.GetUInt32 (JavaNameWidthSymbolName); + + // MUST be kept in sync with: src/monodroid/jni/xamarin-app.hh (struct TypeMapModule) + ulong size = 0; + size += GetPaddedSize (size); // module_index + size += GetPaddedSize (size); // type_token_id + size += javaNameWidth; + + byte[] data = ELF.GetData (MapJavaSymbolName); + if (data.Length == 0) + throw new InvalidOperationException ($"{filePath} doesn't have a valid '{MapJavaSymbolName}' symbol"); + + ulong calculatedJavaTypeCount = (ulong)data.LongLength / size; + if (calculatedJavaTypeCount != javaTypeCount) + throw new InvalidOperationException ($"{filePath} has invalid '{JavaTypeCountSymbolName}' symbol value ({javaTypeCount}), '{JavaTypeCountSymbolName}' size indicates there are {calculatedJavaTypeCount} managedToJava instead"); + + var ret = new List (); + ulong offset = 0; + for (ulong i = 0; i < javaTypeCount; i++) { + var javaEntry = new TypeMapJava { + module_index = ReadUInt32 (data, ref offset, packed: true), + type_token_id = ReadUInt32 (data, ref offset, packed: true), + }; + + javaEntry.java_name = ELF.GetASCIIZ (data, offset); + offset += javaNameWidth; + ret.Add (javaEntry); + } + + return ret; + } + + List LoadMapModules (string filePath) + { + ulong moduleCount = (ulong)ELF.GetUInt32 (ModuleCountSymbolName); + + // MUST be kept in sync with: src/monodroid/jni/xamarin-app.hh (struct TypeMapModule) + ulong size; + + size = 16; // module_uuid + size += GetPaddedSize (size); // entry_count + size += GetPaddedSize (size); // duplicate_count + size += GetPaddedSize (size); // map (pointer) + size += GetPaddedSize (size); // duplicate_map (pointer) + size += GetPaddedSize (size); // assembly_name (pointer) + size += GetPaddedSize (size); // image (pointer) + size += GetPaddedSize (size); // java_name_width + size += GetPaddedSize (size); // java_map (pointer) + + byte[] moduleData = ELF.GetData (MapModulesSymbolName); + if (moduleData.Length == 0) + throw new InvalidOperationException ($"{filePath} doesn't have a valid '{MapModulesSymbolName}' symbol"); + + ulong calculatedModuleCount = (ulong)moduleData.Length / size; + if (calculatedModuleCount != moduleCount) + throw new InvalidOperationException ($"{filePath} has invalid '{ModuleCountSymbolName}' symbol value ({moduleCount}), '{MapModulesSymbolName}' size indicates there are {calculatedModuleCount} managedToJava instead"); + + var ret = new List (); + ulong offset = 0; + for (ulong i = 0; i < moduleCount; i++) { + Log.Debug ($"Module {i + 1}"); + var module = new TypeMapModule (); + + byte[] mvid = new byte[16]; + Array.Copy (moduleData, (int)offset, mvid, 0, mvid.Length); + module.module_uuid = new Guid (mvid); + offset += (ulong)mvid.Length; + Log.Debug ($" module_uuid == {module.module_uuid}"); + + module.entry_count = ReadUInt32 (moduleData, ref offset); + Log.Debug ($" entry_count == {module.entry_count}"); + + module.duplicate_count = ReadUInt32 (moduleData, ref offset); + Log.Debug ($" duplicate_count == {module.duplicate_count}"); + + // MUST be kept in sync with: src/monodroid/jni/xamarin-app.hh (struct TypeMapModuleEntry) + ulong pointer = ReadPointer (moduleData, ref offset); + size = 0; + size += GetPaddedSize (size); // type_token_id + size += GetPaddedSize (size); // java_map_index + + ulong mapSize = size * module.entry_count; + byte[] data = ELF.GetData (pointer, mapSize); + + module.map = new List (); + ReadMapEntries (module.map, data, module.entry_count); + + // MUST be kept in sync with: src/monodroid/jni/xamarin-app.hh (struct TypeMapModuleEntry) + pointer = ReadPointer (moduleData, ref offset); + if (pointer != 0) { + mapSize = size * module.duplicate_count; + data = ELF.GetData (pointer, mapSize); + module.duplicate_map = new List (); + ReadMapEntries (module.duplicate_map, data, module.duplicate_count); + } + + pointer = ReadPointer (moduleData, ref offset); + module.assembly_name = ELF.GetASCIIZ (pointer); + Log.Debug ($" assembly_name == {module.assembly_name}"); + Log.Debug (""); + + // Read the values to properly adjust the offset taking padding into account + ReadPointer (moduleData, ref offset); + ReadUInt32 (moduleData, ref offset); + ReadPointer (moduleData, ref offset); + + ret.Add (module); + } + + return ret; + + void ReadMapEntries (List map, byte[] inputData, uint entryCount) + { + ulong mapOffset = 0; + for (uint i = 0; i < entryCount; i++) { + var entry = new TypeMapModuleEntry { + type_token_id = ReadUInt32 (inputData, ref mapOffset), + java_map_index = ReadUInt32 (inputData, ref mapOffset) + }; + + map.Add (entry); + } + } + } + } +} diff --git a/tools/tmt/main.cs b/tools/tmt/main.cs new file mode 100644 index 00000000000..e86796ef029 --- /dev/null +++ b/tools/tmt/main.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; + +using Mono.Options; + +namespace tmt +{ + sealed class ParsedOptions + { + bool onlyJava; + bool onlyManaged; + AndroidArch archFilter = AndroidArch.All; + + public bool ShowHelp { get; set; } + public bool Verbose { get; set; } + public bool ShortReport { get; set; } + public bool? LoadOnlyFirst { get; set; } = null; + public bool? GenerateReportFiles { get; set; } = null; + public string OutputDirectory { get; set; } = String.Empty; + + public AndroidArch ArchFilter { + get => archFilter; + set => archFilter = value; + } + + public bool OnlyJava { + get => onlyJava; + set { + onlyJava = value; + if (value) { + onlyManaged = false; + } + } + } + + public bool OnlyManaged { + get => onlyManaged; + set { + onlyManaged = value; + if (value) { + onlyJava = false; + } + } + } + + public void SetArchitectures (string architectures) + { + if (architectures.Length == 0) { + return; + } + + AndroidArch parsedFilter = AndroidArch.None; + string[] values = architectures.Split (new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); + foreach (string arch in values) { + if (String.Compare ("ARM", arch, StringComparison.OrdinalIgnoreCase) == 0) { + parsedFilter |= AndroidArch.ARM; + continue; + } + + if (String.Compare ("ARM64", arch, StringComparison.OrdinalIgnoreCase) == 0) { + parsedFilter |= AndroidArch.ARM64; + continue; + } + + if (String.Compare ("X86", arch, StringComparison.OrdinalIgnoreCase) == 0) { + parsedFilter |= AndroidArch.X86; + continue; + } + + if (String.Compare ("X86_64", arch, StringComparison.OrdinalIgnoreCase) == 0) { + parsedFilter |= AndroidArch.X86_64; + continue; + } + + Log.Warning ($"Unknown architecture '{arch}'"); + } + + if (parsedFilter != AndroidArch.None) { + archFilter = parsedFilter; + } + } + } + + class TypeMapTool + { + public static int Main (string[] args) + { + var parsedOptions = new ParsedOptions (); + + var opts = new OptionSet { + "Usage: tmt [OPTIONS] [FILTER_REGEX]", + "", + "The only required parameter is a path to a location containing the type maps.", + "It is a path to an APK archive, AAB archive, Xamarin.Android application", + "shared library or a Xamarin.Android project directory with compiled application.", + "", + "The optional FILTER_REGEX argument following the path is a regular expression to", + "apply to the mapped types in order to filter the results. Full output files are still created,", + "however the matching entries are also printed to console.", + "", + "OPTIONS are:", + "", + { "j|only-java", "Process only the java-to-managed map", v => parsedOptions.OnlyJava = true }, + { "m|only-managed", "Process only the managed-to-java map", v => parsedOptions.OnlyManaged = true }, + { "s|short-report", "Omit some map details from the report (e.g. MVID and TokenID from managed-to-java map)", v => parsedOptions.ShortReport = true }, + { "o|output-directory=", "Write the report files in the {DIR} directory instead of the current one", v => parsedOptions.OutputDirectory = v?.Trim () ?? String.Empty }, + { "a|arch=", "Limit reporting only to the specified architectures. {ARCH_LIST} is a comma-separated list of architectures (one of, case-insensitive: ARM, ARM64, X86, X86_64)", v => parsedOptions.SetArchitectures (v?.Trim () ?? String.Empty) }, + { "1|only-first", "Process only the first shared library from the APK/AAB archive or a project directory. Architecture filter is ignored in this case. This is the default action when regex filtering is used, otherwise it defaults to `false`", v => parsedOptions.LoadOnlyFirst = true }, + { "g|generate-files", "Generate report files. If regex filtering is used, this setting defaults to `false`, it is `true` otherwise.", v => parsedOptions.GenerateReportFiles = true }, + "", + { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, + { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, + }; + + List rest = opts.Parse (args); + Log.SetVerbose (parsedOptions.Verbose); + if (rest.Count == 0 || parsedOptions.ShowHelp) { + opts.WriteOptionDescriptions (Console.Out); + return 0; + } + + string loadFrom = rest[0]; + string filterRegex = String.Empty; + + if (rest.Count > 1) { + filterRegex = rest [1].Trim (); + } + + if (filterRegex.Length > 0) { + parsedOptions.ArchFilter = AndroidArch.All; + + if (!parsedOptions.LoadOnlyFirst.HasValue) { + parsedOptions.LoadOnlyFirst = true; + } + + if (!parsedOptions.GenerateReportFiles.HasValue) { + parsedOptions.GenerateReportFiles = false; + } + } else { + if (!parsedOptions.LoadOnlyFirst.HasValue) { + parsedOptions.LoadOnlyFirst = false; + } + + if (!parsedOptions.GenerateReportFiles.HasValue) { + parsedOptions.GenerateReportFiles = true; + } + } + + var report = new Report (parsedOptions.OutputDirectory, filterRegex, !parsedOptions.ShortReport, parsedOptions.OnlyJava, parsedOptions.OnlyManaged, parsedOptions.GenerateReportFiles.Value); + var loader = new Loader (parsedOptions.ArchFilter, parsedOptions.LoadOnlyFirst.Value); + List typemaps = loader.TryLoad (loadFrom); + if (typemaps.Count == 0) { + Log.Info ($"No type maps found in '{loadFrom}"); + return 1; + } + + bool somethingFailed = false; + foreach (ITypemap typemap in typemaps) { + Log.Info ($"{typemap.FullPath}:"); + + if (!typemap.Load (parsedOptions.OutputDirectory, parsedOptions.GenerateReportFiles.Value)) { + Log.Error ($" load failed"); + continue; + } + + Log.Info ($" File Type: {typemap.Description}"); + Log.Info ($" Format version: {typemap.FormatVersion}"); + Log.Info ($" Map kind: {typemap.Map.Kind}"); + Log.Info ($" Map architecture: {typemap.Map.Architecture}"); + Log.Info ($" Managed to Java entries: {typemap.Map.ManagedToJava.Count}"); + Log.Info ($" Java to Managed entries: {typemap.Map.JavaToManaged.Count} (without duplicates)"); + + report.Generate (typemap); + Log.Info (); + } + + return somethingFailed ? 1 : 0; + } + } +} diff --git a/tools/tmt/tmt.csproj b/tools/tmt/tmt.csproj new file mode 100644 index 00000000000..19766aa468e --- /dev/null +++ b/tools/tmt/tmt.csproj @@ -0,0 +1,23 @@ + + + + Microsoft Corporation + 2020 Microsoft Corporation + 0.0.1 + netcoreapp3.1 + false + ../../bin/$(Configuration)/bin/typemap-tool + Exe + true + enable + + + + + + + + + + +