From ae2af4015b2150eb971ed43ded51f7a6b07b0069 Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 18 Mar 2021 11:56:35 -0700 Subject: [PATCH 01/14] versioning --- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 59 +++++++++++++++---- .../Bundle/FileEntry.cs | 13 +++- .../Bundle/Manifest.cs | 28 ++++----- .../Bundle/TargetInfo.cs | 18 ++++-- src/native/corehost/bundle/extractor.cpp | 16 ++++- src/native/corehost/bundle/file_entry.cpp | 40 +++++++++++-- src/native/corehost/bundle/file_entry.h | 7 ++- src/native/corehost/bundle/header.cpp | 8 ++- src/native/corehost/bundle/header.h | 12 ++-- src/native/corehost/bundle/info.cpp | 5 +- src/native/corehost/bundle/manifest.cpp | 2 +- 11 files changed, 157 insertions(+), 51 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 0505d2dcefbf34..00809772eab427 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -9,6 +9,7 @@ using System.IO; using System.Reflection.PortableExecutable; using System.Runtime.InteropServices; +using System.IO.Compression; namespace Microsoft.NET.HostModel.Bundle { @@ -18,6 +19,9 @@ namespace Microsoft.NET.HostModel.Bundle /// public class Bundler { + public const uint BundlerMajorVersion = 3; + public const uint BundlerMinorVersion = 0; + private readonly string HostName; private readonly string OutputDir; private readonly string DepsJson; @@ -49,17 +53,52 @@ public Bundler(string hostName, RuntimeConfigJson = appAssemblyName + ".runtimeconfig.json"; RuntimeConfigDevJson = appAssemblyName + ".runtimeconfig.dev.json"; - BundleManifest = new Manifest(Target.BundleVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent)); + BundleManifest = new Manifest(Target.BundleMajorVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent)); Options = Target.DefaultOptions | options; } + private bool ShouldCompress(FileType type) + { + // compression is not supported before bundle vesion 3 + if (Target.BundleMajorVersion < 3) + { + return false; + } + + return false; // type == FileType.Symbols; + } + /// /// Embed 'file' into 'bundle' /// - /// Returns the offset of the start 'file' within 'bundle' - - private long AddToBundle(Stream bundle, Stream file, FileType type) + /// + /// startOffset: offset of the start 'file' within 'bundle' + /// compressedSize: size of the compressed data, if entry was compressed, otherwise 0 + /// + private (long startOffset, long compressedSize) AddToBundle(Stream bundle, Stream file, FileType type) { + long startOffset = bundle.Position; + if (ShouldCompress(type)) + { + long fileLength = file.Length; + file.Position = 0; + + using (GZipStream compressionStream = new GZipStream(bundle, CompressionMode.Compress, leaveOpen: true)) + { + file.CopyTo(compressionStream); + } + + long compressedSize = bundle.Position - startOffset; + if (compressedSize < fileLength * 0.75) + { + return (startOffset, compressedSize); + } + + // compression rate was not good enough + // roll back the bundle and let the uncompressed code path take care of the entry. + bundle.Seek(startOffset, SeekOrigin.Begin); + } + if (type == FileType.Assembly) { long misalignment = (bundle.Position % Target.AssemblyAlignment); @@ -72,10 +111,10 @@ private long AddToBundle(Stream bundle, Stream file, FileType type) } file.Position = 0; - long startOffset = bundle.Position; + startOffset = bundle.Position; file.CopyTo(bundle); - return startOffset; + return (startOffset, 42); } private bool IsHost(string fileRelativePath) @@ -186,8 +225,8 @@ private FileType InferType(FileSpec fileSpec) /// public string GenerateBundle(IReadOnlyList fileSpecs) { - Tracer.Log($"Bundler version: {Manifest.CurrentVersion}"); - Tracer.Log($"Bundler Header: {BundleManifest.DesiredVersion}"); + Tracer.Log($"Bundler Version: {BundlerMajorVersion}.{BundlerMinorVersion}"); + Tracer.Log($"Bundle Version: {BundleManifest.BundleVersion}"); Tracer.Log($"Target Runtime: {Target}"); Tracer.Log($"Bundler Options: {Options}"); @@ -254,8 +293,8 @@ public string GenerateBundle(IReadOnlyList fileSpecs) using (FileStream file = File.OpenRead(fileSpec.SourcePath)) { FileType targetType = Target.TargetSpecificFileType(type); - long startOffset = AddToBundle(bundle, file, targetType); - FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length); + (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType); + FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion); Tracer.Log($"Embed: {entry}"); } } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs index 7315c2a026313a..5627d81401a430 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs @@ -15,28 +15,39 @@ namespace Microsoft.NET.HostModel.Bundle /// * Name ("NameLength" Bytes) /// * Offset (Int64) /// * Size (Int64) + /// === present only in bundle version 3+ + /// * CompressedSize (Int64) /// public class FileEntry { + public readonly uint BundleMajorVersion; + public readonly long Offset; public readonly long Size; + public readonly long CompressedSize; public readonly FileType Type; public readonly string RelativePath; // Path of an embedded file, relative to the Bundle source-directory. public const char DirectorySeparatorChar = '/'; - public FileEntry(FileType fileType, string relativePath, long offset, long size) + public FileEntry(FileType fileType, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion) { + BundleMajorVersion = bundleMajorVersion; Type = fileType; RelativePath = relativePath.Replace('\\', DirectorySeparatorChar); Offset = offset; Size = size; + CompressedSize = compressedSize; } public void Write(BinaryWriter writer) { writer.Write(Offset); writer.Write(Size); + if (BundleMajorVersion >= 3) + { + writer.Write(CompressedSize); + } writer.Write((byte)Type); writer.Write(RelativePath); } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs index 0beddb2b8cacc9..73e7b3b10fd243 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs @@ -66,32 +66,26 @@ private enum HeaderFlags : ulong // with path-names so that the AppHost can use it in // extraction path. public readonly string BundleID; - - public const uint CurrentMajorVersion = 2; - public readonly uint DesiredMajorVersion; + public readonly uint BundleMajorVersion; // The Minor version is currently unused, and is always zero - public const uint MinorVersion = 0; - - public static string CurrentVersion => $"{CurrentMajorVersion}.{MinorVersion}"; - public string DesiredVersion => $"{DesiredMajorVersion}.{MinorVersion}"; - + public const uint BundleMinorVersion = 0; private FileEntry DepsJsonEntry; private FileEntry RuntimeConfigJsonEntry; private HeaderFlags Flags; - public List Files; + public string BundleVersion => $"{BundleMajorVersion}.{BundleMinorVersion}"; - public Manifest(uint desiredVersion, bool netcoreapp3CompatMode = false) + public Manifest(uint bundleMajorVersion, bool netcoreapp3CompatMode = false) { - DesiredMajorVersion = desiredVersion; + BundleMajorVersion = bundleMajorVersion; Files = new List(); BundleID = Path.GetRandomFileName(); - Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode: HeaderFlags.None; + Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None; } - public FileEntry AddEntry(FileType type, string relativePath, long offset, long size) + public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion) { - FileEntry entry = new FileEntry(type, relativePath, offset, size); + FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion); Files.Add(entry); switch (entry.Type) @@ -118,12 +112,12 @@ public long Write(BinaryWriter writer) long startOffset = writer.BaseStream.Position; // Write the bundle header - writer.Write(DesiredMajorVersion); - writer.Write(MinorVersion); + writer.Write(BundleMajorVersion); + writer.Write(BundleMinorVersion); writer.Write(Files.Count); writer.Write(BundleID); - if (DesiredMajorVersion == 2) + if (BundleMajorVersion >= 2) { writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Offset : 0); writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Size : 0); diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs index 05d5ff9d89ee19..93a76db7650ccf 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs @@ -25,7 +25,7 @@ public class TargetInfo public readonly OSPlatform OS; public readonly Architecture Arch; public readonly Version FrameworkVersion; - public readonly uint BundleVersion; + public readonly uint BundleMajorVersion; public readonly BundleOptions DefaultOptions; public readonly int AssemblyAlignment; @@ -33,18 +33,23 @@ public TargetInfo(OSPlatform? os, Architecture? arch, Version targetFrameworkVer { OS = os ?? HostOS; Arch = arch ?? RuntimeInformation.OSArchitecture; - FrameworkVersion = targetFrameworkVersion ?? net50; + FrameworkVersion = targetFrameworkVersion ?? net60; Debug.Assert(IsLinux || IsOSX || IsWindows); - if (FrameworkVersion.CompareTo(net50) >= 0) + if (FrameworkVersion.CompareTo(net60) >= 0) { - BundleVersion = 2u; + BundleMajorVersion = 3u; + DefaultOptions = BundleOptions.None; + } + else if (FrameworkVersion.CompareTo(net50) >= 0) + { + BundleMajorVersion = 2u; DefaultOptions = BundleOptions.None; } else if (FrameworkVersion.Major == 3 && (FrameworkVersion.Minor == 0 || FrameworkVersion.Minor == 1)) { - BundleVersion = 1u; + BundleMajorVersion = 1u; DefaultOptions = BundleOptions.BundleAllContent; } else @@ -94,7 +99,7 @@ public override string ToString() // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup. // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration. - public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType; + public FileType TargetSpecificFileType(FileType fileType) => (BundleMajorVersion == 1) ? FileType.Unknown : fileType; // In .net core 3.x, bundle processing happens within the AppHost. // Therefore HostFxr and HostPolicy can be bundled within the single-file app. @@ -105,6 +110,7 @@ public override string ToString() public bool ShouldExclude(string relativePath) => (FrameworkVersion.Major != 3) && (relativePath.Equals(HostFxr) || relativePath.Equals(HostPolicy)); + private readonly Version net60 = new Version(6, 0); private readonly Version net50 = new Version(5, 0); private string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib"; private string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib"; diff --git a/src/native/corehost/bundle/extractor.cpp b/src/native/corehost/bundle/extractor.cpp index be3f2077a37892..ca3dd0e87cfd67 100644 --- a/src/native/corehost/bundle/extractor.cpp +++ b/src/native/corehost/bundle/extractor.cpp @@ -103,9 +103,21 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) { FILE* file = create_extraction_file(entry.relative_path()); reader.set_offset(entry.offset()); - int64_t size = entry.size(); + size_t size = entry.size(); size_t cast_size = to_size_t_dbgchecked(size); - if (fwrite(reader, 1, cast_size, file) != cast_size) + size_t extracted_size; + + if (entry.compressedSize() != 0) + { + // TODO: VS decompress + extracted_size = fwrite(reader, 1, size, file); + } + else + { + extracted_size = fwrite(reader, 1, cast_size, file); + } + + if (extracted_size != cast_size) { trace::error(_X("Failure extracting contents of the application bundle.")); trace::error(_X("I/O failure when writing extracted files.")); diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index e33b2cfa32dcf0..3f5d35c03602d4 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -10,15 +10,47 @@ using namespace bundle; bool file_entry_t::is_valid() const { - return m_offset > 0 && m_size >= 0 && + return m_offset > 0 && m_size >= 0 && m_compressedSize >= 0 && static_cast(m_type) < file_type_t::__last; } -file_entry_t file_entry_t::read(reader_t &reader, bool force_extraction) +file_entry_t file_entry_t::read(reader_t &reader, uint32_t major_version, bool force_extraction) { // First read the fixed-sized portion of file-entry - const file_entry_fixed_t* fixed_data = reinterpret_cast(reader.read_direct(sizeof(file_entry_fixed_t))); - file_entry_t entry(fixed_data, force_extraction); + file_entry_fixed_t fixed_data; + + fixed_data.offset = *(int64_t*)reader.read_direct(sizeof(int64_t)); + fixed_data.size = *(int64_t*)reader.read_direct(sizeof(int64_t)); + + // compressedSize is present only in v3+ headers + fixed_data.compressedSize = major_version >= 3 ? + *(int64_t*)reader.read_direct(sizeof(int64_t)) : + 0; + + fixed_data.type = *(file_type_t*)reader.read_direct(sizeof(file_type_t)); + + file_entry_t entry(&fixed_data, force_extraction); + + if (major_version == 2) + { + if (fixed_data.compressedSize != 0) + { + trace::error(_X("v2 must have csize == 0, has %d"), (int)fixed_data.compressedSize); + throw StatusCode::BundleExtractionFailure; + } + } + else if (major_version == 3) + { + if (fixed_data.compressedSize != 42) + { + trace::error(_X("v3 must have csize == 42, has %d"), (int)fixed_data.compressedSize); + throw StatusCode::BundleExtractionFailure; + } + } + else + { + trace::error(_X("version must be 2 or 3, has %d"), major_version); + } if (!entry.is_valid()) { diff --git a/src/native/corehost/bundle/file_entry.h b/src/native/corehost/bundle/file_entry.h index eb122efe868354..7a98c8b69ffadc 100644 --- a/src/native/corehost/bundle/file_entry.h +++ b/src/native/corehost/bundle/file_entry.h @@ -16,6 +16,7 @@ namespace bundle // Fixed size portion (file_entry_fixed_t) // - Offset // - Size + // - CompressedSize - only in bundleVersion 3+ // - File Entry Type // Variable Size portion // - relative path (7-bit extension encoded length prefixed string) @@ -25,6 +26,7 @@ namespace bundle { int64_t offset; int64_t size; + int64_t compressedSize; file_type_t type; }; #pragma pack(pop) @@ -56,23 +58,26 @@ namespace bundle m_offset = fixed_data->offset; m_size = fixed_data->size; + m_compressedSize = fixed_data->compressedSize; m_type = fixed_data->type; } const pal::string_t relative_path() const { return m_relative_path; } int64_t offset() const { return m_offset; } int64_t size() const { return m_size; } + int64_t compressedSize() const { return m_compressedSize; } file_type_t type() const { return m_type; } void disable() { m_disabled = true; } bool is_disabled() const { return m_disabled; } bool needs_extraction() const; bool matches(const pal::string_t& path) const { return (pal::pathcmp(relative_path(), path) == 0) && !is_disabled(); } - static file_entry_t read(reader_t &reader, bool force_extraction); + static file_entry_t read(reader_t &reader, uint32_t major_version, bool force_extraction); private: int64_t m_offset; int64_t m_size; + int64_t m_compressedSize; file_type_t m_type; pal::string_t m_relative_path; // Path of an embedded file, relative to the extraction directory. // If the file represented by this entry is also found in a servicing location, the servicing location must take precedence. diff --git a/src/native/corehost/bundle/header.cpp b/src/native/corehost/bundle/header.cpp index 05aa736c175b9f..f1ea7a66d8e043 100644 --- a/src/native/corehost/bundle/header.cpp +++ b/src/native/corehost/bundle/header.cpp @@ -15,9 +15,11 @@ bool header_fixed_t::is_valid() const return false; } - // .net 5 host expects the version information to be 2.0 + // .net 6 host expects the version information to be 3.0 + // .net 5 host expects the version information to be 2.0 (this code is not .net 5.0 host) // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework. - return (major_version == header_t::major_version) && (minor_version == header_t::minor_version); + return ((major_version == 3) && (minor_version == 0)) || + ((major_version == 2) && (minor_version == 0)); } header_t header_t::read(reader_t& reader) @@ -32,7 +34,7 @@ header_t header_t::read(reader_t& reader) throw StatusCode::BundleExtractionFailure; } - header_t header(fixed_header->num_embedded_files); + header_t header(fixed_header->major_version, fixed_header->minor_version, fixed_header->num_embedded_files); // bundle_id is a component of the extraction path reader.read_path_string(header.m_bundle_id); diff --git a/src/native/corehost/bundle/header.h b/src/native/corehost/bundle/header.h index f785203f38a0c2..1bc9241429c23b 100644 --- a/src/native/corehost/bundle/header.h +++ b/src/native/corehost/bundle/header.h @@ -34,7 +34,7 @@ namespace bundle bool is_valid() const; }; - // netcoreapp3_compat_mode flag is set on a .net5 app, which chooses to build single-file apps in .netcore3.x compat mode, + // netcoreapp3_compat_mode flag is set on a .net5+ app, which chooses to build single-file apps in .netcore3.x compat mode, // This indicates that: // All published files are bundled into the app; some of them will be extracted to disk. // AppContext.BaseDirectory is set to the extraction directory (and not the AppHost directory). @@ -72,8 +72,10 @@ namespace bundle struct header_t { public: - header_t(int32_t num_embedded_files = 0) + header_t(uint32_t major_version, uint32_t minor_version, int32_t num_embedded_files) : m_num_embedded_files(num_embedded_files) + , m_major_version(major_version) + , m_minor_version(minor_version) , m_bundle_id() , m_v2_header() { @@ -87,11 +89,13 @@ namespace bundle const location_t& runtimeconfig_json_location() const { return m_v2_header.runtimeconfig_json_location; } bool is_netcoreapp3_compat_mode() const { return m_v2_header.is_netcoreapp3_compat_mode(); } - static const uint32_t major_version = 2; - static const uint32_t minor_version = 0; + const uint32_t major_version() const { return m_major_version; }; + const uint32_t minor_version() const { return m_minor_version; }; private: int32_t m_num_embedded_files; + uint32_t m_major_version; + uint32_t m_minor_version; pal::string_t m_bundle_id; header_fixed_v2_t m_v2_header; }; diff --git a/src/native/corehost/bundle/info.cpp b/src/native/corehost/bundle/info.cpp index afb1ee1381f5f0..5f9f0dadcd08a3 100644 --- a/src/native/corehost/bundle/info.cpp +++ b/src/native/corehost/bundle/info.cpp @@ -11,11 +11,12 @@ using namespace bundle; const info_t* info_t::the_app = nullptr; info_t::info_t(const pal::char_t* bundle_path, - const pal::char_t* app_path, - int64_t header_offset) + const pal::char_t* app_path, + int64_t header_offset) : m_bundle_path(bundle_path) , m_bundle_size(0) , m_header_offset(header_offset) + , m_header(0, 0, 0) { m_base_path = get_directory(m_bundle_path); diff --git a/src/native/corehost/bundle/manifest.cpp b/src/native/corehost/bundle/manifest.cpp index cf439952871c7c..ed7ced64f3aaf1 100644 --- a/src/native/corehost/bundle/manifest.cpp +++ b/src/native/corehost/bundle/manifest.cpp @@ -11,7 +11,7 @@ manifest_t manifest_t::read(reader_t& reader, const header_t& header) for (int32_t i = 0; i < header.num_embedded_files(); i++) { - file_entry_t entry = file_entry_t::read(reader, header.is_netcoreapp3_compat_mode()); + file_entry_t entry = file_entry_t::read(reader, header.major_version(), header.is_netcoreapp3_compat_mode()); manifest.files.push_back(std::move(entry)); manifest.m_files_need_extraction |= entry.needs_extraction(); } From 812707c8de9e669da8c8ef5dc5b4737156ac4aec Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 18 Mar 2021 12:53:46 -0700 Subject: [PATCH 02/14] no need to support 2.0 bundle version --- src/native/corehost/bundle/file_entry.cpp | 38 ++++------------------- src/native/corehost/bundle/file_entry.h | 2 +- src/native/corehost/bundle/header.cpp | 3 +- src/native/corehost/bundle/manifest.cpp | 2 +- 4 files changed, 9 insertions(+), 36 deletions(-) diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index 3f5d35c03602d4..6f68fa50f53295 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -14,42 +14,16 @@ bool file_entry_t::is_valid() const static_cast(m_type) < file_type_t::__last; } -file_entry_t file_entry_t::read(reader_t &reader, uint32_t major_version, bool force_extraction) +file_entry_t file_entry_t::read(reader_t &reader, bool force_extraction) { // First read the fixed-sized portion of file-entry - file_entry_fixed_t fixed_data; + const file_entry_fixed_t* fixed_data = reinterpret_cast(reader.read_direct(sizeof(file_entry_fixed_t))); + file_entry_t entry(fixed_data, force_extraction); - fixed_data.offset = *(int64_t*)reader.read_direct(sizeof(int64_t)); - fixed_data.size = *(int64_t*)reader.read_direct(sizeof(int64_t)); - - // compressedSize is present only in v3+ headers - fixed_data.compressedSize = major_version >= 3 ? - *(int64_t*)reader.read_direct(sizeof(int64_t)) : - 0; - - fixed_data.type = *(file_type_t*)reader.read_direct(sizeof(file_type_t)); - - file_entry_t entry(&fixed_data, force_extraction); - - if (major_version == 2) - { - if (fixed_data.compressedSize != 0) - { - trace::error(_X("v2 must have csize == 0, has %d"), (int)fixed_data.compressedSize); - throw StatusCode::BundleExtractionFailure; - } - } - else if (major_version == 3) - { - if (fixed_data.compressedSize != 42) - { - trace::error(_X("v3 must have csize == 42, has %d"), (int)fixed_data.compressedSize); - throw StatusCode::BundleExtractionFailure; - } - } - else + if (fixed_data->compressedSize != 42) { - trace::error(_X("version must be 2 or 3, has %d"), major_version); + trace::error(_X("v3 must have csize == 42, has %d"), (int)fixed_data->compressedSize); + throw StatusCode::BundleExtractionFailure; } if (!entry.is_valid()) diff --git a/src/native/corehost/bundle/file_entry.h b/src/native/corehost/bundle/file_entry.h index 7a98c8b69ffadc..da1f6f1e79af27 100644 --- a/src/native/corehost/bundle/file_entry.h +++ b/src/native/corehost/bundle/file_entry.h @@ -72,7 +72,7 @@ namespace bundle bool needs_extraction() const; bool matches(const pal::string_t& path) const { return (pal::pathcmp(relative_path(), path) == 0) && !is_disabled(); } - static file_entry_t read(reader_t &reader, uint32_t major_version, bool force_extraction); + static file_entry_t read(reader_t &reader, bool force_extraction); private: int64_t m_offset; diff --git a/src/native/corehost/bundle/header.cpp b/src/native/corehost/bundle/header.cpp index f1ea7a66d8e043..3f2ca0c1e02078 100644 --- a/src/native/corehost/bundle/header.cpp +++ b/src/native/corehost/bundle/header.cpp @@ -18,8 +18,7 @@ bool header_fixed_t::is_valid() const // .net 6 host expects the version information to be 3.0 // .net 5 host expects the version information to be 2.0 (this code is not .net 5.0 host) // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework. - return ((major_version == 3) && (minor_version == 0)) || - ((major_version == 2) && (minor_version == 0)); + return ((major_version == 3) && (minor_version == 0)); } header_t header_t::read(reader_t& reader) diff --git a/src/native/corehost/bundle/manifest.cpp b/src/native/corehost/bundle/manifest.cpp index ed7ced64f3aaf1..cf439952871c7c 100644 --- a/src/native/corehost/bundle/manifest.cpp +++ b/src/native/corehost/bundle/manifest.cpp @@ -11,7 +11,7 @@ manifest_t manifest_t::read(reader_t& reader, const header_t& header) for (int32_t i = 0; i < header.num_embedded_files(); i++) { - file_entry_t entry = file_entry_t::read(reader, header.major_version(), header.is_netcoreapp3_compat_mode()); + file_entry_t entry = file_entry_t::read(reader, header.is_netcoreapp3_compat_mode()); manifest.files.push_back(std::move(entry)); manifest.m_files_need_extraction |= entry.needs_extraction(); } From 853bd9a90b5c189a2fab40bc94ccdb21be6ea4fd Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 18 Mar 2021 21:22:06 -0700 Subject: [PATCH 03/14] enable compression --- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 18 ++++-- .../BundledAppWithSubDirs.cs | 4 +- .../SingleFileApiTests.cs | 2 + src/native/corehost/bundle/extractor.cpp | 63 +++++++++++++++++-- src/native/corehost/bundle/file_entry.cpp | 6 -- 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 00809772eab427..5808ac26a985e5 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -65,7 +65,15 @@ private bool ShouldCompress(FileType type) return false; } - return false; // type == FileType.Symbols; + switch (type) + { + case FileType.Symbols: + case FileType.NativeBinary: + return true; + + default: + return false; + } } /// @@ -83,7 +91,9 @@ private bool ShouldCompress(FileType type) long fileLength = file.Length; file.Position = 0; - using (GZipStream compressionStream = new GZipStream(bundle, CompressionMode.Compress, leaveOpen: true)) + // We use DeflateStream here. + // It uses GZip algorithm, but with a trivial header that does not contain file info. + using (DeflateStream compressionStream = new DeflateStream(bundle, CompressionLevel.Optimal, leaveOpen: true)) { file.CopyTo(compressionStream); } @@ -95,7 +105,7 @@ private bool ShouldCompress(FileType type) } // compression rate was not good enough - // roll back the bundle and let the uncompressed code path take care of the entry. + // roll back the bundle offset and let the uncompressed code path take care of the entry. bundle.Seek(startOffset, SeekOrigin.Begin); } @@ -114,7 +124,7 @@ private bool ShouldCompress(FileType type) startOffset = bundle.Position; file.CopyTo(bundle); - return (startOffset, 42); + return (startOffset, 0); } private bool IsHost(string fileRelativePath) diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 2a0dc9ad537be6..2d9389a9285033 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -42,7 +42,7 @@ private void RunTheApp(string path, TestProjectFixture fixture) public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestFrameworkDependentFixture.Copy(); - UseFrameworkDependentHost(fixture); + UseSingleFileSelfContainedHost(fixture); var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) @@ -59,6 +59,7 @@ public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options) public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) @@ -75,6 +76,7 @@ public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) public void Bundled_With_Empty_File_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); var singleFile = BundleHelper.BundleApp(fixture, options); // Run the app diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs index 60425b9c7192f4..5e765a9c6c185c 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs @@ -91,6 +91,7 @@ public void GetCommandLineArgs_0_Non_Bundled_App() public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir() { var fixture = sharedTestState.TestFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile); string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name; string bundleDir = BundleHelper.GetBundleDir(fixture).FullName; @@ -110,6 +111,7 @@ public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir() public void AppContext_Native_Search_Dirs_Contains_Bundle_And_Extraction_Dirs() { var fixture = sharedTestState.TestFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries); string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name; string bundleDir = BundleHelper.GetBundleDir(fixture).FullName; diff --git a/src/native/corehost/bundle/extractor.cpp b/src/native/corehost/bundle/extractor.cpp index ca3dd0e87cfd67..9cc452db5ad466 100644 --- a/src/native/corehost/bundle/extractor.cpp +++ b/src/native/corehost/bundle/extractor.cpp @@ -7,6 +7,16 @@ #include "pal.h" #include "utils.h" +#if defined(NATIVE_LIBS_EMBEDDED) +extern "C" +{ +#include "../../../libraries/Native/AnyOS/zlib/pal_zlib.h" +} +#endif + +// Suppress prefast warning #6255: alloca indicates failure by raising a stack overflow exception +#pragma warning(disable:6255) + using namespace bundle; pal::string_t& extractor_t::extraction_dir() @@ -105,12 +115,57 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) reader.set_offset(entry.offset()); size_t size = entry.size(); size_t cast_size = to_size_t_dbgchecked(size); - size_t extracted_size; + size_t extracted_size = 0; if (entry.compressedSize() != 0) { - // TODO: VS decompress - extracted_size = fwrite(reader, 1, size, file); +#if defined(NATIVE_LIBS_EMBEDDED) + PAL_ZStream zStream; + zStream.nextIn = (uint8_t*)(const void*)reader; + zStream.availIn = entry.compressedSize(); + + const int Deflate_DefaultWindowBits = -15; // Legal values are 8..15 and -8..-15. 15 is the window size, + // negative val causes deflate to produce raw deflate data (no zlib header). + + int ret = CompressionNative_InflateInit2_(&zStream, Deflate_DefaultWindowBits); + if (ret != PAL_Z_OK) + { + trace::error(_X("Failure initializing zLib stream.")); + throw StatusCode::BundleExtractionIOError; + } + + const int bufSize = 4096; + uint8_t* buf = (uint8_t*)alloca(bufSize); + + do + { + zStream.nextOut = buf; + zStream.availOut = bufSize; + + ret = CompressionNative_Inflate(&zStream, PAL_Z_NOFLUSH); + if (ret < 0) + { + CompressionNative_InflateEnd(&zStream); + trace::error(_X("Failure inflating zLib stream. %s"), zStream.msg); + throw StatusCode::BundleExtractionIOError; + } + + int produced = bufSize - zStream.availOut; + if (fwrite(buf, 1, produced, file) != produced) + { + CompressionNative_InflateEnd(&zStream); + trace::error(_X("I/O failure when writing decompressed file.")); + throw StatusCode::BundleExtractionIOError; + } + + extracted_size += produced; + } while (zStream.availOut == 0); + + CompressionNative_InflateEnd(&zStream); +#else + trace::error(_X("Compressed file in a standalone host scenario?")); + throw StatusCode::BundleExtractionIOError; +#endif } else { @@ -119,7 +174,7 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) if (extracted_size != cast_size) { - trace::error(_X("Failure extracting contents of the application bundle.")); + trace::error(_X("Failure extracting contents of the application bundle. Expected size:%d Actual size:%d"), size, extracted_size); trace::error(_X("I/O failure when writing extracted files.")); throw StatusCode::BundleExtractionIOError; } diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index 6f68fa50f53295..bf6e9776c16507 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -20,12 +20,6 @@ file_entry_t file_entry_t::read(reader_t &reader, bool force_extraction) const file_entry_fixed_t* fixed_data = reinterpret_cast(reader.read_direct(sizeof(file_entry_fixed_t))); file_entry_t entry(fixed_data, force_extraction); - if (fixed_data->compressedSize != 42) - { - trace::error(_X("v3 must have csize == 42, has %d"), (int)fixed_data->compressedSize); - throw StatusCode::BundleExtractionFailure; - } - if (!entry.is_valid()) { trace::error(_X("Failure processing application bundle; possible file corruption.")); From 0c1b756b3fbb201f2126b2e33571664692b41aa4 Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 18 Mar 2021 23:29:48 -0700 Subject: [PATCH 04/14] fixes after rebase --- src/native/corehost/apphost/static/CMakeLists.txt | 4 ++++ src/native/corehost/hostmisc/utils.h | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/native/corehost/apphost/static/CMakeLists.txt b/src/native/corehost/apphost/static/CMakeLists.txt index c9222058ffdb85..81615e76487f55 100644 --- a/src/native/corehost/apphost/static/CMakeLists.txt +++ b/src/native/corehost/apphost/static/CMakeLists.txt @@ -19,6 +19,10 @@ set(SKIP_VERSIONING 1) include_directories(..) include_directories(../../json) +if(NOT CLR_CMAKE_TARGET_WIN32) + include_directories(../../../libraries/Native/Unix/Common) +endif() + set(SOURCES ../bundle_marker.cpp ./hostfxr_resolver.cpp diff --git a/src/native/corehost/hostmisc/utils.h b/src/native/corehost/hostmisc/utils.h index 932cb804675249..96b2a764559406 100644 --- a/src/native/corehost/hostmisc/utils.h +++ b/src/native/corehost/hostmisc/utils.h @@ -120,8 +120,9 @@ template size_t to_size_t_dbgchecked(T value) { assert(value >= 0); - assert(value < static_cast(std::numeric_limits::max())); - return static_cast(value); + size_t result = static_cast(value); + assert(result >= 0); + return result; } #endif From 84e379567a47edec7acd2d42cabcc1d203cd743d Mon Sep 17 00:00:00 2001 From: vsadov Date: Fri, 19 Mar 2021 10:17:35 -0700 Subject: [PATCH 05/14] build fix for Unix --- src/native/corehost/apphost/static/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/corehost/apphost/static/CMakeLists.txt b/src/native/corehost/apphost/static/CMakeLists.txt index 81615e76487f55..d7e129c5d2060f 100644 --- a/src/native/corehost/apphost/static/CMakeLists.txt +++ b/src/native/corehost/apphost/static/CMakeLists.txt @@ -20,7 +20,7 @@ include_directories(..) include_directories(../../json) if(NOT CLR_CMAKE_TARGET_WIN32) - include_directories(../../../libraries/Native/Unix/Common) + include_directories(../../../../libraries/Native/Unix/Common) endif() set(SOURCES From 10775b681b0a7caa2708f29f760663818d8e8cc3 Mon Sep 17 00:00:00 2001 From: vsadov Date: Fri, 19 Mar 2021 15:08:57 -0700 Subject: [PATCH 06/14] Fix build with GCC --- src/native/corehost/bundle/extractor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/corehost/bundle/extractor.cpp b/src/native/corehost/bundle/extractor.cpp index 9cc452db5ad466..a17b53fe672f5b 100644 --- a/src/native/corehost/bundle/extractor.cpp +++ b/src/native/corehost/bundle/extractor.cpp @@ -151,7 +151,7 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) } int produced = bufSize - zStream.availOut; - if (fwrite(buf, 1, produced, file) != produced) + if (fwrite(buf, 1, produced, file) != (size_t)produced) { CompressionNative_InflateEnd(&zStream); trace::error(_X("I/O failure when writing decompressed file.")); From 88c23b224e6fe524736bbfeb0c7e7abe73c85b16 Mon Sep 17 00:00:00 2001 From: vsadov Date: Fri, 19 Mar 2021 15:09:26 -0700 Subject: [PATCH 07/14] disable a test temporarily to see what else breaks --- .../AppHost.Bundle.Tests/BundledAppWithSubDirs.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 2d9389a9285033..088306d7519f1d 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -35,6 +35,7 @@ private void RunTheApp(string path, TestProjectFixture fixture) .HaveStdOutContaining("Wow! We now say hello to the big world and you."); } + [ActiveIssue("NYI, disable for now to work on the rest")] [InlineData(BundleOptions.None)] [InlineData(BundleOptions.BundleNativeBinaries)] [InlineData(BundleOptions.BundleAllContent)] @@ -42,7 +43,7 @@ private void RunTheApp(string path, TestProjectFixture fixture) public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestFrameworkDependentFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); + UseFrameworkDependentHost(fixture); var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) From 05398d21047d46550de0ec6d7b727757879c13fc Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 22 Mar 2021 10:48:08 -0700 Subject: [PATCH 08/14] Add EnableCompression option to Bundler + PR feedback --- .../Microsoft.NET.HostModel/Bundle/BundleOptions.cs | 1 + .../managed/Microsoft.NET.HostModel/Bundle/Bundler.cs | 9 +++++++-- .../Microsoft.NET.HostModel/Bundle/FileEntry.cs | 2 +- .../Microsoft.NET.HostModel/Bundle/TargetInfo.cs | 2 +- .../BundleExtractToSpecificPath.cs | 10 +++++++--- .../AppHost.Bundle.Tests/BundleTestBase.cs | 1 + .../AppHost.Bundle.Tests/BundledAppWithSubDirs.cs | 3 ++- .../AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs | 3 ++- .../Helpers/BundleHelper.cs | 2 +- src/native/corehost/bundle/file_entry.h | 2 +- src/native/corehost/bundle/header.cpp | 6 +++--- 11 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs index 646e1b094310dd..22030386d26737 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs @@ -17,5 +17,6 @@ public enum BundleOptions BundleOtherFiles = 2, BundleSymbolFiles = 4, BundleAllContent = BundleNativeBinaries | BundleOtherFiles, + EnableCompression = 8, }; } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 5808ac26a985e5..a210bb3c07f705 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -59,8 +59,13 @@ public Bundler(string hostName, private bool ShouldCompress(FileType type) { - // compression is not supported before bundle vesion 3 - if (Target.BundleMajorVersion < 3) + // compression is not supported before bundle vesion 6 + if (Target.BundleMajorVersion < 6) + { + return false; + } + + if (!Options.HasFlag(BundleOptions.EnableCompression)) { return false; } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs index 5627d81401a430..df3f54e316d43e 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs @@ -16,7 +16,7 @@ namespace Microsoft.NET.HostModel.Bundle /// * Offset (Int64) /// * Size (Int64) /// === present only in bundle version 3+ - /// * CompressedSize (Int64) + /// * CompressedSize (Int64) 0 indicates No Compression /// public class FileEntry { diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs index 93a76db7650ccf..2d3f6566a5ad5c 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs @@ -39,7 +39,7 @@ public TargetInfo(OSPlatform? os, Architecture? arch, Version targetFrameworkVer if (FrameworkVersion.CompareTo(net60) >= 0) { - BundleMajorVersion = 3u; + BundleMajorVersion = 6u; DefaultOptions = BundleOptions.None; } else if (FrameworkVersion.CompareTo(net50) >= 0) diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index 1741c9584b3263..bfcdc7090e0df5 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -32,7 +32,8 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() // Publish the bundle UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options: BundleOptions.BundleNativeBinaries); + BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); // Verify expected files in the bundle directory var bundleDir = BundleHelper.GetBundleDir(fixture); @@ -81,6 +82,7 @@ private void Bundle_Extraction_To_Relative_Path_Succeeds(string relativePath, Bu var fixture = sharedTestState.TestFixture.Copy(); UseSingleFileSelfContainedHost(fixture); + bundleOptions |= BundleOptions.EnableCompression; var bundler = BundleHelper.BundleApp(fixture, out var singleFile, bundleOptions); // Run the bundled app (extract files to ) @@ -111,7 +113,8 @@ private void Bundle_extraction_is_reused() // Publish the bundle UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries); + BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); @@ -161,7 +164,8 @@ private void Bundle_extraction_can_recover_missing_files() // Publish the bundle UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries); + BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs index c71aaa35e5bdb9..21da1e259681fd 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs @@ -47,6 +47,7 @@ public static string BundleSelfContainedApp( Version targetFrameworkVersion = null) { UseSingleFileSelfContainedHost(testFixture); + options |= BundleOptions.EnableCompression; return BundleHelper.BundleApp(testFixture, options, targetFrameworkVersion); } diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 088306d7519f1d..a2edee6d1c5b10 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -35,7 +35,6 @@ private void RunTheApp(string path, TestProjectFixture fixture) .HaveStdOutContaining("Wow! We now say hello to the big world and you."); } - [ActiveIssue("NYI, disable for now to work on the rest")] [InlineData(BundleOptions.None)] [InlineData(BundleOptions.BundleNativeBinaries)] [InlineData(BundleOptions.BundleAllContent)] @@ -61,6 +60,7 @@ public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); UseSingleFileSelfContainedHost(fixture); + options |= BundleOptions.EnableCompression; var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) @@ -78,6 +78,7 @@ public void Bundled_With_Empty_File_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy(); UseSingleFileSelfContainedHost(fixture); + options |= BundleOptions.EnableCompression; var singleFile = BundleHelper.BundleApp(fixture, options); // Run the app diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs index 95e1061f20c27a..db32593391fc5b 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs @@ -27,7 +27,8 @@ public void Bundle_Is_Extracted() { var fixture = sharedTestState.TestFixture.Copy(); UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleAllContent); + BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleAllContent; + Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); var extractionBaseDir = BundleHelper.GetExtractionRootDir(fixture); Command.Create(singleFile, "executing_assembly_location trusted_platform_assemblies assembly_location System.Console") diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs index 6a102ddade96a6..ecd00f75c6fd1f 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs @@ -60,7 +60,7 @@ public static string[] GetBundledFiles(TestProjectFixture fixture) public static string[] GetExtractedFiles(TestProjectFixture fixture, BundleOptions bundleOptions) { - switch (bundleOptions) + switch (bundleOptions & ~BundleOptions.EnableCompression) { case BundleOptions.None: case BundleOptions.BundleOtherFiles: diff --git a/src/native/corehost/bundle/file_entry.h b/src/native/corehost/bundle/file_entry.h index da1f6f1e79af27..2c42dd5587a20f 100644 --- a/src/native/corehost/bundle/file_entry.h +++ b/src/native/corehost/bundle/file_entry.h @@ -16,7 +16,7 @@ namespace bundle // Fixed size portion (file_entry_fixed_t) // - Offset // - Size - // - CompressedSize - only in bundleVersion 3+ + // - CompressedSize - only in bundleVersion 6+ // - File Entry Type // Variable Size portion // - relative path (7-bit extension encoded length prefixed string) diff --git a/src/native/corehost/bundle/header.cpp b/src/native/corehost/bundle/header.cpp index 3f2ca0c1e02078..d212b9873838d4 100644 --- a/src/native/corehost/bundle/header.cpp +++ b/src/native/corehost/bundle/header.cpp @@ -15,10 +15,10 @@ bool header_fixed_t::is_valid() const return false; } - // .net 6 host expects the version information to be 3.0 - // .net 5 host expects the version information to be 2.0 (this code is not .net 5.0 host) + // .net 6 host expects the header version to be 6.0 + // .net 5 host expects the header version to be 2.0 (this code is not .net 5.0 host) // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework. - return ((major_version == 3) && (minor_version == 0)); + return ((major_version == 6) && (minor_version == 0)); } header_t header_t::read(reader_t& reader) From dcacd81e02742ff12a272a9d95b1a0a04c9e6f64 Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 22 Mar 2021 11:00:03 -0700 Subject: [PATCH 09/14] Couple more places should use version 6.0 --- .../managed/Microsoft.NET.HostModel/Bundle/Bundler.cs | 2 +- .../managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index a210bb3c07f705..f8a071d89a3092 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -19,7 +19,7 @@ namespace Microsoft.NET.HostModel.Bundle /// public class Bundler { - public const uint BundlerMajorVersion = 3; + public const uint BundlerMajorVersion = 6; public const uint BundlerMinorVersion = 0; private readonly string HostName; diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs index df3f54e316d43e..f2e537af49c770 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs @@ -44,7 +44,8 @@ public void Write(BinaryWriter writer) { writer.Write(Offset); writer.Write(Size); - if (BundleMajorVersion >= 3) + // compression is used only in version 6.0+ + if (BundleMajorVersion >= 6) { writer.Write(CompressedSize); } From 2649177da6a62c1c3535de9d06524e08e6aba78c Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 25 Mar 2021 19:34:13 -0700 Subject: [PATCH 10/14] PR feedback (header versioning, more tests, fixed an assert) --- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 12 ++-- .../BundledAppWithSubDirs.cs | 62 +++++++++++++++++++ src/native/corehost/bundle/file_entry.cpp | 17 ++++- src/native/corehost/bundle/file_entry.h | 2 +- src/native/corehost/bundle/header.cpp | 7 ++- src/native/corehost/bundle/manifest.cpp | 2 +- src/native/corehost/hostmisc/utils.h | 2 +- 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index f8a071d89a3092..11f691ab4b6cc6 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -48,6 +48,12 @@ public Bundler(string hostName, OutputDir = Path.GetFullPath(string.IsNullOrEmpty(outputDir) ? Environment.CurrentDirectory : outputDir); Target = new TargetInfo(targetOS, targetArch, targetFrameworkVersion); + if (Target.BundleMajorVersion < 6 && + (options & BundleOptions.EnableCompression) != 0) + { + throw new ArgumentException("Compression requires framework version 6.0 or above", nameof(options)); + } + appAssemblyName ??= Target.GetAssemblyName(hostName); DepsJson = appAssemblyName + ".deps.json"; RuntimeConfigJson = appAssemblyName + ".runtimeconfig.json"; @@ -59,12 +65,6 @@ public Bundler(string hostName, private bool ShouldCompress(FileType type) { - // compression is not supported before bundle vesion 6 - if (Target.BundleMajorVersion < 6) - { - return false; - } - if (!Options.HasFlag(BundleOptions.EnableCompression)) { return false; diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index a2edee6d1c5b10..8c57b6e2a3eb64 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -70,6 +70,68 @@ public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) RunTheApp(singleFile, fixture); } + [InlineData(BundleOptions.None)] + [InlineData(BundleOptions.BundleNativeBinaries)] + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_Self_Contained_NoCompression_App_Run_Succeeds(BundleOptions options) + { + var fixture = sharedTestState.TestSelfContainedFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); + // it is ok to not enable compression + var singleFile = BundleHelper.BundleApp(fixture, options); + + // Run the bundled app (extract files) + RunTheApp(singleFile, fixture); + + // Run the bundled app again (reuse extracted files) + RunTheApp(singleFile, fixture); + } + + [InlineData(BundleOptions.None)] + [InlineData(BundleOptions.BundleNativeBinaries)] + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_Self_Contained_Targeting50_App_Run_Succeeds(BundleOptions options) + { + var fixture = sharedTestState.TestSelfContainedFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); + // compression must be off when targeting 5.0 + var singleFile = BundleHelper.BundleApp(fixture, options, new Version(5, 0)); + + // Run the bundled app (extract files) + RunTheApp(singleFile, fixture); + + // Run the bundled app again (reuse extracted files) + RunTheApp(singleFile, fixture); + } + + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_Framework_dependent_Targeting50_App_Run_Succeeds(BundleOptions options) + { + var fixture = sharedTestState.TestSelfContainedFixture.Copy(); + UseFrameworkDependentHost(fixture); + var singleFile = BundleHelper.BundleApp(fixture, options, new Version(5, 0)); + + // Run the bundled app (extract files) + RunTheApp(singleFile, fixture); + + // Run the bundled app again (reuse extracted files) + RunTheApp(singleFile, fixture); + } + + [Fact] + public void Bundled_Self_Contained_Targeting50_WithCompression_Throws() + { + var fixture = sharedTestState.TestSelfContainedFixture.Copy(); + UseSingleFileSelfContainedHost(fixture); + // compression must be off when targeting 5.0 + var options = BundleOptions.EnableCompression; + + Assert.Throws(()=>BundleHelper.BundleApp(fixture, options, new Version(5, 0))); + } + [InlineData(BundleOptions.None)] [InlineData(BundleOptions.BundleNativeBinaries)] [InlineData(BundleOptions.BundleAllContent)] diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index bf6e9776c16507..979f7e02f2ca70 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -14,11 +14,22 @@ bool file_entry_t::is_valid() const static_cast(m_type) < file_type_t::__last; } -file_entry_t file_entry_t::read(reader_t &reader, bool force_extraction) +file_entry_t file_entry_t::read(reader_t &reader, uint32_t major_version, bool force_extraction) { // First read the fixed-sized portion of file-entry - const file_entry_fixed_t* fixed_data = reinterpret_cast(reader.read_direct(sizeof(file_entry_fixed_t))); - file_entry_t entry(fixed_data, force_extraction); + file_entry_fixed_t fixed_data; + + fixed_data.offset = *(int64_t*)reader.read_direct(sizeof(int64_t)); + fixed_data.size = *(int64_t*)reader.read_direct(sizeof(int64_t)); + + // compressedSize is present only in v6+ headers + fixed_data.compressedSize = major_version >= 6 ? + *(int64_t*)reader.read_direct(sizeof(int64_t)) : + 0; + + fixed_data.type = *(file_type_t*)reader.read_direct(sizeof(file_type_t)); + + file_entry_t entry(&fixed_data, force_extraction); if (!entry.is_valid()) { diff --git a/src/native/corehost/bundle/file_entry.h b/src/native/corehost/bundle/file_entry.h index 2c42dd5587a20f..3e22b083d02567 100644 --- a/src/native/corehost/bundle/file_entry.h +++ b/src/native/corehost/bundle/file_entry.h @@ -72,7 +72,7 @@ namespace bundle bool needs_extraction() const; bool matches(const pal::string_t& path) const { return (pal::pathcmp(relative_path(), path) == 0) && !is_disabled(); } - static file_entry_t read(reader_t &reader, bool force_extraction); + static file_entry_t read(reader_t &reader, uint32_t major_version, bool force_extraction); private: int64_t m_offset; diff --git a/src/native/corehost/bundle/header.cpp b/src/native/corehost/bundle/header.cpp index d212b9873838d4..268ef867146ae0 100644 --- a/src/native/corehost/bundle/header.cpp +++ b/src/native/corehost/bundle/header.cpp @@ -15,10 +15,11 @@ bool header_fixed_t::is_valid() const return false; } - // .net 6 host expects the header version to be 6.0 - // .net 5 host expects the header version to be 2.0 (this code is not .net 5.0 host) + // .net 6 host expects the version information to be 6.0 + // .net 5 host expects the version information to be 2.0 // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework. - return ((major_version == 6) && (minor_version == 0)); + return ((major_version == 6) && (minor_version == 0)) || + ((major_version == 2) && (minor_version == 0)); } header_t header_t::read(reader_t& reader) diff --git a/src/native/corehost/bundle/manifest.cpp b/src/native/corehost/bundle/manifest.cpp index cf439952871c7c..ed7ced64f3aaf1 100644 --- a/src/native/corehost/bundle/manifest.cpp +++ b/src/native/corehost/bundle/manifest.cpp @@ -11,7 +11,7 @@ manifest_t manifest_t::read(reader_t& reader, const header_t& header) for (int32_t i = 0; i < header.num_embedded_files(); i++) { - file_entry_t entry = file_entry_t::read(reader, header.is_netcoreapp3_compat_mode()); + file_entry_t entry = file_entry_t::read(reader, header.major_version(), header.is_netcoreapp3_compat_mode()); manifest.files.push_back(std::move(entry)); manifest.m_files_need_extraction |= entry.needs_extraction(); } diff --git a/src/native/corehost/hostmisc/utils.h b/src/native/corehost/hostmisc/utils.h index 96b2a764559406..d0dc381b69cca1 100644 --- a/src/native/corehost/hostmisc/utils.h +++ b/src/native/corehost/hostmisc/utils.h @@ -121,7 +121,7 @@ size_t to_size_t_dbgchecked(T value) { assert(value >= 0); size_t result = static_cast(value); - assert(result >= 0); + assert(static_cast(result) == value); return result; } From 15d4b61c0e9519d9326c7570ae4d583f2cbe1be6 Mon Sep 17 00:00:00 2001 From: Vladimir Sadov Date: Mon, 29 Mar 2021 08:13:07 -0700 Subject: [PATCH 11/14] Suggestion from PR review Co-authored-by: Vitek Karas --- src/native/corehost/bundle/file_entry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index 979f7e02f2ca70..e5ae492d1e4ccf 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -14,7 +14,7 @@ bool file_entry_t::is_valid() const static_cast(m_type) < file_type_t::__last; } -file_entry_t file_entry_t::read(reader_t &reader, uint32_t major_version, bool force_extraction) +file_entry_t file_entry_t::read(reader_t &reader, uint32_t bundle_major_version, bool force_extraction) { // First read the fixed-sized portion of file-entry file_entry_fixed_t fixed_data; From 19b3bbc1f7cf24edfc6accac692596657096543c Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 29 Mar 2021 08:26:35 -0700 Subject: [PATCH 12/14] sorted usings --- .../managed/Microsoft.NET.HostModel/Bundle/Bundler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 11f691ab4b6cc6..bee21ddc065dbd 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -1,15 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.NET.HostModel.AppHost; using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.IO; +using System.IO.Compression; +using System.Linq; using System.Reflection.PortableExecutable; using System.Runtime.InteropServices; -using System.IO.Compression; +using Microsoft.NET.HostModel.AppHost; namespace Microsoft.NET.HostModel.Bundle { From 772797634931360a75701fd6fd4fd025d1285ed1 Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 29 Mar 2021 08:30:53 -0700 Subject: [PATCH 13/14] should be bundle_major_version in two more places. --- src/native/corehost/bundle/file_entry.cpp | 2 +- src/native/corehost/bundle/file_entry.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index e5ae492d1e4ccf..ace0ef514927e6 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -23,7 +23,7 @@ file_entry_t file_entry_t::read(reader_t &reader, uint32_t bundle_major_version, fixed_data.size = *(int64_t*)reader.read_direct(sizeof(int64_t)); // compressedSize is present only in v6+ headers - fixed_data.compressedSize = major_version >= 6 ? + fixed_data.compressedSize = bundle_major_version >= 6 ? *(int64_t*)reader.read_direct(sizeof(int64_t)) : 0; diff --git a/src/native/corehost/bundle/file_entry.h b/src/native/corehost/bundle/file_entry.h index 3e22b083d02567..225cf225bcde4e 100644 --- a/src/native/corehost/bundle/file_entry.h +++ b/src/native/corehost/bundle/file_entry.h @@ -72,7 +72,7 @@ namespace bundle bool needs_extraction() const; bool matches(const pal::string_t& path) const { return (pal::pathcmp(relative_path(), path) == 0) && !is_disabled(); } - static file_entry_t read(reader_t &reader, uint32_t major_version, bool force_extraction); + static file_entry_t read(reader_t &reader, uint32_t bundle_major_version, bool force_extraction); private: int64_t m_offset; From 92cec71f1017d61f76f3d8097c7722ccaed03fe0 Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 29 Mar 2021 12:35:32 -0700 Subject: [PATCH 14/14] More PR feedback --- .../Bundle/FileEntry.cs | 2 +- .../BundleExtractToSpecificPath.cs | 20 ++++++---------- .../AppHost.Bundle.Tests/BundleTestBase.cs | 23 ++++++++++++++++--- .../BundledAppWithSubDirs.cs | 16 ++++--------- .../NetCoreApp3CompatModeTests.cs | 5 ++-- .../SingleFileApiTests.cs | 6 ++--- src/native/corehost/bundle/extractor.cpp | 4 ++-- 7 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs index f2e537af49c770..e71a7faaa45789 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs @@ -53,6 +53,6 @@ public void Write(BinaryWriter writer) writer.Write(RelativePath); } - public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size}"; + public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size} CompressedSz={CompressedSize}"; } } diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index bfcdc7090e0df5..33c895548b2195 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -31,9 +31,8 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() var hostName = BundleHelper.GetHostName(fixture); // Publish the bundle - UseSingleFileSelfContainedHost(fixture); - BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); + BundleOptions options = BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options); // Verify expected files in the bundle directory var bundleDir = BundleHelper.GetBundleDir(fixture); @@ -81,9 +80,7 @@ private void Bundle_Extraction_To_Relative_Path_Succeeds(string relativePath, Bu return; var fixture = sharedTestState.TestFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - bundleOptions |= BundleOptions.EnableCompression; - var bundler = BundleHelper.BundleApp(fixture, out var singleFile, bundleOptions); + var bundler = BundleSelfContainedApp(fixture, out var singleFile, bundleOptions); // Run the bundled app (extract files to ) var cmd = Command.Create(singleFile); @@ -112,9 +109,8 @@ private void Bundle_extraction_is_reused() var fixture = sharedTestState.TestFixture.Copy(); // Publish the bundle - UseSingleFileSelfContainedHost(fixture); - BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); + BundleOptions options = BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); @@ -163,14 +159,12 @@ private void Bundle_extraction_can_recover_missing_files() var appName = Path.GetFileNameWithoutExtension(hostName); // Publish the bundle - UseSingleFileSelfContainedHost(fixture); - BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries; - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); + BundleOptions options = BundleOptions.BundleNativeBinaries; + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); - // Run the bunded app for the first time, and extract files to // $DOTNET_BUNDLE_EXTRACT_BASE_DIR//bundle-id Command.Create(singleFile) diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs index 21da1e259681fd..18aba06f6d9786 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs @@ -44,11 +44,28 @@ public static string UseFrameworkDependentHost(TestProjectFixture testFixture) public static string BundleSelfContainedApp( TestProjectFixture testFixture, BundleOptions options = BundleOptions.None, - Version targetFrameworkVersion = null) + Version targetFrameworkVersion = null, + bool disableCompression = false) + { + string singleFile; + BundleSelfContainedApp(testFixture, out singleFile, options, targetFrameworkVersion); + return singleFile; + } + + public static Bundler BundleSelfContainedApp( + TestProjectFixture testFixture, + out string singleFile, + BundleOptions options = BundleOptions.None, + Version targetFrameworkVersion = null, + bool disableCompression = false) { UseSingleFileSelfContainedHost(testFixture); - options |= BundleOptions.EnableCompression; - return BundleHelper.BundleApp(testFixture, options, targetFrameworkVersion); + if (targetFrameworkVersion == null || targetFrameworkVersion >= new Version(6, 0)) + { + options |= BundleOptions.EnableCompression; + } + + return BundleHelper.BundleApp(testFixture, out singleFile, options, targetFrameworkVersion); } public abstract class SharedTestStateBase diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 8c57b6e2a3eb64..9ede20c516e393 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -59,9 +59,7 @@ public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options) public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - options |= BundleOptions.EnableCompression; - var singleFile = BundleHelper.BundleApp(fixture, options); + var singleFile = BundleSelfContainedApp(fixture, options); // Run the bundled app (extract files) RunTheApp(singleFile, fixture); @@ -77,9 +75,7 @@ public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) public void Bundled_Self_Contained_NoCompression_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - // it is ok to not enable compression - var singleFile = BundleHelper.BundleApp(fixture, options); + var singleFile = BundleSelfContainedApp(fixture, options, disableCompression: true); // Run the bundled app (extract files) RunTheApp(singleFile, fixture); @@ -95,9 +91,7 @@ public void Bundled_Self_Contained_NoCompression_App_Run_Succeeds(BundleOptions public void Bundled_Self_Contained_Targeting50_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - // compression must be off when targeting 5.0 - var singleFile = BundleHelper.BundleApp(fixture, options, new Version(5, 0)); + var singleFile = BundleSelfContainedApp(fixture, options, new Version(5, 0)); // Run the bundled app (extract files) RunTheApp(singleFile, fixture); @@ -139,9 +133,7 @@ public void Bundled_Self_Contained_Targeting50_WithCompression_Throws() public void Bundled_With_Empty_File_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - options |= BundleOptions.EnableCompression; - var singleFile = BundleHelper.BundleApp(fixture, options); + var singleFile = BundleSelfContainedApp(fixture, options); // Run the app RunTheApp(singleFile, fixture); diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs index db32593391fc5b..cb8bdf9d44a8e9 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs @@ -26,9 +26,8 @@ public NetCoreApp3CompatModeTests(SingleFileSharedState fixture) public void Bundle_Is_Extracted() { var fixture = sharedTestState.TestFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleAllContent; - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options); + BundleOptions options = BundleOptions.BundleAllContent; + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options); var extractionBaseDir = BundleHelper.GetExtractionRootDir(fixture); Command.Create(singleFile, "executing_assembly_location trusted_platform_assemblies assembly_location System.Console") diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs index 5e765a9c6c185c..0efa9a7819189c 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs @@ -91,8 +91,7 @@ public void GetCommandLineArgs_0_Non_Bundled_App() public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir() { var fixture = sharedTestState.TestFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile); + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile); string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name; string bundleDir = BundleHelper.GetBundleDir(fixture).FullName; @@ -111,8 +110,7 @@ public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir() public void AppContext_Native_Search_Dirs_Contains_Bundle_And_Extraction_Dirs() { var fixture = sharedTestState.TestFixture.Copy(); - UseSingleFileSelfContainedHost(fixture); - Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries); + Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries); string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name; string bundleDir = BundleHelper.GetBundleDir(fixture).FullName; diff --git a/src/native/corehost/bundle/extractor.cpp b/src/native/corehost/bundle/extractor.cpp index a17b53fe672f5b..e33daf8e70ce06 100644 --- a/src/native/corehost/bundle/extractor.cpp +++ b/src/native/corehost/bundle/extractor.cpp @@ -113,7 +113,7 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) { FILE* file = create_extraction_file(entry.relative_path()); reader.set_offset(entry.offset()); - size_t size = entry.size(); + int64_t size = entry.size(); size_t cast_size = to_size_t_dbgchecked(size); size_t extracted_size = 0; @@ -163,7 +163,7 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader) CompressionNative_InflateEnd(&zStream); #else - trace::error(_X("Compressed file in a standalone host scenario?")); + trace::error(_X("Failure extracting contents of the application bundle. Compressed files used with a standalone (not singlefile) apphost.")); throw StatusCode::BundleExtractionIOError; #endif }