Singlefile compression of native files

src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs

@@ -17,5 +17,6 @@ public enum BundleOptions
         BundleOtherFiles = 2,
         BundleSymbolFiles = 4,
         BundleAllContent = BundleNativeBinaries | BundleOtherFiles,
+        EnableCompression = 8,
     };
 }

src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs

@@ -1,14 +1,15 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Microsoft.NET.HostModel.AppHost;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.IO;
+using System.IO.Compression;
+using System.Linq;
 using System.Reflection.PortableExecutable;
 using System.Runtime.InteropServices;
+using Microsoft.NET.HostModel.AppHost;
 
 namespace Microsoft.NET.HostModel.Bundle
 {
@@ -18,6 +19,9 @@ namespace Microsoft.NET.HostModel.Bundle
     /// </summary>
     public class Bundler
     {
+        public const uint BundlerMajorVersion = 6;
+        public const uint BundlerMinorVersion = 0;
+
         private readonly string HostName;
         private readonly string OutputDir;
         private readonly string DepsJson;
@@ -44,22 +48,72 @@ public Bundler(string hostName,
             OutputDir = Path.GetFullPath(string.IsNullOrEmpty(outputDir) ? Environment.CurrentDirectory : outputDir);
             Target = new TargetInfo(targetOS, targetArch, targetFrameworkVersion);
 
+            if (Target.BundleMajorVersion < 6 &&
+                (options & BundleOptions.EnableCompression) != 0)
+            {
+                throw new ArgumentException("Compression requires framework version 6.0 or above", nameof(options));
+            }
+
             appAssemblyName ??= Target.GetAssemblyName(hostName);
             DepsJson = appAssemblyName + ".deps.json";
             RuntimeConfigJson = appAssemblyName + ".runtimeconfig.json";
             RuntimeConfigDevJson = appAssemblyName + ".runtimeconfig.dev.json";
 
-            BundleManifest = new Manifest(Target.BundleVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
+            BundleManifest = new Manifest(Target.BundleMajorVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
             Options = Target.DefaultOptions | options;
         }
 
+        private bool ShouldCompress(FileType type)
+        {
+            if (!Options.HasFlag(BundleOptions.EnableCompression))
+            {
+                return false;
+            }
+
+            switch (type)
+            {
+                case FileType.Symbols:
+                case FileType.NativeBinary:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
         /// <summary>
         /// Embed 'file' into 'bundle'
         /// </summary>
-        /// <returns>Returns the offset of the start 'file' within 'bundle'</returns>
-
-        private long AddToBundle(Stream bundle, Stream file, FileType type)
+        /// <returns>
+        /// startOffset: offset of the start 'file' within 'bundle'
+        /// compressedSize: size of the compressed data, if entry was compressed, otherwise 0
+        /// </returns>
+        private (long startOffset, long compressedSize) AddToBundle(Stream bundle, Stream file, FileType type)
         {
+            long startOffset = bundle.Position;
+            if (ShouldCompress(type))
+            {
+                long fileLength = file.Length;
+                file.Position = 0;
+
+                // We use DeflateStream here.
+                // It uses GZip algorithm, but with a trivial header that does not contain file info.
+                using (DeflateStream compressionStream = new DeflateStream(bundle, CompressionLevel.Optimal, leaveOpen: true))
+                {
+                    file.CopyTo(compressionStream);
+                }
+
+                long compressedSize = bundle.Position - startOffset;
+                if (compressedSize < fileLength * 0.75)
+                {
+                    return (startOffset, compressedSize);
+                }
+
+                // compression rate was not good enough
+                // roll back the bundle offset and let the uncompressed code path take care of the entry.
+                bundle.Seek(startOffset, SeekOrigin.Begin);
+            }
+
             if (type == FileType.Assembly)
             {
                 long misalignment = (bundle.Position % Target.AssemblyAlignment);
@@ -72,10 +126,10 @@ private long AddToBundle(Stream bundle, Stream file, FileType type)
             }
 
             file.Position = 0;
-            long startOffset = bundle.Position;
+            startOffset = bundle.Position;
             file.CopyTo(bundle);
 
-            return startOffset;
+            return (startOffset, 0);
         }
 
         private bool IsHost(string fileRelativePath)
@@ -186,8 +240,8 @@ private FileType InferType(FileSpec fileSpec)
         /// </exceptions>
         public string GenerateBundle(IReadOnlyList<FileSpec> fileSpecs)
         {
-            Tracer.Log($"Bundler version: {Manifest.CurrentVersion}");
-            Tracer.Log($"Bundler Header: {BundleManifest.DesiredVersion}");
+            Tracer.Log($"Bundler Version: {BundlerMajorVersion}.{BundlerMinorVersion}");
+            Tracer.Log($"Bundle  Version: {BundleManifest.BundleVersion}");
             Tracer.Log($"Target Runtime: {Target}");
             Tracer.Log($"Bundler Options: {Options}");
 
@@ -254,8 +308,8 @@ public string GenerateBundle(IReadOnlyList<FileSpec> fileSpecs)
                     using (FileStream file = File.OpenRead(fileSpec.SourcePath))
                     {
                         FileType targetType = Target.TargetSpecificFileType(type);
-                        long startOffset = AddToBundle(bundle, file, targetType);
-                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length);
+                        (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType);
+                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion);
                         Tracer.Log($"Embed: {entry}");
                     }
                 }

src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs

@@ -15,32 +15,44 @@ namespace Microsoft.NET.HostModel.Bundle
     /// * Name       ("NameLength" Bytes)
     /// * Offset     (Int64)
     /// * Size       (Int64)
+    /// === present only in bundle version 3+
+    /// * CompressedSize   (Int64)  0 indicates No Compression
     /// </summary>
     public class FileEntry
     {
+        public readonly uint BundleMajorVersion;
+
         public readonly long Offset;
         public readonly long Size;
+        public readonly long CompressedSize;
         public readonly FileType Type;
         public readonly string RelativePath; // Path of an embedded file, relative to the Bundle source-directory.
 
         public const char DirectorySeparatorChar = '/';
 
-        public FileEntry(FileType fileType, string relativePath, long offset, long size)
+        public FileEntry(FileType fileType, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
+            BundleMajorVersion = bundleMajorVersion;
             Type = fileType;
             RelativePath = relativePath.Replace('\\', DirectorySeparatorChar);
             Offset = offset;
             Size = size;
+            CompressedSize = compressedSize;
         }
 
         public void Write(BinaryWriter writer)
         {
             writer.Write(Offset);
             writer.Write(Size);
+            // compression is used only in version 6.0+
+            if (BundleMajorVersion >= 6)
+            {
+                writer.Write(CompressedSize);
+            }
             writer.Write((byte)Type);
             writer.Write(RelativePath);
         }
 
-        public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size}";
+        public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size} CompressedSz={CompressedSize}";
     }
 }

src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs

@@ -66,32 +66,26 @@ private enum HeaderFlags : ulong
         // with path-names so that the AppHost can use it in
         // extraction path.
         public readonly string BundleID;
-
-        public const uint CurrentMajorVersion = 2;
-        public readonly uint DesiredMajorVersion;
+        public readonly uint BundleMajorVersion;
         // The Minor version is currently unused, and is always zero
-        public const uint MinorVersion = 0;
-
-        public static string CurrentVersion => $"{CurrentMajorVersion}.{MinorVersion}";
-        public string DesiredVersion => $"{DesiredMajorVersion}.{MinorVersion}";
-
+        public const uint BundleMinorVersion = 0;
         private FileEntry DepsJsonEntry;
         private FileEntry RuntimeConfigJsonEntry;
         private HeaderFlags Flags;
-
         public List<FileEntry> Files;
+        public string BundleVersion => $"{BundleMajorVersion}.{BundleMinorVersion}";
 
-        public Manifest(uint desiredVersion, bool netcoreapp3CompatMode = false)
+        public Manifest(uint bundleMajorVersion, bool netcoreapp3CompatMode = false)
         {
-            DesiredMajorVersion = desiredVersion;
+            BundleMajorVersion = bundleMajorVersion;
             Files = new List<FileEntry>();
             BundleID = Path.GetRandomFileName();
-            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode: HeaderFlags.None;
+            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None;
         }
 
-        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size)
+        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
-            FileEntry entry = new FileEntry(type, relativePath, offset, size);
+            FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion);
             Files.Add(entry);
 
             switch (entry.Type)
@@ -118,12 +112,12 @@ public long Write(BinaryWriter writer)
             long startOffset = writer.BaseStream.Position;
 
             // Write the bundle header
-            writer.Write(DesiredMajorVersion);
-            writer.Write(MinorVersion);
+            writer.Write(BundleMajorVersion);
+            writer.Write(BundleMinorVersion);
             writer.Write(Files.Count);
             writer.Write(BundleID);
 
-            if (DesiredMajorVersion == 2)
+            if (BundleMajorVersion >= 2)
             {
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Offset : 0);
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Size : 0);

src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs

@@ -25,26 +25,31 @@ public class TargetInfo
         public readonly OSPlatform OS;
         public readonly Architecture Arch;
         public readonly Version FrameworkVersion;
-        public readonly uint BundleVersion;
+        public readonly uint BundleMajorVersion;
         public readonly BundleOptions DefaultOptions;
         public readonly int AssemblyAlignment;
 
         public TargetInfo(OSPlatform? os, Architecture? arch, Version targetFrameworkVersion)
         {
             OS = os ?? HostOS;
             Arch = arch ?? RuntimeInformation.OSArchitecture;
-            FrameworkVersion = targetFrameworkVersion ?? net50;
+            FrameworkVersion = targetFrameworkVersion ?? net60;
 
             Debug.Assert(IsLinux || IsOSX || IsWindows);
 
-            if (FrameworkVersion.CompareTo(net50) >= 0)
+            if (FrameworkVersion.CompareTo(net60) >= 0)
             {
-                BundleVersion = 2u;
+                BundleMajorVersion = 6u;
+                DefaultOptions = BundleOptions.None;
+            }
+            else if (FrameworkVersion.CompareTo(net50) >= 0)
+            {
+                BundleMajorVersion = 2u;
                 DefaultOptions = BundleOptions.None;
             }
             else if (FrameworkVersion.Major == 3 && (FrameworkVersion.Minor == 0 || FrameworkVersion.Minor == 1))
             {
-                BundleVersion = 1u;
+                BundleMajorVersion = 1u;
                 DefaultOptions = BundleOptions.BundleAllContent;
             }
             else
@@ -94,7 +99,7 @@ public override string ToString()
 
         // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup.
         // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration.
-        public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType;
+        public FileType TargetSpecificFileType(FileType fileType) => (BundleMajorVersion == 1) ? FileType.Unknown : fileType;
 
         // In .net core 3.x, bundle processing happens within the AppHost.
         // Therefore HostFxr and HostPolicy can be bundled within the single-file app.
@@ -105,6 +110,7 @@ public override string ToString()
         public bool ShouldExclude(string relativePath) =>
             (FrameworkVersion.Major != 3) && (relativePath.Equals(HostFxr) || relativePath.Equals(HostPolicy));
 
+        private readonly Version net60 = new Version(6, 0);
         private readonly Version net50 = new Version(5, 0);
         private string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib";
         private string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib";

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs

@@ -31,8 +31,8 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds()
             var hostName = BundleHelper.GetHostName(fixture);
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options: BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Verify expected files in the bundle directory
             var bundleDir = BundleHelper.GetBundleDir(fixture);
@@ -80,8 +80,7 @@ private void Bundle_Extraction_To_Relative_Path_Succeeds(string relativePath, Bu
                 return;
 
             var fixture = sharedTestState.TestFixture.Copy();
-            UseSingleFileSelfContainedHost(fixture);
-            var bundler = BundleHelper.BundleApp(fixture, out var singleFile, bundleOptions);
+            var bundler = BundleSelfContainedApp(fixture, out var singleFile, bundleOptions);
 
             // Run the bundled app (extract files to <path>)
             var cmd = Command.Create(singleFile);
@@ -110,8 +109,8 @@ private void Bundle_extraction_is_reused()
             var fixture = sharedTestState.TestFixture.Copy();
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);
@@ -160,13 +159,12 @@ private void Bundle_extraction_can_recover_missing_files()
             var appName = Path.GetFileNameWithoutExtension(hostName);
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);
 
-
             // Run the bunded app for the first time, and extract files to 
             // $DOTNET_BUNDLE_EXTRACT_BASE_DIR/<app>/bundle-id
             Command.Create(singleFile)

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs

@@ -44,10 +44,28 @@ public static string UseFrameworkDependentHost(TestProjectFixture testFixture)
         public static string BundleSelfContainedApp(
             TestProjectFixture testFixture,
             BundleOptions options = BundleOptions.None,
-            Version targetFrameworkVersion = null)
+            Version targetFrameworkVersion = null,
+            bool disableCompression = false)
+        {
+            string singleFile;
+            BundleSelfContainedApp(testFixture, out singleFile, options, targetFrameworkVersion);
+            return singleFile;
+        }
+
+        public static Bundler BundleSelfContainedApp(
+            TestProjectFixture testFixture,
+            out string singleFile,
+            BundleOptions options = BundleOptions.None,
+            Version targetFrameworkVersion = null,
+            bool disableCompression = false)
         {
             UseSingleFileSelfContainedHost(testFixture);
-            return BundleHelper.BundleApp(testFixture, options, targetFrameworkVersion);
+            if (targetFrameworkVersion == null || targetFrameworkVersion >= new Version(6, 0))
+            {
+                options |= BundleOptions.EnableCompression;
+            }
+
+            return BundleHelper.BundleApp(testFixture, out singleFile, options, targetFrameworkVersion);
         }
 
         public abstract class SharedTestStateBase