[API Proposal] Add APIs that needed for emitting symbols with reflection emit PersistedAssemblyBuilder

[buyaa-n]

Background and motivation

Now we have added PersistedAssemblyBuilder in NET 9.0, further we need to add PDB support. For this we need the APIs that used for adding symbol info with reflection emit APIs. The proposed APIs are quite similar the ones that now exist in .NET framework reflection emit implementation:

• In .NET framework AssemblyBuilder.DefineDynamicModule overloads had bool emitSymbolInfo parameter, we will not have that option, in order to populate PDB user should use MetadataBuilder GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder mappedFieldData, out MetadataBuilder pdbMetadata) method and use the pdbMetadata out parameter for producing PDB as desired. Further, based on the customer feedback we could add Save overload that sets some options and defaulting other options.
• We will not add ModuleBuilder.GetSymWriter() method that returns ISymbolWriter. It was used with native code to add debug info, it cannot work with portable PDB.

API Proposal

public abstract partial class ModuleBuilder : System.Reflection.Module
{
    // .NET framework version: ISymbolDocumentWriter DefineDocument (string url, Guid language, Guid languageVendor, Guid documentType)
+    public virtual ISymbolDocumentWriter DefineDocument(string url, Guid language = default) { }
}

 public abstract class ILGenerator
 {
    public abstract void BeginScope();
    public abstract void EndScope();
+   public virtual void MarkSequencePoint(ISymbolDocumentWriter document, int startLine, int startColumn, int endLine, int endColumn) { }
    public abstract void UsingNamespace(string usingNamespace);
 }
 
public abstract class LocalBuilder : LocalVariableInfo
{
    public override int LocalIndex { get; }
    public override Type LocalType { get; }
    // .NET framework version: SetLocalSymInfo(string name), SetLocalSymInfo(string name, int startOffset, int endOffset) methods
+   public virtual void SetLocalSymInfo(string name) { }
}

public sealed class PersistedAssemblyBuilder : AssemblyBuilder
{
    public MetadataBuilder GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder mappedFieldData) { }
+   public MetadataBuilder GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder mappedFieldData, out MetadataBuilder pdbMetadata) { }
}

The symbols metadata will be populating with PersistedAssemblyBuilder.GenerateMetadata(...) overload that has additional parameter out MetadataBuilder pdbMetadata. Further steps for producing portable PDB:

1. Create PortablePdbBuilder instance with the PDB metadata and type-system metadata produced from above method
2. Serialize the PortablePdbBuilder into Blob, write the Blob into a PDB file in case generating standalone PDB
3. Create DebugDirectoryBuilder instance and add a CodeViewEntry or EmbeddedPortablePdbEntry
4. Set the optional debugDirectoryBuilder argument when creating ManagedPEBuilder
5. Serialize the PEBuilder into a Blob, and write the Blob into a file or a stream

API usage:

static void Test ()
{
    PersistedAssemblyBuilder ab = new PersistedAssemblyBuilder(new AssemblyName("MyAssembly"), typeof(object).Assembly);
    ModuleBuilder mb = ab.DefineDynamicModule("MyModule");
    TypeBuilder tb = mb.DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class);
    MethodBuilder mb1 = tb.DefineMethod("SumMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(int)]);
    ISymbolDocumentWriter srcDoc = mb.DefineDocument("MySourceFile.cs", SymLanguageType.CSharp);
    ILGenerator il = mb1.GetILGenerator();
    LocalBuilder local = il.DeclareLocal(typeof(int));
    local.SetLocalSymInfo("myLocal");
    il.MarkSequencePoint(srcDoc, 7, 0, 7, 11);
    ...
    il.Emit(OpCodes.Ret);

    MethodBuilder entryPoint = tb.DefineMethod("Main", MethodAttributes.HideBySig | MethodAttributes.Public | MethodAttributes.Static);
    ILGenerator il2 = entryPoint.GetILGenerator();
    il2.MarkSequencePoint(srcDoc, 12, 0, 12, 38);
    ...
    tb.CreateType();

    MetadataBuilder metadataBuilder = ab.GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder _, out MetadataBuilder pdbMetadata);
    MethodDefinitionHandle entryPointHandle = MetadataTokens.MethodDefinitionHandle(entryPoint.MetadataToken);
    DebugDirectoryBuilder debugDirectoryBuilder = GeneratePDB(pdbMetadata, metadataBuilder.GetRowCounts(), entryPointHandle);

    ManagedPEBuilder peBuilder = new ManagedPEBuilder(
                    header: new PEHeaderBuilder(imageCharacteristics: Characteristics.ExecutableImage, subsystem: Subsystem.WindowsCui),
                    metadataRootBuilder: new MetadataRootBuilder(metadataBuilder),
                    ilStream: ilStream,
                    debugDirectoryBuilder: debugDirectoryBuilder,
                    entryPoint: entryPointHandle);

    BlobBuilder peBlob = new BlobBuilder();
    peBuilder.Serialize(peBlob);

    using var fileStream = new FileStream("MyAssembly.exe", FileMode.Create, FileAccess.Write);
    peBlob.WriteContentTo(fileStream);
}

private static DebugDirectoryBuilder GeneratePDB(MetadataBuilder pdbMetadata, ImmutableArray<int> rowCounts, MethodDefinitionHandle entryPointHandle)
{
    BlobBuilder portablePdbBlob = new BlobBuilder();
    PortablePdbBuilder pdbBuilder = new PortablePdbBuilder(pdbMetadata, rowCounts, entryPointHandle);
    BlobContentId pdbContentId = pdbBuilder.Serialize(portablePdbBlob);
    // In case saving PDB to a file
    using FileStream fileStream = new FileStream("MyAssemblyEmbeddedSource.pdb", FileMode.Create, FileAccess.Write);
    portablePdbBlob.WriteContentTo(fileStream);

    DebugDirectoryBuilder debugDirectoryBuilder = new DebugDirectoryBuilder();
    debugDirectoryBuilder.AddCodeViewEntry("MyAssemblyEmbeddedSource.pdb", pdbContentId, pdbBuilder.FormatVersion);
    // In case embedded in PE:
    // debugDirectoryBuilder.AddEmbeddedPortablePdbEntry(portablePdbBlob, pdbBuilder.FormatVersion);
    return debugDirectoryBuilder;
}

Furter user could add CustomDebugInformation by calling the AddCustomDebugInformation method on pdbMetadata to add source embedding and source indexing etc.

private static void EmbedSource(MetadataBuilder pdbMetadata)
{
    byte[] sourceBytes = File.ReadAllBytes("MySourceFile2.cs");
    BlobBuilder sourceBlob = new BlobBuilder();
    sourceBlob.WriteBytes(sourceBytes);
    pdbMetadata.AddCustomDebugInformation(MetadataTokens.DocumentHandle(1),
        pdbMetadata.GetOrAddGuid(new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE")), pdbMetadata.GetOrAddBlob(sourceBlob));
}

CC @AaronRobinsonMSFT @ericstj @jeffhandley @jkotas @steveharter
Contributes to #92975

[dotnet-policy-service]

Tagging subscribers to this area: @dotnet/area-system-reflection-metadata
See info in area-owners.md if you want to be subscribed.

[bartonjs]

Video

• We added ModuleBuilder.DefineDocument taking 4 parameters for better signature/module compatibility with netfx
• Ref.Emit has already been using the Template Method Pattern, avoiding "public virtual", and should continue to do so.
• PersistedAssemblyBuilder.GenerateMetadata, we renamed the new parameter pdbBuilder

namespace System.Reflection.Emit;

public abstract partial class ModuleBuilder : System.Reflection.Module
{
    [EditorBrowsable(Never)]
    public ISymbolDocumentWriter DefineDocument (string url, Guid language, Guid languageVendor, Guid documentType) => DefineDocument(url, language);

    public ISymbolDocumentWriter DefineDocument(string url, Guid language = default);
    protected virtual ISymbolDocumentWriter DefineDocumentCore(string url, Guid language = default);
}

 public abstract partial class ILGenerator
 {
    public void MarkSequencePoint(ISymbolDocumentWriter document, int startLine, int startColumn, int endLine, int endColumn) { }
    protected virtual void MarkSequencePointCore(ISymbolDocumentWriter document, int startLine, int startColumn, int endLine, int endColumn) { }
 }
 
public abstract partial class LocalBuilder : LocalVariableInfo
{
    public void SetLocalSymInfo(string name);
    protected virtual void SetLocalSymInfoCore(string name) { }
}

public sealed class PersistedAssemblyBuilder : AssemblyBuilder
{
    public MetadataBuilder GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder mappedFieldData, out MetadataBuilder pdbBuilder) { }
}