Add UnmanagedCallConvAttribute

src/coreclr/inc/corcompile.h

@@ -1713,6 +1713,9 @@ class ICorCompileInfo
     // to 1 on the clone. The buffer has to be large enough to hold the stub object and the code
     virtual HRESULT GetStubClone(void *pStub, BYTE *pBuffer, DWORD dwBufferSize) = 0;
 
+    // true if the method has [UnmanagedCallConvAttribute]
+    virtual BOOL IsUnmanagedCallConvMethod(CORINFO_METHOD_HANDLE handle) = 0;
+
     // true if the method has [UnmanagedCallersOnlyAttribute]
     virtual BOOL IsUnmanagedCallersOnlyMethod(CORINFO_METHOD_HANDLE handle) = 0;
 

src/coreclr/tools/Common/JitInterface/CallConvHelper.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Reflection.Metadata;
+
+using Internal.TypeSystem;
+
+namespace Internal.JitInterface
+{
+    internal static unsafe class CallConvHelper
+    {
+        internal static IEnumerable<DefType> EnumerateCallConvsFromAttribute(CustomAttributeValue<TypeDesc> attributeWithCallConvsArray)
+        {
+            ImmutableArray<CustomAttributeTypedArgument<TypeDesc>> callConvArray = default;
+            foreach (var arg in attributeWithCallConvsArray.NamedArguments)
+            {
+                if (arg.Name == "CallConvs")
+                {
+                    callConvArray = (ImmutableArray<CustomAttributeTypedArgument<TypeDesc>>)arg.Value;
+                }
+            }
+
+            // No calling convention was specified in the attribute
+            if (callConvArray.IsDefault)
+                yield break;
+
+            foreach (CustomAttributeTypedArgument<TypeDesc> type in callConvArray)
+            {
+                if (!(type.Value is DefType defType))
+                    continue;
+
+                if (defType.Namespace != "System.Runtime.CompilerServices")
+                    continue;
+
+                yield return defType;
+            }
+        }
+    }
+}

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -637,41 +637,27 @@ private void Get_CORINFO_SIG_INFO(MethodDesc method, CORINFO_SIG_INFO* sig, bool
             }
         }
 
-        private CorInfoCallConvExtension GetUnmanagedCallingConventionFromAttribute(CustomAttributeValue<TypeDesc> unmanagedCallersOnlyAttribute)
+        private CorInfoCallConvExtension GetUnmanagedCallingConventionFromAttribute(CustomAttributeValue<TypeDesc> attributeWithCallConvsArray, out bool suppressGCTransition)
         {
+            suppressGCTransition = false;
             CorInfoCallConvExtension callConv = (CorInfoCallConvExtension)PlatformDefaultUnmanagedCallingConvention();
 
-            ImmutableArray<CustomAttributeTypedArgument<TypeDesc>> callConvArray = default;
-            foreach (var arg in unmanagedCallersOnlyAttribute.NamedArguments)
-            {
-                if (arg.Name == "CallConvs")
-                {
-                    callConvArray = (ImmutableArray<CustomAttributeTypedArgument<TypeDesc>>)arg.Value;
-                }
-            }
-
-            // No calling convention was specified in the attribute, so return the default.
-            if (callConvArray.IsDefault)
-            {
-                return callConv;
-            }
-
             bool found = false;
             bool memberFunctionVariant = false;
-            foreach (CustomAttributeTypedArgument<TypeDesc> type in callConvArray)
+            foreach (DefType defType in CallConvHelper.EnumerateCallConvsFromAttribute(attributeWithCallConvsArray))
             {
-                if (!(type.Value is DefType defType))
-                    continue;
-
-                if (defType.Namespace != "System.Runtime.CompilerServices")
-                    continue;
-
                 if (defType.Name == "CallConvMemberFunction")
                 {
                     memberFunctionVariant = true;
                     continue;
                 }
 
+                if (defType.Name == "CallConvSuppressGCTransition")
+                {
+                    suppressGCTransition = true;
+                    continue;
+                }
+
                 CorInfoCallConvExtension? callConvLocal = GetCallingConventionForCallConvType(defType);
 
                 if (callConvLocal.HasValue)
@@ -1258,11 +1244,28 @@ private CorInfoCallConvExtension GetUnmanagedCallConv(MethodDesc methodDesc, out
             {
                 if (methodDesc.IsPInvoke)
                 {
-                    suppressGCTransition = methodDesc.IsSuppressGCTransition();
+                    suppressGCTransition = methodDesc.HasSuppressGCTransitionAttribute();
                     MethodSignatureFlags unmanagedCallConv = methodDesc.GetPInvokeMethodMetadata().Flags.UnmanagedCallingConvention;
 
                     if (unmanagedCallConv == MethodSignatureFlags.None)
+                    {
+                        MethodDesc methodDescLocal = methodDesc;
+                        if (methodDesc is IL.Stubs.PInvokeTargetNativeMethod rawPInvoke)
+                        {
+                            methodDescLocal = rawPInvoke.Target;
+                        }
+
+                        CustomAttributeValue<TypeDesc>? unmanagedCallConvAttribute = ((EcmaMethod)methodDescLocal).GetDecodedCustomAttribute("System.Runtime.InteropServices", "UnmanagedCallConvAttribute");
+                        if (unmanagedCallConvAttribute != null)
+                        {
+                            bool suppressGCTransitionLocal;
+                            CorInfoCallConvExtension callConvFromAttribute = GetUnmanagedCallingConventionFromAttribute(unmanagedCallConvAttribute.Value, out suppressGCTransitionLocal);
+                            suppressGCTransition |= suppressGCTransitionLocal;
+                            return callConvFromAttribute;
+                        }
+
                         unmanagedCallConv = PlatformDefaultUnmanagedCallingConvention();
+                    }
 
                     // Verify that it is safe to convert MethodSignatureFlags.UnmanagedCallingConvention to CorInfoCallConvExtension via a simple cast
                     Debug.Assert((int)CorInfoCallConvExtension.C == (int)MethodSignatureFlags.UnmanagedCallingConventionCdecl);
@@ -1275,7 +1278,7 @@ private CorInfoCallConvExtension GetUnmanagedCallConv(MethodDesc methodDesc, out
                 {
                     Debug.Assert(methodDesc.IsUnmanagedCallersOnly);
                     CustomAttributeValue<TypeDesc> unmanagedCallersOnlyAttribute = ((EcmaMethod)methodDesc).GetDecodedCustomAttribute("System.Runtime.InteropServices", "UnmanagedCallersOnlyAttribute").Value;
-                    return GetUnmanagedCallingConventionFromAttribute(unmanagedCallersOnlyAttribute);
+                    return GetUnmanagedCallingConventionFromAttribute(unmanagedCallersOnlyAttribute, out _);
                 }
             }
             return GetUnmanagedCallConv(methodDesc.Signature, out suppressGCTransition);

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/MethodExtensions.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using Internal.TypeSystem;
+using Internal.TypeSystem.Ecma;
 
 using Debug = System.Diagnostics.Debug;
 
@@ -21,18 +22,53 @@ public static bool IsRawPInvoke(this MethodDesc method)
         }
 
         /// <summary>
-        /// Gets a value indicating whether GC transition should be suppressed on the given p/invoke.
+        /// Gets a value indicating whether the method has the SuppressGCTransition attribute
         /// </summary>
-        public static bool IsSuppressGCTransition(this MethodDesc method)
+        public static bool HasSuppressGCTransitionAttribute(this MethodDesc method)
         {
             Debug.Assert(method.IsPInvoke);
 
             if (method is Internal.IL.Stubs.PInvokeTargetNativeMethod rawPinvoke)
                 method = rawPinvoke.Target;
 
+            // Check SuppressGCTransition attribute
             return method.HasCustomAttribute("System.Runtime.InteropServices", "SuppressGCTransitionAttribute");
         }
 
+        /// <summary>
+        /// Gets a value indicating whether GC transition should be suppressed on the given p/invoke.
+        /// </summary>
+        public static bool IsSuppressGCTransition(this MethodDesc method)
+        {
+            Debug.Assert(method.IsPInvoke);
+
+            // Check SuppressGCTransition attribute
+            if (method.HasSuppressGCTransitionAttribute())
+                return true;
+
+            MethodSignatureFlags unmanagedCallConv = method.GetPInvokeMethodMetadata().Flags.UnmanagedCallingConvention;
+            if (unmanagedCallConv != MethodSignatureFlags.None)
+                return false;
+
+            if (!(method is Internal.TypeSystem.Ecma.EcmaMethod ecmaMethod))
+                return false;
+
+            // Check UnmanagedCallConv attribute
+            System.Reflection.Metadata.CustomAttributeValue<TypeDesc>? unmanagedCallConvAttribute = ecmaMethod.GetDecodedCustomAttribute("System.Runtime.InteropServices", "UnmanagedCallConvAttribute");
+            if (unmanagedCallConvAttribute == null)
+                return false;
+
+            foreach (DefType defType in Internal.JitInterface.CallConvHelper.EnumerateCallConvsFromAttribute(unmanagedCallConvAttribute.Value))
+            {
+                if (defType.Name == "CallConvSuppressGCTransition")
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
         /// <summary>
         /// Return true when the method is marked as non-versionable. Non-versionable methods
         /// may be freely inlined into ReadyToRun images even when they don't reside in the

src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj

@@ -248,6 +248,9 @@
     <Compile Include="..\..\Common\JitInterface\TypeString.cs">
       <Link>JitInterface\TypeString.cs</Link>
     </Compile>
+    <Compile Include="..\..\Common\JitInterface\CallConvHelper.cs">
+      <Link>JitInterface\CallConvHelper.cs</Link>
+    </Compile>
     <Compile Include="..\..\Common\JitInterface\CorInfoBase.cs">
       <Link>JitInterface\CorInfoBase.cs</Link>
     </Compile>

src/coreclr/vm/callconvbuilder.cpp

@@ -387,32 +387,86 @@ HRESULT CallConv::TryGetUnmanagedCallingConventionFromModOpt(
 #ifndef CROSSGEN_COMPILE
 namespace
 {
-    bool TryGetCallingConventionFromTypeArray(_In_ CaValue* arrayOfTypes, _Out_ CorInfoCallConvExtension* callConv)
+    bool TryGetCallingConventionFromTypeArray(_In_ CaValue* arrayOfTypes, _Inout_ CallConvBuilder* builder)
     {
         CONTRACTL
         {
             STANDARD_VM_CHECK;
             PRECONDITION(arrayOfTypes != NULL);
-            PRECONDITION(callConv != NULL);
+            PRECONDITION(builder != NULL);
         }
         CONTRACTL_END
 
-        CallConvBuilder builder;
         for (ULONG i = 0; i < arrayOfTypes->arr.length; i++)
         {
             CaValue& typeNameValue = arrayOfTypes->arr[i];
-            if (!builder.AddFullyQualifiedTypeName(typeNameValue.str.cbStr, typeNameValue.str.pStr))
+            if (!builder->AddFullyQualifiedTypeName(typeNameValue.str.cbStr, typeNameValue.str.pStr))
             {
                 // We found a second base calling convention.
                 return false;
             }
         }
 
-        *callConv = builder.GetCurrentCallConv();
         return true;
     }
 }
 
+HRESULT CallConv::TryGetCallingConventionFromUnmanagedCallConv(
+    _In_ MethodDesc* pMD,
+    _Inout_ CallConvBuilder* builder,
+    _Out_opt_ UINT* errorResID)
+{
+    CONTRACTL
+    {
+        STANDARD_VM_CHECK;
+        PRECONDITION(pMD != NULL);
+        PRECONDITION(builder != NULL);
+    }
+    CONTRACTL_END;
+
+    BYTE *pData = NULL;
+    LONG cData = 0;
+
+    // System.Runtime.InteropServices.UnmanagedCallConvAttribute
+    HRESULT hr = pMD->GetCustomAttribute(WellKnownAttribute::UnmanagedCallConv, (const VOID **)(&pData), (ULONG *)&cData);
+    if (hr != S_OK) // GetCustomAttribute returns S_FALSE if the method does not have the attribute
+        return hr;
+
+    _ASSERTE(cData > 0);
+    CustomAttributeParser ca(pData, cData);
+
+    // CallConvs named argument
+    CaTypeCtor callConvsType(SERIALIZATION_TYPE_SZARRAY, SERIALIZATION_TYPE_TYPE, SERIALIZATION_TYPE_UNDEFINED, NULL, 0);
+    CaNamedArg callConvsArg;
+    callConvsArg.Init("CallConvs", SERIALIZATION_TYPE_SZARRAY, callConvsType);
+
+    InlineFactory<SArray<CaValue>, 4> caValueArrayFactory;
+    DomainAssembly* domainAssembly = pMD->GetLoaderModule()->GetDomainAssembly();
+    IfFailThrow(Attribute::ParseAttributeArgumentValues(
+        pData,
+        cData,
+        &caValueArrayFactory,
+        NULL,
+        0,
+        &callConvsArg,
+        1,
+        domainAssembly));
+
+    // Value isn't defined
+    if (callConvsArg.val.type.tag == SERIALIZATION_TYPE_UNDEFINED)
+        return S_FALSE;
+
+    if (!TryGetCallingConventionFromTypeArray(&callConvsArg.val, builder))
+    {
+        if (errorResID != NULL)
+            *errorResID = IDS_EE_MULTIPLE_CALLCONV_UNSUPPORTED;
+
+        return COR_E_INVALIDPROGRAM;
+    }
+
+    return S_OK;
+}
+
 bool CallConv::TryGetCallingConventionFromUnmanagedCallersOnly(_In_ MethodDesc* pMD, _Out_ CorInfoCallConvExtension* pCallConv)
 {
     STANDARD_VM_CONTRACT;
@@ -484,12 +538,14 @@ bool CallConv::TryGetCallingConventionFromUnmanagedCallersOnly(_In_ MethodDesc*
     }
     else
     {
-        if (!TryGetCallingConventionFromTypeArray(&namedArgs[0].val, &callConvLocal))
+        CallConvBuilder builder;
+        if (!TryGetCallingConventionFromTypeArray(&namedArgs[0].val, &builder))
         {
             // We found a second base calling convention.
             return false;
         }
 
+        callConvLocal = builder.GetCurrentCallConv();
         if (callConvLocal == CallConvBuilder::UnsetValue)
         {
             callConvLocal = CallConv::GetDefaultUnmanagedCallingConvention();

src/coreclr/vm/callconvbuilder.hpp

@@ -80,6 +80,20 @@ namespace CallConv
         _Inout_ CallConvBuilder *builder,
         _Out_ UINT *errorResID);
 
+    //-------------------------------------------------------------------------
+    // Gets the calling convention from the UnmanagedCallConv attribute
+    //
+    // Returns:
+    //   S_OK - No errors
+    //   S_FALSE - UnmanagedCallConv or UnmanagedCallConv.CallConvs not specified
+    //   COR_E_INVALIDPROGRAM - Program is considered invalid (more
+    //                          than one calling convention specified)
+    //-------------------------------------------------------------------------
+    HRESULT TryGetCallingConventionFromUnmanagedCallConv(
+        _In_ MethodDesc* pMD,
+        _Inout_ CallConvBuilder* builder,
+        _Out_opt_ UINT* errorResID);
+
     //-------------------------------------------------------------------------
     // Gets the unmanaged calling convention from the UnmanagedCallersOnly attribute.
     //

src/coreclr/vm/compile.cpp

@@ -968,6 +968,14 @@ BOOL CEEPreloader::DoesMethodNeedRestoringBeforePrestubIsRun(
     return FALSE;
 }
 
+BOOL CEECompileInfo::IsUnmanagedCallConvMethod(CORINFO_METHOD_HANDLE handle)
+{
+    WRAPPER_NO_CONTRACT;
+
+    MethodDesc * pMethod = GetMethod(handle);
+    return pMethod->HasUnmanagedCallConvAttribute();
+}
+
 BOOL CEECompileInfo::IsUnmanagedCallersOnlyMethod(CORINFO_METHOD_HANDLE handle)
 {
     WRAPPER_NO_CONTRACT;

src/coreclr/vm/compile.h

@@ -280,6 +280,8 @@ class CEECompileInfo : public ICorCompileInfo
     BOOL IsEmptyString(mdString token,
                        CORINFO_MODULE_HANDLE module);
 
+    BOOL IsUnmanagedCallConvMethod(CORINFO_METHOD_HANDLE handle);
+
     BOOL IsUnmanagedCallersOnlyMethod(CORINFO_METHOD_HANDLE handle);
 
     BOOL IsCachingOfInliningHintsEnabled()