diff --git a/src/coreclr/src/jit/hwintrinsic.cpp b/src/coreclr/src/jit/hwintrinsic.cpp
index dcff6dcaaefcce..0ce3ec25b69f5c 100644
--- a/src/coreclr/src/jit/hwintrinsic.cpp
+++ b/src/coreclr/src/jit/hwintrinsic.cpp
@@ -91,6 +91,10 @@ var_types Compiler::getBaseTypeFromArgIfNeeded(NamedIntrinsic       intrinsic,
 
 CORINFO_CLASS_HANDLE Compiler::gtGetStructHandleForHWSIMD(var_types simdType, var_types simdBaseType)
 {
+    if (m_simdHandleCache == nullptr)
+    {
+        return NO_CLASS_HANDLE;
+    }
     if (simdType == TYP_SIMD16)
     {
         switch (simdBaseType)
diff --git a/src/coreclr/src/jit/optcse.cpp b/src/coreclr/src/jit/optcse.cpp
index ddf7d8af1d6a68..b7ece7ced07374 100644
--- a/src/coreclr/src/jit/optcse.cpp
+++ b/src/coreclr/src/jit/optcse.cpp
@@ -2638,17 +2638,15 @@ class CSE_Heuristic
             // If all occurances were in GT_IND nodes it could still be NO_CLASS_HANDLE
             //
             CORINFO_CLASS_HANDLE structHnd = successfulCandidate->CseDsc()->csdStructHnd;
-            assert((structHnd != NO_CLASS_HANDLE) || (cseLclVarTyp != TYP_STRUCT));
-            if (structHnd != NO_CLASS_HANDLE)
-            {
-                m_pCompiler->lvaSetStruct(cseLclVarNum, structHnd, false);
-            }
-#ifdef FEATURE_SIMD
-            else if (varTypeIsSIMD(cseLclVarTyp))
+            if (structHnd == NO_CLASS_HANDLE)
             {
-                m_pCompiler->lvaGetDesc(cseLclVarNum)->lvSIMDType = true;
+                assert(varTypeIsSIMD(cseLclVarTyp));
+                // We are not setting it for `SIMD* indir` during the first path
+                // because it is not precise, see `optValnumCSE_Index`.
+                structHnd = m_pCompiler->gtGetStructHandle(successfulCandidate->CseDsc()->csdTree);
             }
-#endif // FEATURE_SIMD
+            assert(structHnd != NO_CLASS_HANDLE);
+            m_pCompiler->lvaSetStruct(cseLclVarNum, structHnd, false);
         }
         m_pCompiler->lvaTable[cseLclVarNum].lvType  = cseLclVarTyp;
         m_pCompiler->lvaTable[cseLclVarNum].lvIsCSE = true;
@@ -3214,9 +3212,9 @@ bool Compiler::optIsCSEcandidate(GenTree* tree)
         return false;
     }
 
-    // If this is a struct type, we can only consider it for CSE-ing if we can get at
-    // its handle, so that we can create a temp.
-    if ((type == TYP_STRUCT) && (gtGetStructHandleIfPresent(tree) == NO_CLASS_HANDLE))
+    // If this is a struct type (including SIMD*), we can only consider it for CSE-ing
+    // if we can get its handle, so that we can create a temp.
+    if (varTypeIsStruct(type) && (gtGetStructHandleIfPresent(tree) == NO_CLASS_HANDLE))
     {
         return false;
     }
diff --git a/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs b/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs
index e51357e61fefb4..fc82fb83850f01 100644
--- a/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs
+++ b/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs
@@ -500,7 +500,6 @@ public void Matrix4x4CreateRotationYCenterTest()
 
         // A test for CreateFromAxisAngle(Vector3f,float)
         [Fact]
-        [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36586", RuntimeTestModes.JitStress)]
         public void Matrix4x4CreateFromAxisAngleTest()
         {
             float radians = MathHelper.ToRadians(-30.0f);