From 105015171996cb4662d90bcc044af82ec80de302 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 14 Jul 2026 16:11:08 -0700 Subject: [PATCH 01/26] Add PackedSimd load/store mappings to table --- src/coreclr/jit/codegenwasm.cpp | 11 ++--------- src/coreclr/jit/hwintrinsiclistwasm.h | 10 +++++----- src/coreclr/jit/hwintrinsicwasm.cpp | 8 -------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 1c00c386259e20..85f4ff65d7f52d 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2747,15 +2747,8 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) } else // A normal store, not a WriteBarrier store { - var_types type = tree->TypeGet(); - if (type == TYP_SIMD16) - { - // Storing a SIMD16 value emits v128.store, but the data operand is not - // materialized as a v128 (it comes through as an i32), producing an invalid - // module. Bail until SIMD16 store is properly supported. - NYI_WASM_SIMD("SIMD16 store indirect"); - } - instruction ins = ins_Store(type); + var_types type = tree->TypeGet(); + instruction ins = ins_Store(type); // TODO-WASM: Memory barriers diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index 3f48197b9d1f53..9892035f9b7c90 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -44,11 +44,11 @@ HARDWARE_INTRINSIC(PackedSimd, Divide, HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) @@ -72,7 +72,7 @@ HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/hwintrinsicwasm.cpp b/src/coreclr/jit/hwintrinsicwasm.cpp index 9e5fe0b4ec45f7..4db20a60d05a83 100644 --- a/src/coreclr/jit/hwintrinsicwasm.cpp +++ b/src/coreclr/jit/hwintrinsicwasm.cpp @@ -170,14 +170,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_PackedSimd_LoadScalarVector128: - case NI_PackedSimd_LoadScalarAndSplatVector128: - case NI_PackedSimd_LoadScalarAndInsert: - case NI_PackedSimd_LoadWideningVector128: - { - break; - } - case NI_PackedSimd_Store: { assert(sig->numArgs == 2); From 45f618eeba2039c018a5dd5c3e72285c5864aa17 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 15 Jul 2026 09:54:18 -0700 Subject: [PATCH 02/26] Fix OperIsMemory{Load,Store} for PackedSimd operations --- src/coreclr/jit/gentree.cpp | 8 ++++---- src/coreclr/jit/hwintrinsiclistwasm.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index bbec675a6f07a1..4bb94a8b30ddf9 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30252,7 +30252,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -30442,7 +30442,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { @@ -30509,7 +30509,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -30582,7 +30582,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index 9892035f9b7c90..f2493480c30047 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -45,7 +45,7 @@ HARDWARE_INTRINSIC(PackedSimd, Dot, HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat, INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) From 7fd077ac967b5109f7b1b99800a42cd1c20a39cf Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 15 Jul 2026 13:27:14 -0700 Subject: [PATCH 03/26] Implement SIMD element-wise load/store codegen --- src/coreclr/jit/hwintrinsic.h | 13 +++++++-- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 32 +++++++++++++++++++++- src/coreclr/jit/hwintrinsiclistwasm.h | 8 +++--- src/coreclr/jit/lowerwasm.cpp | 32 ++++++++++++---------- 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 0b7e966a5e4bf0..b98d83c6c4a289 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -264,6 +264,8 @@ enum HWIntrinsicFlag : uint64_t // The intrinsic supports some sort of containment analysis HW_Flag_SupportsContainment = 0x400, HW_Flag_ReturnsPerElementMask = 0x800, + // The intrinsic has a required immediate operand + HW_Flag_HasImmediateOperand = 0x1000, #else #error Unsupported platform #endif @@ -1003,10 +1005,10 @@ struct HWIntrinsicInfo static bool HasImmediateOperand(NamedIntrinsic id) { -#if defined(TARGET_ARM64) +#if defined(TARGET_ARM64) || defined(TARGET_WASM) const HWIntrinsicFlag flags = lookupFlags(id); return ((flags & HW_Flag_HasImmediateOperand) != 0); -#elif defined(TARGET_XARCH) || defined(TARGET_WASM) +#elif defined(TARGET_XARCH) return lookupCategory(id) == HW_Category_IMM; #else return false; @@ -1472,7 +1474,12 @@ struct HWIntrinsic final inline bool needsJumpTableFallback() const { - return !m_node->GetImmOp()->IsCnsIntOrI(); + if (HWIntrinsicInfo::HasImmediateOperand(id)) + { + return !m_node->GetImmOp()->IsCnsIntOrI(); + } + + return false; } uint8_t GetImmediateLaneOperand() const diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index d9d47e7914db7f..39ad759d018829 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -55,6 +55,25 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; } + case HW_Category_MemoryStore: + case HW_Category_MemoryLoad: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + if (info.needsJumpTableFallback()) + { + genHWIntrinsicJumpTableFallback(node, info); + } + else if (HWIntrinsicInfo::HasImmediateOperand(info.id)) + { + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, info.GetImmediateLaneOperand()); + } + else + { + GetEmitter()->emitIns_I(ins, elemSize, 0); + } + + break; + } default: { NYI_WASM_SIMD("CodeGen::genHWIntrinsic: Unsupported category for table-driven intrinsic"); @@ -111,7 +130,11 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin int simdSize = node->GetSimdSize(); instruction const ins = HWIntrinsicInfo::lookupIns(info.id, info.baseType, m_compiler); int immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(info.id, simdSize, info.baseType); - WasmValueType resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + WasmValueType resultType = WasmValueType::Invalid; + if (!node->TypeIs(TYP_VOID)) + { + resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + } GenTree* immOp = node->GetImmOp(); regNumber immReg = GetMultiUseOperandReg(immOp); @@ -177,6 +200,13 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin GetEmitter()->emitIns_Lane(ins, static_cast(i)); break; } + case HW_Category_MemoryLoad: + case HW_Category_MemoryStore: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, static_cast(i)); + break; + } default: { NYI_WASM_SIMD( diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index f2493480c30047..1c5385f7a9522a 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -42,9 +42,9 @@ HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat, INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) @@ -61,7 +61,7 @@ HARDWARE_INTRINSIC(PackedSimd, Or, HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) @@ -72,7 +72,7 @@ HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index da8e304283aed5..5f611e7da9e2da 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -828,6 +828,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsic = node->GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); + bool hasImmOp = HWIntrinsicInfo::HasImmediateOperand(intrinsic); switch (intrinsic) { @@ -868,11 +869,22 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_PackedSimd_ExtractScalar: case NI_PackedSimd_ReplaceScalar: + case NI_PackedSimd_LoadScalarAndInsert: + case NI_PackedSimd_StoreSelectedScalar: { - assert(category == HW_Category_IMM); + assert(hasImmOp); return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_LoadScalarAndSplatVector128: + case NI_PackedSimd_LoadScalarVector128: + case NI_PackedSimd_LoadWideningVector128: + { + // These intrinsics don't a required immediate operand + assert(!hasImmOp); + break; + } + default: { assert(category == HW_Category_SIMD); @@ -1210,21 +1222,13 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { - HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->GetHWIntrinsicId()); - switch (category) + NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId)) { - case HWIntrinsicCategory::HW_Category_IMM: + GenTree* immOp = node->GetImmOp(); + if (immOp->IsCnsIntOrI()) { - GenTree* immOp = node->GetImmOp(); - if (immOp->IsCnsIntOrI()) - { - MakeSrcContained(node, immOp); - } - break; - } - default: - { - break; + MakeSrcContained(node, immOp); } } } From f9504c5450b4cec032fad06fb98682845f2cbf72 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 16 Jul 2026 17:56:34 -0700 Subject: [PATCH 04/26] Properly lower PackedSimd.Shuffle with non-constant operands --- src/coreclr/jit/hwintrinsic.h | 13 +++ src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 13 ++- src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerwasm.cpp | 93 +++++++++++++++++++++- 4 files changed, 114 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index b98d83c6c4a289..7ba21d056f02a1 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1331,6 +1331,12 @@ struct HWIntrinsicInfo *imm1Pos = 3; break; } + case NI_PackedSimd_Shuffle: + { + // (v128, v128, lane_imm) + *imm1Pos = 3; + break; + } default: { unreached(); @@ -1495,6 +1501,13 @@ struct HWIntrinsic final return static_cast(lane); } + simd16_t GetImmediateVecOperand() const + { + GenTree* immOp = m_node->GetImmOp(); + assert(immOp->IsCnsVec()); + return immOp->AsVecCon()->gtSimdVal; + } + NamedIntrinsic id; HWIntrinsicCategory category; GenTree* op1; diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 39ad759d018829..7c942ec3df93b0 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -26,9 +26,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { - // emitIns_Lane - // emitIns_Memarg_Lane - const HWIntrinsic info(node); genConsumeMultiOpOperands(node); @@ -40,7 +37,15 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { case HW_Category_SIMD: { - GetEmitter()->emitIns(ins); + if (info.id == NI_PackedSimd_Shuffle) + { + simd16_t shuffleMask = info.GetImmediateVecOperand(); + GetEmitter()->emitIns_V128Imm(ins, shuffleMask.u8); + } + else + { + GetEmitter()->emitIns(ins); + } break; } case HW_Category_IMM: diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 2926e1c2477562..8f899884975899 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -523,6 +523,7 @@ class Lowering final : public Phase #elif defined(TARGET_WASM) GenTree* LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node); + GenTree* LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node); #endif // !TARGET_XARCH && !TARGET_ARM64 GenTree* InsertNewSimdCreateScalarUnsafeNode(var_types type, GenTree* op1, diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 5f611e7da9e2da..ef2a01bbd12767 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -876,6 +876,11 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_Shuffle: + { + return LowerHWIntrinsicNativeShuffle(node); + } + case NI_PackedSimd_LoadScalarAndSplatVector128: case NI_PackedSimd_LoadScalarVector128: case NI_PackedSimd_LoadWideningVector128: @@ -1214,6 +1219,87 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) return LowerNode(node); } +GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) +{ + assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + GenTree* shuffleMask = node->Op(3); + var_types resultType = node->TypeGet(); + + // No extra work to do if the shuffle is a constant vector, it can be contained as an immediate and emitted. + if (shuffleMask->IsCnsVec()) + { + ContainCheckHWIntrinsic(node); + return node->gtNext; + } + + // If the shuffle mask is not a constant vector, we will need to rewrite the shuffle into two swizzles: + // 1 to handle elements from the first vector, and 1 to handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. + // We will be constructing IR like the following: + // /--* op1 simd + // +--* originalMask simd + // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> + // /--* originalMask simd + // +--* upperBnd simd + // upperMask = * HWINRINSIC simd byte PackedSimd.Subtract + // /--* op2 simd + // +--* upperMask simd + // tmp2 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // /--* tmp1 simd + // +--* tmp2 simd + // res = * HWINTRINSIC simd byte PackedSimd.Or + + + // Shuffle mask will be used twice, replace with a local + LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); + unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); + + // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a vector with the elements from the first vector in the correct order, and zero's for all elements which correspond to the second vector. + GenTree* swizzle1 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle1); + LowerNode(swizzle1); + + // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading to a zero in the result of the swizzle. + GenTreeVecCon* upperBound = m_compiler->gtNewVconNode(shuffleMask->TypeGet()); + upperBound->EvaluateBroadcastInPlace(TYP_BYTE, static_cast(16)); + BlockRange().InsertBefore(node, upperBound); + LowerNode(upperBound); + + // Re-load the original shuffle mask + GenTreeLclVar* shuffleMaskLclVar = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); + BlockRange().InsertBefore(node, shuffleMaskLclVar); + LowerNode(shuffleMaskLclVar); + + GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, NI_PackedSimd_Subtract, TYP_BYTE, 16); + BlockRange().InsertBefore(node, upperMask); + LowerNode(upperMask); + + GenTree* swizzle2 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle2); + LowerNode(swizzle2); + + // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two swizzles together to get the final result. + GenTreeHWIntrinsic* result = m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); + BlockRange().InsertBefore(node, result); + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + use.ReplaceWith(result); + } + else + { + result->SetUnusedValue(); + } + + BlockRange().Remove(node); + + return LowerNode(result); +} + //---------------------------------------------------------------------------------------------- // ContainCheckHWIntrinsic: Perform containment analysis for a hardware intrinsic node. // @@ -1223,10 +1309,13 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); - if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId)) + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId) || node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle) { GenTree* immOp = node->GetImmOp(); - if (immOp->IsCnsIntOrI()) + // An immediate operand for SIMD should either be: + // - An integer lane index OR + // - A constant vector (for example, for a shuffle) + if (immOp->IsCnsIntOrI() || immOp->IsCnsVec()) { MakeSrcContained(node, immOp); } From 8d07863d4bb092238c84846fc549876676e917ca Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 16 Jul 2026 18:39:51 -0700 Subject: [PATCH 05/26] Enable null checks for PackedSimd loads/stores --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 7 +++++++ src/coreclr/jit/lowerwasm.cpp | 6 ++++++ src/coreclr/jit/stacklevelsetter.cpp | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 39ad759d018829..07bed6302a7f86 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -59,6 +59,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case HW_Category_MemoryLoad: { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GenTree* addr = nullptr; + assert(node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)); + assert(addr != nullptr); + + regNumber addrReg = GetMultiUseOperandReg(addr); + genEmitNullCheck(addrReg); + if (info.needsJumpTableFallback()) { genHWIntrinsicJumpTableFallback(node, info); diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 5f611e7da9e2da..bd94ca4869aaf0 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -829,6 +829,12 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) NamedIntrinsic intrinsic = node->GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); bool hasImmOp = HWIntrinsicInfo::HasImmediateOperand(intrinsic); + GenTree* addr = nullptr; + + if (node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)) + { + SetMultiplyUsed(addr DEBUGARG("LowerHWIntrinsic memory address (null check)")); + } switch (intrinsic) { diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index c33de2c033d374..925ae2c8ea8ce4 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -273,8 +273,20 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) } } break; +#elif defined(FEATURE_HW_INTRINSICS) && defined(TARGET_WASM) + case GT_HWINTRINSIC: + { + HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); + if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) + { + SetThrowHelperBlock(SCK_NULL_CHECK, block); + } + } + break; + #endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + case GT_INDEX_ADDR: if (node->AsIndexAddr()->IsBoundsChecked()) { From 0df646ae10c9ee1b882584f5e62ca825cde093a4 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 10:34:23 -0700 Subject: [PATCH 06/26] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerwasm.cpp | 2 +- src/coreclr/jit/stacklevelsetter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index bd94ca4869aaf0..b89f9324027aaf 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -886,7 +886,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_PackedSimd_LoadScalarVector128: case NI_PackedSimd_LoadWideningVector128: { - // These intrinsics don't a required immediate operand + // These intrinsics don't require an immediate operand assert(!hasImmOp); break; } diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 925ae2c8ea8ce4..47b5fc8f0a733d 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -284,7 +284,7 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) } break; -#endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +#endif // defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_WASM)) case GT_INDEX_ADDR: From 6ddbd32ed8bc387ecea6b4ebda469dc4996cb586 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 10:55:11 -0700 Subject: [PATCH 07/26] Add PackedSimdTests to exercise null check for loads/stores --- .../Wasm/PackedSimd/PackedSimdTests.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index db66eb3b89a36c..7ff2ea5b268f2d 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics.CodeAnalysis; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Wasm; using Xunit; @@ -491,6 +492,48 @@ public static unsafe void LoadScalarAndSplatTest() Assert.Equal(Vector128.Create(3.14f, 3.14f, 3.14f, 3.14f), floatSplat); } + [Fact] + public static unsafe void LoadStoreNullCheckTest() + { + Assert.Throws(() => LoadScalarAndSplatVector128(null)); + Assert.Throws(() => LoadScalarVector128(null)); + Assert.Throws(() => LoadWideningVector128(null)); + Assert.Throws(() => LoadScalarAndInsert(null, 2)); + Assert.Throws(() => StoreSelectedScalar(null, 2)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndSplatVector128(int* address) + { + return PackedSimd.LoadScalarAndSplatVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarVector128(int* address) + { + return PackedSimd.LoadScalarVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadWideningVector128(sbyte* address) + { + return PackedSimd.LoadWideningVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndInsert(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + return PackedSimd.LoadScalarAndInsert(address, vector, index); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe void StoreSelectedScalar(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + PackedSimd.StoreSelectedScalar(address, vector, index); + } + [Fact] public static unsafe void LoadWideningTest() { From f1f45cec48b134108ece5621931acb45435dc0ee Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 11:36:36 -0700 Subject: [PATCH 08/26] jit-format --- src/coreclr/jit/stacklevelsetter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 47b5fc8f0a733d..ccba1b54334f46 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -286,7 +286,6 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) #endif // defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_WASM)) - case GT_INDEX_ADDR: if (node->AsIndexAddr()->IsBoundsChecked()) { From d53c1cc428bf248b83dd9072121164ba6e9c5f0d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 11:59:48 -0700 Subject: [PATCH 09/26] Add header comment --- src/coreclr/jit/lowerwasm.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index e89a2f081bedbd..0438a10de105c5 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1309,6 +1309,15 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) return LowerNode(node); } +// -------------------------------------------------------------------------------- +// LowerHWIntrinsicNativeShuffle: Lowers a PackedSimd Shuffle call with a possibly non-constant mask +// +// Arguments: +// node - The hardware intrinsic node. +// +// Notes: +// If the shuffle mask is a constant vector, it can be contained as an immediate and emitted. Otherwise, +// the shuffle is rewritten into two swizzles for the upper and lower input vectors and combined into the final result. GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) { assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); From e78535b29e75251d63cd1f53f09336c2ef90caee Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 17:50:46 -0700 Subject: [PATCH 10/26] Fix segfault in release: move side effecting statement out of assert() --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index ca3125f7fd91f9..a39265cd3609bd 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -75,8 +75,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); GenTree* addr = nullptr; - assert(node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)); - assert(addr != nullptr); + bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); + assert(isMem && addr != nullptr); regNumber addrReg = GetMultiUseOperandReg(addr); genEmitNullCheck(addrReg); From bdbba8c38589e81cc4b2c17e2bbe71c1b9f8aee1 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 20 Jul 2026 09:26:50 -0700 Subject: [PATCH 11/26] jit-format --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index a39265cd3609bd..0bb051dfcfdc13 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -75,7 +75,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); GenTree* addr = nullptr; - bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); + bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); assert(isMem && addr != nullptr); regNumber addrReg = GetMultiUseOperandReg(addr); From 27bf1040f6d78e5e30c68419953fef06eb8f9005 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 20 Jul 2026 09:53:50 -0700 Subject: [PATCH 12/26] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsic.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 7ba21d056f02a1..dcb7ebe20c8c17 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1333,8 +1333,7 @@ struct HWIntrinsicInfo } case NI_PackedSimd_Shuffle: { - // (v128, v128, lane_imm) - *imm1Pos = 3; + // (v128, v128, shuffle_mask) break; } default: From 726aa2b509b9b607fb76af6fb2898962e2bc0b50 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 15:51:57 -0700 Subject: [PATCH 13/26] Fix immediate operand position unset bug for shuffle --- src/coreclr/jit/hwintrinsic.h | 1 + src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index dcb7ebe20c8c17..3d7205c4cd1288 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1334,6 +1334,7 @@ struct HWIntrinsicInfo case NI_PackedSimd_Shuffle: { // (v128, v128, shuffle_mask) + *imm1Pos = 3; break; } default: diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 4939a0bfc8af21..fd52a598ee8db7 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -52,7 +52,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GenTree* src = node->Op(1); regNumber srcReg = GetMultiUseOperandReg(src); GetEmitter()->emitIns_I(INS_local_get, emitActualTypeSize(src), WasmRegToIndex(srcReg)); - GetEmitter()->emitIns_V128Imm(INS_i8x16_shuffle, info.GetImmediateVecOperand().u8); + GetEmitter()->emitIns_V128Imm(INS_i8x16_shuffle, node->Op(2)->AsVecCon()->gtSimdVal.u8); } else { From 067c299ca34c23a8031b13481d398e9638a48070 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 15:53:30 -0700 Subject: [PATCH 14/26] More review feedback --- src/coreclr/jit/codegenwasm.cpp | 2 -- src/coreclr/jit/lowerwasm.cpp | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 58c0018687da2e..515a09248160a9 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2863,10 +2863,8 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) else // A normal store, not a WriteBarrier store { var_types type = tree->TypeGet(); - instruction ins = ins_Store(type); // TODO-WASM: Memory barriers - if (type == TYP_SIMD8) { // stack: [addr, value] -> store the low 8 bytes. diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index e0145cf9479b31..d156ee8c7d725e 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1355,11 +1355,11 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> // /--* originalMask simd // +--* upperBnd simd - // upperMask = * HWINRINSIC simd byte PackedSimd.Subtract + // upperMask = * HWINTRINSIC simd byte PackedSimd.Subtract // /--* op2 simd // +--* upperMask simd // tmp2 = * HWINTRINSIC simd byte PackedSimd.Swizzle - // /--* tmp1 simd + // /--* tmp1 simd // +--* tmp2 simd // res = * HWINTRINSIC simd byte PackedSimd.Or From 3b00dc3783abb45093ea8f7aaf7269892d16ea43 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 17:57:09 -0700 Subject: [PATCH 15/26] More review feedback: handle shuffle masks that aren't in bounds --- src/coreclr/jit/lowerwasm.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index d156ee8c7d725e..c6ffa6dd69eef4 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1339,14 +1339,27 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) GenTree* shuffleMask = node->Op(3); var_types resultType = node->TypeGet(); - // No extra work to do if the shuffle is a constant vector, it can be contained as an immediate and emitted. + // No extra work to do if the shuffle is an in-range constant vector, it can be contained as an immediate and emitted. if (shuffleMask->IsCnsVec()) { - ContainCheckHWIntrinsic(node); - return node->gtNext; + const simd_t& mask = shuffleMask->AsVecCon()->gtSimdVal; + bool allInRange = true; + for (int i = 0; i < 16; i++) + { + if (mask.u8[i] >= 32) + { + allInRange = false; + break; + } + } + if (allInRange) + { + ContainCheckHWIntrinsic(node); + return node->gtNext; + } } - // If the shuffle mask is not a constant vector, we will need to rewrite the shuffle into two swizzles: + // If the shuffle mask is not a constant vector or the mask is not in range, we will need to rewrite the shuffle into two swizzles: // 1 to handle elements from the first vector, and 1 to handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. // We will be constructing IR like the following: // /--* op1 simd From d925633ab9bf2deb73b3b10c623fcecb11d4d1f0 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 18:00:41 -0700 Subject: [PATCH 16/26] jit-format --- src/coreclr/jit/codegenwasm.cpp | 2 +- src/coreclr/jit/lowerwasm.cpp | 45 ++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 515a09248160a9..1e01aef6938336 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2862,7 +2862,7 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) } else // A normal store, not a WriteBarrier store { - var_types type = tree->TypeGet(); + var_types type = tree->TypeGet(); // TODO-WASM: Memory barriers if (type == TYP_SIMD8) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index c6ffa6dd69eef4..c58b017b8647bc 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1334,15 +1334,16 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) { assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); - GenTree* op1 = node->Op(1); - GenTree* op2 = node->Op(2); - GenTree* shuffleMask = node->Op(3); - var_types resultType = node->TypeGet(); + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + GenTree* shuffleMask = node->Op(3); + var_types resultType = node->TypeGet(); - // No extra work to do if the shuffle is an in-range constant vector, it can be contained as an immediate and emitted. + // No extra work to do if the shuffle is an in-range constant vector, it can be contained as an immediate and + // emitted. if (shuffleMask->IsCnsVec()) { - const simd_t& mask = shuffleMask->AsVecCon()->gtSimdVal; + const simd_t& mask = shuffleMask->AsVecCon()->gtSimdVal; bool allInRange = true; for (int i = 0; i < 16; i++) { @@ -1359,9 +1360,10 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) } } - // If the shuffle mask is not a constant vector or the mask is not in range, we will need to rewrite the shuffle into two swizzles: - // 1 to handle elements from the first vector, and 1 to handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. - // We will be constructing IR like the following: + // If the shuffle mask is not a constant vector or the mask is not in range, we will need to rewrite the shuffle + // into two swizzles: 1 to handle elements from the first vector, and 1 to handle elements from the second vector. + // The two swizzles will then be or'd together to produce the final result. We will be constructing IR like the + // following: // /--* op1 simd // +--* originalMask simd // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle @@ -1376,17 +1378,20 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // +--* tmp2 simd // res = * HWINTRINSIC simd byte PackedSimd.Or - // Shuffle mask will be used twice, replace with a local - LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); + LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); - // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a vector with the elements from the first vector in the correct order, and zero's for all elements which correspond to the second vector. - GenTree* swizzle1 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); + // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a + // vector with the elements from the first vector in the correct order, and zero's for all elements which correspond + // to the second vector. + GenTree* swizzle1 = + m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); BlockRange().InsertBefore(node, swizzle1); LowerNode(swizzle1); - // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading to a zero in the result of the swizzle. + // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading + // to a zero in the result of the swizzle. GenTreeVecCon* upperBound = m_compiler->gtNewVconNode(shuffleMask->TypeGet()); upperBound->EvaluateBroadcastInPlace(TYP_BYTE, static_cast(16)); BlockRange().InsertBefore(node, upperBound); @@ -1397,16 +1402,20 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) BlockRange().InsertBefore(node, shuffleMaskLclVar); LowerNode(shuffleMaskLclVar); - GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, NI_PackedSimd_Subtract, TYP_BYTE, 16); + GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, + NI_PackedSimd_Subtract, TYP_BYTE, 16); BlockRange().InsertBefore(node, upperMask); LowerNode(upperMask); - GenTree* swizzle2 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + GenTree* swizzle2 = + m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); BlockRange().InsertBefore(node, swizzle2); LowerNode(swizzle2); - // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two swizzles together to get the final result. - GenTreeHWIntrinsic* result = m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); + // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two + // swizzles together to get the final result. + GenTreeHWIntrinsic* result = + m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); BlockRange().InsertBefore(node, result); LIR::Use use; From 4ed524ff054223caef5bf3603438f2a3c3944b17 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 18:07:10 -0700 Subject: [PATCH 17/26] Revert accidental diff --- src/coreclr/jit/codegenwasm.cpp | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 1e01aef6938336..75b6403dec8fb9 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -103,18 +103,6 @@ void CodeGen::genMarkLabelsForCodegen() // void CodeGen::genBeginFnProlog() { - // SIMD (Vector2/3/4, Vector128) parameters are lowered to i32 in the wasm signature, so any - // vector operation performed on them produces an invalid module (e.g. a v128/f64 op with - // an i32 operand). Bail such methods to the interpreter until SIMD parameters are - // properly supported in the wasm calling convention. - for (unsigned lclNum = 0; lclNum < m_compiler->info.compArgsCount; lclNum++) - { - if (varTypeIsSIMD(m_compiler->lvaGetDesc(lclNum)->TypeGet())) - { - NYI_WASM_SIMD("SIMD parameter"); - } - } - GetEmitter()->emitIns(INS_code_size); FuncInfoDsc* const func = m_compiler->funGetFunc(ROOT_FUNC_IDX); @@ -2592,11 +2580,6 @@ void CodeGen::genCodeForLclFld(GenTreeLclFld* tree) LclVarDsc* varDsc = m_compiler->lvaGetDesc(tree); var_types type = tree->TypeGet(); - if (type == TYP_SIMD16) - { - NYI_WASM_SIMD("SIMD16 local field load"); - } - if (type == TYP_SIMD12) { genLoadLclTypeSimd12(tree); @@ -2865,6 +2848,7 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) var_types type = tree->TypeGet(); // TODO-WASM: Memory barriers + if (type == TYP_SIMD8) { // stack: [addr, value] -> store the low 8 bytes. @@ -2972,13 +2956,6 @@ void CodeGen::genCallInstruction(GenTreeCall* call) assert(seg.IsPassedInRegister()); WasmValueType wvt = WasmRegToType(seg.GetRegister()); assert(wvt < WasmValueType::Count); - if (wvt == WasmValueType::V128) - { - // Passing a 16-byte SIMD value by value through a call is not yet correctly - // implemented: the argument is materialized as an i32 (by-ref) while the call - // signature requires v128, producing an invalid module. Bail for now. - NYI_WASM_SIMD("SIMD16 call argument"); - } typeStack.Push((CorInfoWasmType)emitter::GetWasmValueTypeCode(wvt)); } } @@ -3903,10 +3880,6 @@ void CodeGen::genLoadLocalIntoReg(regNumber targetReg, unsigned lclNum) { LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum); var_types type = varDsc->GetRegisterType(); - if (type == TYP_SIMD16) - { - NYI_WASM_SIMD("SIMD16 local load"); - } GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, GetFramePointerRegIndex()); GetEmitter()->emitIns_S(ins_Load(type), emitTypeSize(type), lclNum, 0); From 2656ed466525cc58a6757bb201ee605bd4ffe1a5 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 21 Jul 2026 19:14:56 -0700 Subject: [PATCH 18/26] Fix bad stack ordering in shuffle fallback codegen --- src/coreclr/jit/lowerwasm.cpp | 56 +++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index c58b017b8647bc..fd5ef12566e661 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1364,20 +1364,38 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // into two swizzles: 1 to handle elements from the first vector, and 1 to handle elements from the second vector. // The two swizzles will then be or'd together to produce the final result. We will be constructing IR like the // following: + // op1 = ... + // op2 = ... + // /--* op2 + // /--* LCL_VAR op2Tmp + // STORE_LCL_VAR op2Tmp + // originalMask = ... + // /--* originalMask + // /--* LCL_VAR originalMaskTmp + // STORE_LCL_VAR originalMaskTmp + // m0 = * LCL_VAR originalMaskTmp // /--* op1 simd - // +--* originalMask simd + // +--* m0 simd // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // op2Reload = * LCL_VAR op2tmp + // m1 = * LCL_VAR originalMaskTmp // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> - // /--* originalMask simd + // /--* m1 simd // +--* upperBnd simd // upperMask = * HWINTRINSIC simd byte PackedSimd.Subtract - // /--* op2 simd + // /--* op2Reload simd // +--* upperMask simd - // tmp2 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // tmp3 = * HWINTRINSIC simd byte PackedSimd.Swizzle // /--* tmp1 simd - // +--* tmp2 simd + // +--* tmp3 simd // res = * HWINTRINSIC simd byte PackedSimd.Or + // op2 needs to be moved, replace with a local, but don't immediately reload + LIR::Use op2Use(BlockRange(), &node->Op(2), node); + unsigned int op2Tmp = op2Use.ReplaceWithLclVar(m_compiler); + GenTree* op2Reload = node->Op(2); + BlockRange().Remove(op2Reload); + // Shuffle mask will be used twice, replace with a local LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); @@ -1385,37 +1403,43 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a // vector with the elements from the first vector in the correct order, and zero's for all elements which correspond // to the second vector. + GenTree* maskReload = node->Op(3); GenTree* swizzle1 = - m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); + m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, maskReload, NI_PackedSimd_Swizzle, TYP_BYTE, 16); BlockRange().InsertBefore(node, swizzle1); LowerNode(swizzle1); - // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading - // to a zero in the result of the swizzle. + // Re-load op2 + BlockRange().InsertBefore(node, op2Reload); + LowerNode(op2Reload); + + // Re-load the original shuffle mask + GenTreeLclVar* maskReload2 = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); + BlockRange().InsertBefore(node, maskReload2); + LowerNode(maskReload2); + + // Create constant upper bound vector of <16, 16, ..., 16> GenTreeVecCon* upperBound = m_compiler->gtNewVconNode(shuffleMask->TypeGet()); upperBound->EvaluateBroadcastInPlace(TYP_BYTE, static_cast(16)); BlockRange().InsertBefore(node, upperBound); LowerNode(upperBound); - // Re-load the original shuffle mask - GenTreeLclVar* shuffleMaskLclVar = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); - BlockRange().InsertBefore(node, shuffleMaskLclVar); - LowerNode(shuffleMaskLclVar); - - GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, + // Use the above to subtract 16 from each mask element to mark each element which corresponds to the lower vector as + // unused, leading to a zero in the result of the swizzle. + GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), maskReload2, upperBound, NI_PackedSimd_Subtract, TYP_BYTE, 16); BlockRange().InsertBefore(node, upperMask); LowerNode(upperMask); GenTree* swizzle2 = - m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2Reload, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); BlockRange().InsertBefore(node, swizzle2); LowerNode(swizzle2); // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two // swizzles together to get the final result. GenTreeHWIntrinsic* result = - m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); + m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_BYTE, 16); BlockRange().InsertBefore(node, result); LIR::Use use; From 506a7711db824271e9c69fab2c841634bffa20e1 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 22 Jul 2026 11:10:35 -0700 Subject: [PATCH 19/26] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerwasm.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index fd5ef12566e661..501125afba3a96 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1335,7 +1335,6 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); GenTree* op1 = node->Op(1); - GenTree* op2 = node->Op(2); GenTree* shuffleMask = node->Op(3); var_types resultType = node->TypeGet(); @@ -1391,9 +1390,9 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // res = * HWINTRINSIC simd byte PackedSimd.Or // op2 needs to be moved, replace with a local, but don't immediately reload - LIR::Use op2Use(BlockRange(), &node->Op(2), node); - unsigned int op2Tmp = op2Use.ReplaceWithLclVar(m_compiler); - GenTree* op2Reload = node->Op(2); + LIR::Use op2Use(BlockRange(), &node->Op(2), node); + op2Use.ReplaceWithLclVar(m_compiler); + GenTree* op2Reload = node->Op(2); BlockRange().Remove(op2Reload); // Shuffle mask will be used twice, replace with a local From a546d72230d4fb435cf1c71e257f83e65fedb709 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 22 Jul 2026 18:01:12 -0700 Subject: [PATCH 20/26] Separate out fetching PackedSimd.Shuffle immediate from GetImmOp() for clarity since it is a special case --- src/coreclr/jit/hwintrinsic.h | 13 ------------- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 3 ++- src/coreclr/jit/lowerwasm.cpp | 16 +++++++++++----- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 3d7205c4cd1288..b98d83c6c4a289 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1331,12 +1331,6 @@ struct HWIntrinsicInfo *imm1Pos = 3; break; } - case NI_PackedSimd_Shuffle: - { - // (v128, v128, shuffle_mask) - *imm1Pos = 3; - break; - } default: { unreached(); @@ -1501,13 +1495,6 @@ struct HWIntrinsic final return static_cast(lane); } - simd16_t GetImmediateVecOperand() const - { - GenTree* immOp = m_node->GetImmOp(); - assert(immOp->IsCnsVec()); - return immOp->AsVecCon()->gtSimdVal; - } - NamedIntrinsic id; HWIntrinsicCategory category; GenTree* op1; diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index fd52a598ee8db7..e880132d138d94 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -39,7 +39,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { if (info.id == NI_PackedSimd_Shuffle) { - simd16_t shuffleMask = info.GetImmediateVecOperand(); + assert(node->Op(3)->IsCnsVec()); + simd16_t shuffleMask = node->Op(3)->AsVecCon()->gtSimdVal; GetEmitter()->emitIns_V128Imm(ins, shuffleMask.u8); } else if ((info.id == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 501125afba3a96..bd632635e5d12b 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -967,6 +967,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) GenTree* Lowering::LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node) { GenTree* immOp = node->GetImmOp(); + assert(varTypeIsIntegral(immOp->TypeGet()) && "Immediate operand must be an integral type"); + if (!immOp->IsCnsIntOrI()) { // This node has a non-constant immediate operand, so it will need a jump table @@ -1465,15 +1467,19 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); - if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId) || node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle) + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId)) { GenTree* immOp = node->GetImmOp(); - // An immediate operand for SIMD should either be: - // - An integer lane index OR - // - A constant vector (for example, for a shuffle) - if (immOp->IsCnsIntOrI() || immOp->IsCnsVec()) + if (immOp->IsCnsIntOrI()) { MakeSrcContained(node, immOp); } } + else if (intrinsicId == NI_PackedSimd_Shuffle && node->Op(3)->IsCnsVec()) + { + // Shuffle is a special case where the mask is a constant vector immediate + // which must be contained. If the mask is non-constant we will re-write the shuffle into a fallback equivalent operation. + // (see LowerHWIntrinsicNativeShuffle). + MakeSrcContained(node, node->Op(3)); + } } From 5fb29db876fd3227c501d8a7a95a603674534bf3 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 16:50:41 -0700 Subject: [PATCH 21/26] Emit range check around PackedSimd.Shuffle mask --- src/coreclr/jit/codegen.h | 5 +++ src/coreclr/jit/codegenwasm.cpp | 15 ++++++- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 6 +-- src/coreclr/jit/lowerwasm.cpp | 31 +++++++++++--- src/coreclr/jit/stacklevelsetter.cpp | 30 ++++++++++++++ .../Wasm/PackedSimd/PackedSimdTests.cs | 40 +++++++++++++------ 6 files changed, 105 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f7d9192cb588ec..fe01d34bd51409 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -229,6 +229,10 @@ class CodeGen final : public CodeGenInterface void genEmitBeginBlock(WasmValueType blockType = WasmValueType::Invalid); void genEmitEndBlock(); void genEmitFunctionEnd(bool emitTerminalUnreachable = true); +#ifdef FEATURE_SIMD + void genEmitSimdRangeCheck(GenTreeBoundsChk* boundsCheck); +#endif + #endif void genEmitStartBlock(BasicBlock* block); @@ -1089,6 +1093,7 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_WASM) void genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrinsic info); + void genHWIntrinsicShuffleRngCheck(GenTreeHWIntrinsic* node, HWIntrinsic info); #endif #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 75b6403dec8fb9..636673ec91a0b5 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2272,10 +2272,23 @@ void CodeGen::genRangeCheck(GenTree* tree) assert(tree->OperIs(GT_BOUNDS_CHECK)); GenTreeBoundsChk* boundsCheck = tree->AsBoundsChk(); genConsumeOperands(boundsCheck); - GetEmitter()->emitIns(INS_I_ge_u); +#ifdef FEATURE_SIMD + if (varTypeIsSIMD(boundsCheck->GetIndex()->TypeGet())) + { + GetEmitter()->emitIns(INS_i8x16_splat); + GetEmitter()->emitIns(INS_i8x16_ge_u); + GetEmitter()->emitIns(INS_v128_any_true); + } + else +#endif + { + GetEmitter()->emitIns(INS_I_ge_u); + } + genJumpToThrowHlpBlk(boundsCheck->gtThrowKind); } + //------------------------------------------------------------------------ // genCodeForIndexAddr: Produce code for a GT_INDEX_ADDR node. // diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index e880132d138d94..cd6e4d01e23dc0 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -39,9 +39,9 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { if (info.id == NI_PackedSimd_Shuffle) { - assert(node->Op(3)->IsCnsVec()); - simd16_t shuffleMask = node->Op(3)->AsVecCon()->gtSimdVal; - GetEmitter()->emitIns_V128Imm(ins, shuffleMask.u8); + JITDUMP("Lowered to a native shuffle instruction, so the mask is encoded as an immediate"); + assert(node->Op(3)->isContained()); + GetEmitter()->emitIns_V128Imm(ins, node->Op(3)->AsVecCon()->gtSimdVal.u8); } else if ((info.id == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) { diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index bd632635e5d12b..00f05876f7bdcd 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1375,13 +1375,18 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // /--* LCL_VAR originalMaskTmp // STORE_LCL_VAR originalMaskTmp // m0 = * LCL_VAR originalMaskTmp + // b0 = * CNS_INT int 31 + // /--* m0 simd + // +--* b0 int + // GT_BOUNDS_CHECK RNG_CHK_FAIL + // m1 = * LCL_VAR originalMaskTmp // /--* op1 simd - // +--* m0 simd + // +--* m1 simd // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle // op2Reload = * LCL_VAR op2tmp - // m1 = * LCL_VAR originalMaskTmp + // m2 = * LCL_VAR originalMaskTmp // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> - // /--* m1 simd + // /--* m2 simd // +--* upperBnd simd // upperMask = * HWINTRINSIC simd byte PackedSimd.Subtract // /--* op2Reload simd @@ -1397,16 +1402,30 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) GenTree* op2Reload = node->Op(2); BlockRange().Remove(op2Reload); - // Shuffle mask will be used twice, replace with a local + // Shuffle mask will be used several times, replace with a local LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); + GenTree* maskReloadForChk = node->Op(3); + + // Insert a bounds check for the shuffle mask. A bounds check against a simd value will be handled by codegen as a + // GreaterThanAny(vec, bound) type operation. + GenTree* bound = m_compiler->gtNewIconNode(32); + BlockRange().InsertBefore(node, bound); + LowerNode(bound); + + GenTree* boundsCheck = new (m_compiler, GT_BOUNDS_CHECK) GenTreeBoundsChk(maskReloadForChk, bound, SCK_RNGCHK_FAIL); + BlockRange().InsertBefore(node, boundsCheck); + LowerNode(boundsCheck); // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a // vector with the elements from the first vector in the correct order, and zero's for all elements which correspond // to the second vector. - GenTree* maskReload = node->Op(3); + GenTree* maskReload1 = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); + BlockRange().InsertBefore(node, maskReload1); + LowerNode(maskReload1); + GenTree* swizzle1 = - m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, maskReload, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, maskReload1, NI_PackedSimd_Swizzle, TYP_BYTE, 16); BlockRange().InsertBefore(node, swizzle1); LowerNode(swizzle1); diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index ccba1b54334f46..52b20a3c49e466 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -277,10 +277,40 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) case GT_HWINTRINSIC: { HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); + NamedIntrinsic intrinsicId = node->AsHWIntrinsic()->GetHWIntrinsicId(); if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) { SetThrowHelperBlock(SCK_NULL_CHECK, block); } + else if (intrinsicId == NI_PackedSimd_Shuffle) + { + GenTreeHWIntrinsic* hwIntrinsic = node->AsHWIntrinsic(); + if (hwIntrinsic->Op(3)->IsCnsVec()) + { + // If we have an out of range constant shuffle mask, we will need to throw a range check exception. + const simd_t& mask = hwIntrinsic->Op(3)->AsVecCon()->gtSimdVal; + bool allInRange = true; + + for (int i = 0; i < 16; i++) + { + if (mask.u8[i] >= 32) + { + allInRange = false; + break; + } + } + + if (!allInRange) + { + SetThrowHelperBlock(SCK_RNGCHK_FAIL, block); + } + } + else + { + // Non-constant shuffle mask needs a range check + SetThrowHelperBlock(SCK_RNGCHK_FAIL, block); + } + } } break; diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index 9d978d0d4230d8..699320900d3474 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -16,9 +16,9 @@ public sealed class PackedSimdTests [DynamicDependency(DynamicallyAccessedMemberTypes.PublicProperties, typeof(PackedSimd))] public static unsafe void PackedSimdIsSupported() { - MethodInfo? methodInfo = typeof(PackedSimd).GetProperty(nameof(PackedSimd.IsSupported))?.GetGetMethod(); - Assert.NotNull(methodInfo); - Assert.Equal(PackedSimd.IsSupported, methodInfo.Invoke(null, null)); + // MethodInfo? methodInfo = typeof(PackedSimd).GetProperty(nameof(PackedSimd.IsSupported))?.GetGetMethod(); + // Assert.NotNull(methodInfo); + // Assert.Equal(PackedSimd.IsSupported, methodInfo.Invoke(null, null)); Assert.Equal(PackedSimd.IsSupported, Vector128.IsHardwareAccelerated); Assert.True(PackedSimd.IsSupported); } @@ -1074,15 +1074,15 @@ public static unsafe void LoadScalarAndSplatTest() Assert.Equal(Vector128.Create(3.14f, 3.14f, 3.14f, 3.14f), floatSplat); } - [Fact] - public static unsafe void LoadStoreNullCheckTest() - { - Assert.Throws(() => LoadScalarAndSplatVector128(null)); - Assert.Throws(() => LoadScalarVector128(null)); - Assert.Throws(() => LoadWideningVector128(null)); - Assert.Throws(() => LoadScalarAndInsert(null, 2)); - Assert.Throws(() => StoreSelectedScalar(null, 2)); - } + // [Fact] + // public static unsafe void LoadStoreNullCheckTest() + // { + // Assert.Throws(() => LoadScalarAndSplatVector128(null)); + // Assert.Throws(() => LoadScalarVector128(null)); + // Assert.Throws(() => LoadWideningVector128(null)); + // Assert.Throws(() => LoadScalarAndInsert(null, 2)); + // Assert.Throws(() => StoreSelectedScalar(null, 2)); + // } [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe Vector128 LoadScalarAndSplatVector128(int* address) @@ -1560,4 +1560,20 @@ public static unsafe void BitmaskTest() Assert.Equal(v128emsb_s, (uint)bitmask_s); Assert.Equal(v128emsb_i, (uint)bitmask_i); } + + [Fact] + public static void ShuffleTest() + { + var v1 = Vector128.Create((byte)1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16); + var v2 = Vector128.Create((byte)17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32); + + var shuffled = PackedSimd.Shuffle(v1.AsByte(), v2.AsByte(), + Vector128.Create((byte)0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10, 27, 12, 29, 14, 31)); + + var expect = Vector128.Create((byte)1, 18, 3, 20, 5, 22, 7, 24, + 9, 26, 11, 28, 13, 30, 15, 32); + Assert.Equal(expect, shuffled); + } } From 1ccc3470b7b8dfcfdbf31a21745cfeb304c65467 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 16:51:40 -0700 Subject: [PATCH 22/26] jit-format --- src/coreclr/jit/codegenwasm.cpp | 1 - src/coreclr/jit/lowerwasm.cpp | 8 ++++---- src/coreclr/jit/stacklevelsetter.cpp | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 636673ec91a0b5..c1db666dbc7bff 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2288,7 +2288,6 @@ void CodeGen::genRangeCheck(GenTree* tree) genJumpToThrowHlpBlk(boundsCheck->gtThrowKind); } - //------------------------------------------------------------------------ // genCodeForIndexAddr: Produce code for a GT_INDEX_ADDR node. // diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 00f05876f7bdcd..82cfd254056dfc 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1404,8 +1404,8 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // Shuffle mask will be used several times, replace with a local LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); - unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); - GenTree* maskReloadForChk = node->Op(3); + unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); + GenTree* maskReloadForChk = node->Op(3); // Insert a bounds check for the shuffle mask. A bounds check against a simd value will be handled by codegen as a // GreaterThanAny(vec, bound) type operation. @@ -1497,8 +1497,8 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) else if (intrinsicId == NI_PackedSimd_Shuffle && node->Op(3)->IsCnsVec()) { // Shuffle is a special case where the mask is a constant vector immediate - // which must be contained. If the mask is non-constant we will re-write the shuffle into a fallback equivalent operation. - // (see LowerHWIntrinsicNativeShuffle). + // which must be contained. If the mask is non-constant we will re-write the shuffle into a fallback equivalent + // operation. (see LowerHWIntrinsicNativeShuffle). MakeSrcContained(node, node->Op(3)); } } diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 52b20a3c49e466..55bc82c0ae4731 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -277,7 +277,7 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) case GT_HWINTRINSIC: { HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); - NamedIntrinsic intrinsicId = node->AsHWIntrinsic()->GetHWIntrinsicId(); + NamedIntrinsic intrinsicId = node->AsHWIntrinsic()->GetHWIntrinsicId(); if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) { SetThrowHelperBlock(SCK_NULL_CHECK, block); @@ -288,7 +288,7 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) if (hwIntrinsic->Op(3)->IsCnsVec()) { // If we have an out of range constant shuffle mask, we will need to throw a range check exception. - const simd_t& mask = hwIntrinsic->Op(3)->AsVecCon()->gtSimdVal; + const simd_t& mask = hwIntrinsic->Op(3)->AsVecCon()->gtSimdVal; bool allInRange = true; for (int i = 0; i < 16; i++) From 90adaee5d74696987d10003c33026c9d5f46fe2d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 16:57:08 -0700 Subject: [PATCH 23/26] Adjust comments --- src/coreclr/jit/lowerwasm.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 82cfd254056dfc..8f4b2136ec1567 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1362,9 +1362,9 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) } // If the shuffle mask is not a constant vector or the mask is not in range, we will need to rewrite the shuffle - // into two swizzles: 1 to handle elements from the first vector, and 1 to handle elements from the second vector. - // The two swizzles will then be or'd together to produce the final result. We will be constructing IR like the - // following: + // into a BOUNDS_CHECK around the mask, and two swizzles: 1 to handle elements from the first vector, and 1 to + // handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. + // We will be constructing IR like the following: // op1 = ... // op2 = ... // /--* op2 @@ -1395,6 +1395,14 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // /--* tmp1 simd // +--* tmp3 simd // res = * HWINTRINSIC simd byte PackedSimd.Or + // + // This is roughly equivalent to the following C#: + // ... + // if (GreaterThanAny(originalMask, 31)) { throw new ArrayIndexOutOfBoundsException(); } + // tmp1 = PackedSimd.Swizzle(op1, originalMask); + // tmp2 = PackedSimd.Swizzle(op2, PackedSimd.Subtract(originalMask, PackedSimd.Splat(0x10))); + // result = PackedSimd.Or(tmp1, tmp2); + //... // op2 needs to be moved, replace with a local, but don't immediately reload LIR::Use op2Use(BlockRange(), &node->Op(2), node); From 20ce69683ff7d38fd2267eaafa5b094a31f43c1a Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 17:02:24 -0700 Subject: [PATCH 24/26] Reset state of PackedSimdTests.cs --- .../Wasm/PackedSimd/PackedSimdTests.cs | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index 699320900d3474..9d978d0d4230d8 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -16,9 +16,9 @@ public sealed class PackedSimdTests [DynamicDependency(DynamicallyAccessedMemberTypes.PublicProperties, typeof(PackedSimd))] public static unsafe void PackedSimdIsSupported() { - // MethodInfo? methodInfo = typeof(PackedSimd).GetProperty(nameof(PackedSimd.IsSupported))?.GetGetMethod(); - // Assert.NotNull(methodInfo); - // Assert.Equal(PackedSimd.IsSupported, methodInfo.Invoke(null, null)); + MethodInfo? methodInfo = typeof(PackedSimd).GetProperty(nameof(PackedSimd.IsSupported))?.GetGetMethod(); + Assert.NotNull(methodInfo); + Assert.Equal(PackedSimd.IsSupported, methodInfo.Invoke(null, null)); Assert.Equal(PackedSimd.IsSupported, Vector128.IsHardwareAccelerated); Assert.True(PackedSimd.IsSupported); } @@ -1074,15 +1074,15 @@ public static unsafe void LoadScalarAndSplatTest() Assert.Equal(Vector128.Create(3.14f, 3.14f, 3.14f, 3.14f), floatSplat); } - // [Fact] - // public static unsafe void LoadStoreNullCheckTest() - // { - // Assert.Throws(() => LoadScalarAndSplatVector128(null)); - // Assert.Throws(() => LoadScalarVector128(null)); - // Assert.Throws(() => LoadWideningVector128(null)); - // Assert.Throws(() => LoadScalarAndInsert(null, 2)); - // Assert.Throws(() => StoreSelectedScalar(null, 2)); - // } + [Fact] + public static unsafe void LoadStoreNullCheckTest() + { + Assert.Throws(() => LoadScalarAndSplatVector128(null)); + Assert.Throws(() => LoadScalarVector128(null)); + Assert.Throws(() => LoadWideningVector128(null)); + Assert.Throws(() => LoadScalarAndInsert(null, 2)); + Assert.Throws(() => StoreSelectedScalar(null, 2)); + } [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe Vector128 LoadScalarAndSplatVector128(int* address) @@ -1560,20 +1560,4 @@ public static unsafe void BitmaskTest() Assert.Equal(v128emsb_s, (uint)bitmask_s); Assert.Equal(v128emsb_i, (uint)bitmask_i); } - - [Fact] - public static void ShuffleTest() - { - var v1 = Vector128.Create((byte)1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16); - var v2 = Vector128.Create((byte)17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32); - - var shuffled = PackedSimd.Shuffle(v1.AsByte(), v2.AsByte(), - Vector128.Create((byte)0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10, 27, 12, 29, 14, 31)); - - var expect = Vector128.Create((byte)1, 18, 3, 20, 5, 22, 7, 24, - 9, 26, 11, 28, 13, 30, 15, 32); - Assert.Equal(expect, shuffled); - } } From b03f85122f7bef08dfb8b6bf5df8b583d0c7c5a1 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 17:13:00 -0700 Subject: [PATCH 25/26] Cleanup and copilot feedback --- src/coreclr/jit/codegen.h | 5 ---- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 1 - src/coreclr/jit/lowerwasm.cpp | 3 ++- src/coreclr/jit/stacklevelsetter.cpp | 30 ---------------------- 4 files changed, 2 insertions(+), 37 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index fe01d34bd51409..f7d9192cb588ec 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -229,10 +229,6 @@ class CodeGen final : public CodeGenInterface void genEmitBeginBlock(WasmValueType blockType = WasmValueType::Invalid); void genEmitEndBlock(); void genEmitFunctionEnd(bool emitTerminalUnreachable = true); -#ifdef FEATURE_SIMD - void genEmitSimdRangeCheck(GenTreeBoundsChk* boundsCheck); -#endif - #endif void genEmitStartBlock(BasicBlock* block); @@ -1093,7 +1089,6 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_WASM) void genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrinsic info); - void genHWIntrinsicShuffleRngCheck(GenTreeHWIntrinsic* node, HWIntrinsic info); #endif #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index cd6e4d01e23dc0..7d6841f1f550d6 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -39,7 +39,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { if (info.id == NI_PackedSimd_Shuffle) { - JITDUMP("Lowered to a native shuffle instruction, so the mask is encoded as an immediate"); assert(node->Op(3)->isContained()); GetEmitter()->emitIns_V128Imm(ins, node->Op(3)->AsVecCon()->gtSimdVal.u8); } diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 8f4b2136ec1567..3552c568cdd120 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1421,7 +1421,8 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) BlockRange().InsertBefore(node, bound); LowerNode(bound); - GenTree* boundsCheck = new (m_compiler, GT_BOUNDS_CHECK) GenTreeBoundsChk(maskReloadForChk, bound, SCK_RNGCHK_FAIL); + GenTree* boundsCheck = + new (m_compiler, GT_BOUNDS_CHECK) GenTreeBoundsChk(maskReloadForChk, bound, SCK_ARG_RNG_EXCPN); BlockRange().InsertBefore(node, boundsCheck); LowerNode(boundsCheck); diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 55bc82c0ae4731..ccba1b54334f46 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -277,40 +277,10 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) case GT_HWINTRINSIC: { HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); - NamedIntrinsic intrinsicId = node->AsHWIntrinsic()->GetHWIntrinsicId(); if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) { SetThrowHelperBlock(SCK_NULL_CHECK, block); } - else if (intrinsicId == NI_PackedSimd_Shuffle) - { - GenTreeHWIntrinsic* hwIntrinsic = node->AsHWIntrinsic(); - if (hwIntrinsic->Op(3)->IsCnsVec()) - { - // If we have an out of range constant shuffle mask, we will need to throw a range check exception. - const simd_t& mask = hwIntrinsic->Op(3)->AsVecCon()->gtSimdVal; - bool allInRange = true; - - for (int i = 0; i < 16; i++) - { - if (mask.u8[i] >= 32) - { - allInRange = false; - break; - } - } - - if (!allInRange) - { - SetThrowHelperBlock(SCK_RNGCHK_FAIL, block); - } - } - else - { - // Non-constant shuffle mask needs a range check - SetThrowHelperBlock(SCK_RNGCHK_FAIL, block); - } - } } break; From 48ec254d70121676660bda945a6a3fb310147586 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 24 Jul 2026 17:14:52 -0700 Subject: [PATCH 26/26] Fix comment --- src/coreclr/jit/lowerwasm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 3552c568cdd120..e9ac1c3831c163 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1375,7 +1375,7 @@ GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) // /--* LCL_VAR originalMaskTmp // STORE_LCL_VAR originalMaskTmp // m0 = * LCL_VAR originalMaskTmp - // b0 = * CNS_INT int 31 + // b0 = * CNS_INT int 32 // /--* m0 simd // +--* b0 int // GT_BOUNDS_CHECK RNG_CHK_FAIL