diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index b2acfa5e21f7db..6348fee4092dd7 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3636,6 +3636,9 @@ class Compiler var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdNarrowWithSaturationNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdConcatNode(var_types type, GenTree* op1, GenTree* op2, @@ -3677,6 +3680,15 @@ class Compiler unsigned simdSize, bool isShuffleNative); +#if defined(TARGET_WASM) + GenTree* gtNewSimdWasmTwoSourceShuffleNode(var_types type, + GenTree* op1, + GenTree* op2, + const uint32_t* selectors, + var_types simdBaseType, + unsigned simdSize); +#endif // TARGET_WASM + GenTree* gtNewSimdSqrtNode( var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index bbec675a6f07a1..b655db0c33575c 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -390,6 +390,16 @@ void GenTree::ReplaceWith(GenTree* src, Compiler* comp) this->gtPrev = prev; this->gtNext = next; +#if defined(FEATURE_HW_INTRINSICS) + // A MultiOp storing its operands inline has an interior self-pointer that the + // raw copy above left aliasing "src"; re-point it into this node before "src" + // is destroyed below. + if (OperIsMultiOp()) + { + AsMultiOp()->RelocateInlineOperandsFrom(src); + } +#endif // FEATURE_HW_INTRINSICS + #ifdef DEBUG gtSeqNum = 0; #endif @@ -26245,6 +26255,15 @@ GenTree* Compiler::gtNewSimdMinMaxNode(var_types type, op2 = gtNewSimdCreateScalarUnsafeNode(type, op2, simdBaseType, simdSize); } #elif defined(TARGET_WASM) + if (isScalar) + { + simdSize = 16; + type = TYP_SIMD16; + + op1 = gtNewSimdCreateScalarUnsafeNode(type, op1, simdBaseType, simdSize); + op2 = gtNewSimdCreateScalarUnsafeNode(type, op2, simdBaseType, simdSize); + } + if (!isMagnitude && !isNumber) { intrinsic = isMax ? NI_PackedSimd_Max : NI_PackedSimd_Min; @@ -26520,6 +26539,15 @@ GenTree* Compiler::gtNewSimdMinMaxNativeNode( } } #elif defined(TARGET_WASM) + if (isScalar) + { + simdSize = 16; + type = TYP_SIMD16; + + op1 = gtNewSimdCreateScalarUnsafeNode(type, op1, simdBaseType, simdSize); + op2 = gtNewSimdCreateScalarUnsafeNode(type, op2, simdBaseType, simdSize); + } + if (varTypeIsFloating(simdBaseType)) { intrinsic = isMax ? NI_PackedSimd_PseudoMax : NI_PackedSimd_PseudoMin; @@ -26571,8 +26599,10 @@ GenTree* Compiler::gtNewSimdNarrowNode( assert(varTypeIsArithmetic(simdBaseType) && !varTypeIsLong(simdBaseType)); +#if !defined(TARGET_WASM) GenTree* tmp1; GenTree* tmp2; +#endif #if defined(TARGET_XARCH) GenTree* tmp3; @@ -27005,15 +27035,165 @@ GenTree* Compiler::gtNewSimdNarrowNode( return gtNewSimdHWIntrinsicNode(type, tmp2, NI_AdvSimd_ExtractNarrowingLower, simdBaseType, simdSize); } #elif defined(TARGET_WASM) - tmp1 = nullptr; - tmp2 = nullptr; - NYI_WASM_SIMD("gtNewSimdNarrowNode"); - return nullptr; + if (varTypeIsFloating(simdBaseType)) + { + assert(simdBaseType == TYP_FLOAT); + + // WASM's f32x4.demote_f64x2_zero narrows each double vector to [f, f, 0, 0], so demote both + // operands and interleave their low pairs into a single result. + GenTree* lower = gtNewSimdHWIntrinsicNode(type, op1, NI_PackedSimd_ConvertToSingle, TYP_DOUBLE, simdSize); + GenTree* upper = gtNewSimdHWIntrinsicNode(type, op2, NI_PackedSimd_ConvertToSingle, TYP_DOUBLE, simdSize); + + uint32_t selectors[4] = {0, 1, 4, 5}; + return gtNewSimdWasmTwoSourceShuffleNode(type, lower, upper, selectors, TYP_FLOAT, simdSize); + } + + // Integer narrowing truncates each wide element to its low bytes. Gather those low bytes from + // both operands with a byte-granularity two-source shuffle. + uint32_t narrowSize = genTypeSize(simdBaseType); + uint32_t wideSize = narrowSize * 2; + uint32_t wideCount = simdSize / wideSize; + + uint32_t selectors[16]; + + for (uint32_t index = 0; index < simdSize; index++) + { + uint32_t narrowElem = index / narrowSize; + uint32_t byteInElem = index % narrowSize; + uint32_t srcElem = (narrowElem < wideCount) ? narrowElem : (narrowElem - wideCount); + uint32_t srcBase = (narrowElem < wideCount) ? 0 : simdSize; + + selectors[index] = srcBase + (srcElem * wideSize) + byteInElem; + } + + return gtNewSimdWasmTwoSourceShuffleNode(type, op1, op2, selectors, TYP_UBYTE, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 } +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdNarrowWithSaturationNode: Creates a new simd NarrowWithSaturation node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The first operand (narrowed into the lower half of the result) +// op2 - The second operand (narrowed into the upper half of the result) +// simdBaseType - The base type of the source (wider) elements +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created NarrowWithSaturation node +// +// Remarks: +// This is the portable fallback that clamps each source element to the representable range of the +// narrow type and then performs a non-saturating narrow. It is shared by architectures that lack a +// dedicated saturating-narrow instruction for the requested case. +// +GenTree* Compiler::gtNewSimdNarrowWithSaturationNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + + assert(op1 != nullptr); + assert(op1->TypeIs(type)); + + assert(op2 != nullptr); + assert(op2->TypeIs(type)); + + assert(varTypeIsArithmetic(simdBaseType)); + + if (varTypeIsFloating(simdBaseType)) + { + // Narrowing double to float already saturates out-of-range magnitudes to +/-infinity, which + // matches the managed semantics, so it can reuse the plain narrow. + return gtNewSimdNarrowNode(type, op1, op2, TYP_FLOAT, simdSize); + } + + // gtNewSimdNarrowNode uses the base type of the return for the simdBaseType + var_types narrowSimdBaseType; + + GenTreeVecCon* minCns = varTypeIsSigned(simdBaseType) ? gtNewVconNode(type) : nullptr; + GenTreeVecCon* maxCns = gtNewVconNode(type); + + switch (simdBaseType) + { + case TYP_SHORT: + { + minCns->EvaluateBroadcastInPlace(INT8_MIN); + maxCns->EvaluateBroadcastInPlace(INT8_MAX); + + narrowSimdBaseType = TYP_BYTE; + break; + } + + case TYP_USHORT: + { + maxCns->EvaluateBroadcastInPlace(UINT8_MAX); + narrowSimdBaseType = TYP_UBYTE; + break; + } + + case TYP_INT: + { + minCns->EvaluateBroadcastInPlace(INT16_MIN); + maxCns->EvaluateBroadcastInPlace(INT16_MAX); + + narrowSimdBaseType = TYP_SHORT; + break; + } + + case TYP_UINT: + { + maxCns->EvaluateBroadcastInPlace(UINT16_MAX); + narrowSimdBaseType = TYP_USHORT; + break; + } + + case TYP_LONG: + { + minCns->EvaluateBroadcastInPlace(INT32_MIN); + maxCns->EvaluateBroadcastInPlace(INT32_MAX); + + narrowSimdBaseType = TYP_INT; + break; + } + + case TYP_ULONG: + { + maxCns->EvaluateBroadcastInPlace(UINT32_MAX); + narrowSimdBaseType = TYP_UINT; + break; + } + + default: + { + unreached(); + } + } + + // This does a clamp which is defined as: Min(Max(value, min), max) + // which means that we do a max computation if a minimum constant is specified + // There will be none specified for unsigned to unsigned narrowing since + // they share a lower bound (0) and will already be correct. + + if (minCns != nullptr) + { + op1 = gtNewSimdMinMaxNode(type, op1, minCns, simdBaseType, simdSize, /* isMax */ true, + /* isMagnitude */ false, /* isNumber */ false); + op2 = gtNewSimdMinMaxNode(type, op2, gtCloneExpr(minCns), simdBaseType, simdSize, /* isMax */ true, + /* isMagnitude */ false, /* isNumber */ false); + } + + op1 = gtNewSimdMinMaxNode(type, op1, maxCns, simdBaseType, simdSize, /* isMax */ false, + /* isMagnitude */ false, /* isNumber */ false); + op2 = gtNewSimdMinMaxNode(type, op2, gtCloneExpr(maxCns), simdBaseType, simdSize, /* isMax */ false, + /* isMagnitude */ false, /* isNumber */ false); + + return gtNewSimdNarrowNode(type, op1, op2, narrowSimdBaseType, simdSize); +} + //---------------------------------------------------------------------------------------------- // Compiler::gtNewSimdCreateGeometricSequenceNode: Creates a new simd CreateGeometricSequence node // @@ -27169,6 +27349,70 @@ GenTree* Compiler::gtNewSimdCreateAlternatingSequenceNode( return gtNewSimdZipNode(type, even, odd, simdBaseType, simdSize, false); } +#if defined(TARGET_WASM) +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdWasmTwoSourceShuffleNode: Builds a two-source constant permute for WASM +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The first source vector (logical lanes 0 .. simdCount - 1) +// op2 - The second source vector (logical lanes simdCount .. 2 * simdCount - 1) +// selectors - simdCount lane selectors, each in [0, 2 * simdCount) +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// A node that, for each result lane i, selects logical lane selectors[i] from the concatenation +// of op1 and op2 +// +// Remarks: +// The general shuffle IR is single-source; WASM's native two-source constant permute +// (i8x16.shuffle) is only reachable via the WASM-specific PackedSimd.Shuffle intrinsic. +// To stay in the general, foldable representation, express the permute as two single-source +// shuffles -- each scatters the lanes drawn from one operand into place while out-of-range +// indices zero-fill the rest -- combined with a bitwise OR. +// +GenTree* Compiler::gtNewSimdWasmTwoSourceShuffleNode( + var_types type, GenTree* op1, GenTree* op2, const uint32_t* selectors, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + var_types indexType = getIndexTypeForShuffle(simdBaseType); + + GenTreeVecCon* indices1 = gtNewVconNode(type); + GenTreeVecCon* indices2 = gtNewVconNode(type); + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t selector = selectors[index]; + assert(selector < (2 * simdCount)); + + // The operand that does not supply this lane gets an out-of-range index so it zero-fills. + if (selector < simdCount) + { + indices1->SetElementIntegral(indexType, index, selector); + indices2->SetElementIntegral(indexType, index, simdCount); + } + else + { + indices1->SetElementIntegral(indexType, index, simdCount); + indices2->SetElementIntegral(indexType, index, selector - simdCount); + } + } + + assert(IsValidForShuffle(indices1, simdSize, simdBaseType, nullptr, false)); + assert(IsValidForShuffle(indices2, simdSize, simdBaseType, nullptr, false)); + + GenTree* scatter1 = gtNewSimdShuffleNode(type, op1, indices1, simdBaseType, simdSize, false); + GenTree* scatter2 = gtNewSimdShuffleNode(type, op2, indices2, simdBaseType, simdSize, false); + + return gtNewSimdBinOpNode(GT_OR, type, scatter1, scatter2, simdBaseType, simdSize); +} +#endif // TARGET_WASM + //---------------------------------------------------------------------------------------------- // Compiler::gtNewSimdConcatNode: Creates a new simd ConcatLowerLower/... node // @@ -27251,8 +27495,20 @@ GenTree* Compiler::gtNewSimdConcatNode(var_types type, simdSize); } #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdConcatNode"); - return nullptr; + // WASM has no native cross-vector concat. The lower result half comes from op1 and the upper + // from op2, so build the selectors and let the shared two-source shuffle scatter them. + + uint32_t selectors[16]; + uint32_t half = elementCount / 2; + uint32_t leftStart = leftUpper ? half : 0; + uint32_t rightStart = rightUpper ? half : 0; + + for (uint32_t index = 0; index < elementCount; index++) + { + selectors[index] = (index < half) ? (leftStart + index) : (elementCount + rightStart + (index - half)); + } + + return gtNewSimdWasmTwoSourceShuffleNode(type, op1, op2, selectors, simdBaseType, simdSize); #elif !defined(TARGET_ARM64) #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -27398,8 +27654,19 @@ GenTree* Compiler::gtNewSimdZipNode( NamedIntrinsic intrinsic = upper ? NI_AdvSimd_Arm64_ZipHigh : NI_AdvSimd_Arm64_ZipLow; return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdZipNode"); - return nullptr; + // WASM lacks a native interleave. Even lanes come from op1 and odd lanes from op2, so build the + // selectors and let the shared two-source shuffle scatter them. + + uint32_t selectors[16]; + uint32_t base = upper ? (simdCount / 2) : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t element = base + (index / 2); + selectors[index] = ((index & 1) == 0) ? element : (simdCount + element); + } + + return gtNewSimdWasmTwoSourceShuffleNode(type, op1, op2, selectors, simdBaseType, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 && !TARGET_WASM @@ -27590,8 +27857,20 @@ GenTree* Compiler::gtNewSimdUnzipNode( : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector_ToVector512Unsafe, simdBaseType, simdSize / 2); return gtNewSimdWithUpperNode(type, result, higher, simdBaseType, simdSize); #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdUnzipNode"); - return nullptr; + // WASM lacks a native deinterleave. The lower result half gathers op1's even/odd elements and + // the upper half gathers op2's, so build the selectors and let the shared two-source shuffle + // scatter them. + + uint32_t selectors[16]; + uint32_t half = simdCount / 2; + uint32_t start = odd ? 1 : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + selectors[index] = (index < half) ? (start + (2 * index)) : (simdCount + start + (2 * (index - half))); + } + + return gtNewSimdWasmTwoSourceShuffleNode(type, op1, op2, selectors, simdBaseType, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 && !TARGET_WASM @@ -27748,7 +28027,7 @@ GenTree* Compiler::gtNewSimdShuffleVariableNode( #elif defined(TARGET_ARM64) if ((!isShuffleNative) && (elementSize > 1)) #elif defined(TARGET_WASM) - if (!isShuffleNative) + if ((!isShuffleNative) && (elementSize > 1)) #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -28222,8 +28501,65 @@ GenTree* Compiler::gtNewSimdShuffleVariableNode( retNode = gtNewSimdHWIntrinsicNode(type, op1, op2, lookupIntrinsic, simdBaseType, simdSize); #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdShuffleVariableNode"); - retNode = op1; + assert(simdSize == 16); + + // Swizzle (i8x16.swizzle) operates on byte indices, so for non-byte element types we expand each + // element index into the equivalent run of byte indices and then perform a single byte-granular + // lookup. e.g., 3 2 1 0 (int) -> 12 13 14 15 8 9 10 11 4 5 6 7 0 1 2 3 (byte) + if (elementSize > 1) + { + // ShiftLeft is only valid on integral types + if (varTypeIsFloating(simdBaseType)) + { + if (elementSize == 4) + { + simdBaseType = TYP_INT; + } + else + { + assert(elementSize == 8); + simdBaseType = TYP_LONG; + } + } + + // scale each element index into the byte offset of its low byte (index * elementSize) + cnsNode = gtNewIconNode(BitOperations::TrailingZeroCount(static_cast(elementSize)), TYP_INT); + op2 = gtNewSimdHWIntrinsicNode(type, op2, cnsNode, NI_PackedSimd_ShiftLeft, simdBaseType, simdSize); + + // Swizzle is only valid on byte/sbyte + simdBaseType = varTypeIsUnsigned(simdBaseType) ? TYP_UBYTE : TYP_BYTE; + + // broadcast the low byte of each element to every byte of that element + simd_t shufCns = {}; + for (size_t index = 0; index < elementCount; index++) + { + for (size_t i = 0; i < elementSize; i++) + { + shufCns.u8[(index * elementSize) + i] = static_cast(index * elementSize); + } + } + + cnsNode = gtNewVconNode(type); + cnsNode->AsVecCon()->gtSimdVal = shufCns; + + op2 = gtNewSimdHWIntrinsicNode(type, op2, cnsNode, NI_PackedSimd_Swizzle, simdBaseType, simdSize); + + // or in the per-byte offset within each element + simd_t orCns = {}; + for (size_t index = 0; index < simdSize; index++) + { + orCns.u8[index] = static_cast(index & (elementSize - 1)); + } + + cnsNode = gtNewVconNode(type); + cnsNode->AsVecCon()->gtSimdVal = orCns; + + op2 = gtNewSimdBinOpNode(GT_OR, type, op2, cnsNode, simdBaseType, simdSize); + } + + // Swizzle selects zero for any byte index that is out of range (>= 16), which matches Shuffle for + // byte element indices. Larger element types are normalized by the shared masking step below. + retNode = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_PackedSimd_Swizzle, simdBaseType, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -28234,7 +28570,7 @@ GenTree* Compiler::gtNewSimdShuffleVariableNode( #elif defined(TARGET_ARM64) if ((!isShuffleNative) && (elementSize > 1)) #elif defined(TARGET_WASM) - if (!isShuffleNative) + if ((!isShuffleNative) && (elementSize > 1)) #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -28906,8 +29242,40 @@ GenTree* Compiler::gtNewSimdShuffleNode( return gtNewSimdHWIntrinsicNode(type, op1, op2, lookupIntrinsic, simdBaseType, simdSize); #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdShuffleNode"); - return nullptr; + assert(simdSize == 16); + + uint64_t value = 0; + simd_t vecCns = {}; + + for (size_t index = 0; index < elementCount; index++) + { + value = op2->GetIntegralVectorConstElement(index, simdBaseType); + + if (value < elementCount) + { + for (uint32_t i = 0; i < elementSize; i++) + { + vecCns.u8[(index * elementSize) + i] = (uint8_t)((value * elementSize) + i); + } + } + else + { + // Swizzle selects zero for any byte index that is out of range (>= 16), so mark every + // byte of an out-of-range element accordingly. + for (uint32_t i = 0; i < elementSize; i++) + { + vecCns.u8[(index * elementSize) + i] = 0xFF; + } + } + } + + // Swizzle operates on bytes, so reinterpret the shuffle as a byte-granular selection. + simdBaseType = varTypeIsUnsigned(simdBaseType) ? TYP_UBYTE : TYP_BYTE; + + op2 = gtNewVconNode(type); + op2->AsVecCon()->gtSimdVal = vecCns; + + return gtNewSimdHWIntrinsicNode(type, op1, op2, NI_PackedSimd_Swizzle, simdBaseType, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -29337,8 +29705,35 @@ GenTree* Compiler::gtNewSimdSumNode(var_types type, GenTree* op1, var_types simd } } #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdSumNode"); - return nullptr; + // WASM only has V128 and no native horizontal reduction, so reduce via a shuffle + add + // tree, reading lane 0 at the end. Using increasing strides keeps the grouping identical + // to the software `Sum(lower) + Sum(upper)` fallback, which matters for floating-point + // determinism. Indices that walk off the end shuffle in a zero, which is harmless because + // only lane 0 is ever read. + + unsigned vectorLength = getSIMDVectorLength(simdSize, simdBaseType); + var_types indexType = getIndexTypeForShuffle(simdBaseType); + + for (unsigned stride = 1; stride < vectorLength; stride *= 2) + { + GenTree* op1Dup = fgMakeMultiUse(&op1); + + GenTreeVecCon* indices = gtNewVconNode(simdType); + + for (unsigned index = 0; index < vectorLength; index++) + { + indices->SetElementIntegral(indexType, index, index + stride); + } + + assert(IsValidForShuffle(indices, simdSize, simdBaseType, nullptr, false)); + + GenTree* shifted = gtNewSimdShuffleNode(simdType, op1, indices, simdBaseType, simdSize, false); + // `shifted` carries the multi-use definition of `op1`, so it must be the first operand to + // ensure the store is evaluated before `op1Dup` reads it. Element-wise add is commutative. + op1 = gtNewSimdBinOpNode(GT_ADD, simdType, shifted, op1Dup, simdBaseType, simdSize); + } + + return gtNewSimdToScalarNode(type, op1, simdBaseType, simdSize); #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -29948,8 +30343,23 @@ GenTree* Compiler::gtNewSimdWidenUpperNode(var_types type, GenTree* op1, var_typ #elif defined(TARGET_WASM) if (varTypeIsFloating(simdBaseType)) { - NYI_WASM_SIMD("gtNewSimdWidenUpperNode"); - return nullptr; + assert(simdBaseType == TYP_FLOAT); + + // WASM only has f64x2.promote_low_f32x4, so move the upper two floats into the low lanes + // with a shuffle and then promote them; the resulting upper lanes are unused by the promote. + var_types indexType = getIndexTypeForShuffle(simdBaseType); + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + GenTreeVecCon* indices = gtNewVconNode(type); + + for (uint32_t index = 0; index < simdCount; index++) + { + indices->SetElementIntegral(indexType, index, (simdCount / 2) + (index % (simdCount / 2))); + } + + assert(IsValidForShuffle(indices, simdSize, simdBaseType, nullptr, false)); + + op1 = gtNewSimdShuffleNode(type, op1, indices, simdBaseType, simdSize, false); + return gtNewSimdHWIntrinsicNode(type, op1, NI_PackedSimd_ConvertToDoubleLower, simdBaseType, simdSize); } else if (varTypeIsSigned(simdBaseType)) { @@ -30004,7 +30414,9 @@ GenTree* Compiler::gtNewSimdWithElementNode( hwIntrinsicID = NI_AdvSimd_Insert; #elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtNewSimdWithElementNode"); + // Keep the platform-neutral NI_Vector_WithElement so that constant folding + // (gtFoldExprHWIntrinsic) and value numbering apply; it is rewritten to + // NI_PackedSimd_ReplaceScalar during lowering. #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 @@ -34916,8 +35328,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) otherNode->AsVecCon()->gtSimd16Val); otherNode->AsVecCon()->gtSimd16Val = result; } -#elif defined(TARGET_WASM) - NYI_WASM_SIMD("gtFoldExprHWIntrinsic: LSH/RSH/RSZ"); #endif // TARGET_XARCH } @@ -35723,11 +36133,9 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) else { assert(opCount == 3); -#ifndef TARGET_WASM - // TODO-WASM: Implement gtFoldExprHWIntrinsic for Wasm SIMD ternary operations switch (ni) { -#if defined(TARGET_XARCH) +#if defined(TARGET_XARCH) || defined(TARGET_WASM) case NI_Vector_ConditionalSelect: #elif defined(TARGET_ARM64) case NI_AdvSimd_BitwiseSelect: @@ -36053,7 +36461,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) break; } } -#endif // !defined(TARGET_WASM) } #if FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index ace8883a09a500..6b2c5f2d95e70d 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -6375,6 +6375,23 @@ struct GenTreeMultiOp : public GenTree return m_operands + startIndex; } + // Re-point "m_operands" into this node after its raw bytes were copied from + // "src" (e.g. by GenTree::ReplaceWith). When a MultiOp stores its operands + // inline, "m_operands" points into the node's own storage, so a byte copy + // leaves it aliasing "src". A heap operand array lives outside "src" and is + // left untouched. + void RelocateInlineOperandsFrom(GenTree* src) + { + char* srcBegin = reinterpret_cast(src); + char* srcEnd = srcBegin + src->GetNodeSize(); + char* operands = reinterpret_cast(m_operands); + + if ((operands >= srcBegin) && (operands < srcEnd)) + { + m_operands = reinterpret_cast(reinterpret_cast(this) + (operands - srcBegin)); + } + } + protected: // Reconfigures the operand array, leaving it in a "dirty" state. void ResetOperandArray(size_t newOperandCount, diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 1624111f413ccc..5d5f16eb836b37 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -3342,11 +3342,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(simdBaseType == TYP_FLOAT); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_AsVector128(Vector2) - Need WithElement - return nullptr; -#endif - op1 = impSIMDPopStack(); if (op1->IsCnsVec()) @@ -3381,11 +3376,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(simdBaseType == TYP_FLOAT); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_AsVector128(Vector3) - Need WithElement - return nullptr; -#endif - op1 = impSIMDPopStack(); if (op1->IsCnsVec()) @@ -3446,11 +3436,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, assert(simdBaseType == TYP_FLOAT); assert((simdSize == 8) || (simdSize == 12)); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_AsVector128Unsafe - return nullptr; -#endif - op1 = impSIMDPopStack(); retNode = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector_AsVector128Unsafe, simdBaseType, simdSize); break; @@ -3462,11 +3447,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, assert((simdSize == 16) && (simdBaseType == TYP_FLOAT)); assert((retType == TYP_SIMD8) || (retType == TYP_SIMD12)); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_AsVector2/3 - return nullptr; -#endif - assert(sig->numArgs == 1); op1 = impSIMDPopStack(); @@ -3572,7 +3552,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, intrinsic = NI_AdvSimd_Arm64_ConvertToDoubleScalar; } #elif defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ConvertToDouble + // WASM SIMD has no i64/u64 to f64 lane conversion, so fall back to the software path return nullptr; #else unreached(); @@ -3621,7 +3601,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, #endif #if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ConvertToInt64 + // WASM SIMD has no f64 to i64 lane conversion, so fall back to the software path return nullptr; #endif @@ -3648,7 +3628,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, #endif #if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ConvertToInt64Native + // WASM SIMD has no f64 to i64 lane conversion, so fall back to the software path return nullptr; #endif @@ -3764,7 +3744,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, #endif #if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ConvertToUInt64 + // WASM SIMD has no f64 to u64 lane conversion, so fall back to the software path return nullptr; #endif @@ -3791,7 +3771,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, #endif #if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ConvertToUInt64Native + // WASM SIMD has no f64 to u64 lane conversion, so fall back to the software path return nullptr; #endif @@ -3813,14 +3793,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, impSpillSideEffect(true, stackState.esStackDepth - 2 DEBUGARG("Spilling op1 side effects for vector CreateAlternatingSequence")); -#if defined(TARGET_WASM) - if (!impStackTop(0).val->OperIsConst() || !impStackTop(1).val->OperIsConst()) - { - // TODO-WASM-SIMD: Implement NI_Vector_CreateAlternatingSequence - Need Shuffle - return nullptr; - } -#endif - op2 = impPopStack().val; op1 = impPopStack().val; @@ -3874,14 +3846,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - if (!impStackTop(0).val->OperIsConst()) - { - // TODO-WASM-SIMD: Implement NI_Vector_CreateScalar - break; - } -#endif - op1 = impPopStack().val; retNode = gtNewSimdCreateScalarNode(retType, op1, simdBaseType, simdSize); break; @@ -3891,14 +3855,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - if (!impStackTop(0).val->OperIsConst()) - { - // TODO-WASM-SIMD: Implement NI_Vector_CreateScalarUnsafe - break; - } -#endif - op1 = impPopStack().val; retNode = gtNewSimdCreateScalarUnsafeNode(retType, op1, simdBaseType, simdSize); break; @@ -3930,11 +3886,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Dot - Need Shuffle - return nullptr; -#endif - #if defined(TARGET_ARM64) if (varTypeIsLong(simdBaseType)) { @@ -3947,7 +3898,12 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, op2 = impSIMDPopStack(); op1 = impSIMDPopStack(); -#if defined(TARGET_XARCH) +#if defined(TARGET_WASM) + // WASM has no native horizontal reduction, so import as Sum(left * right). + retNode = gtNewSimdBinOpNode(GT_MUL, simdType, op1, op2, simdBaseType, simdSize); + retNode = gtNewSimdSumNode(retType, retNode, simdBaseType, simdSize); + break; +#elif defined(TARGET_XARCH) if ((simdSize == 64) || varTypeIsByte(simdBaseType) || varTypeIsLong(simdBaseType)) { // The lowering for Dot doesn't handle these cases, so import as Sum(left * right) @@ -4088,7 +4044,8 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, impSpillSideEffect(true, stackState.esStackDepth - 2 DEBUGARG("Spilling op2 side effects for FusedMultiplyAdd")); #elif defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_FusedMultiplyAdd + // WASM SIMD has no fused-multiply-add; emulating it as a separate multiply and add would + // round twice and break the single-rounding guarantee, so fall back to the software path return nullptr; #endif @@ -4104,11 +4061,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_GetElement - return nullptr; -#endif - op2 = impPopStack().val; op1 = impSIMDPopStack(); @@ -4610,11 +4562,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Narrow - return nullptr; -#endif - op2 = impSIMDPopStack(); op1 = impSIMDPopStack(); @@ -4754,87 +4701,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, } else { - // gtNewSimdNarrowNode uses the base type of the return for the simdBaseType - var_types narrowSimdBaseType; - - GenTreeVecCon* minCns = varTypeIsSigned(simdBaseType) ? gtNewVconNode(retType) : nullptr; - GenTreeVecCon* maxCns = gtNewVconNode(retType); - - switch (simdBaseType) - { - case TYP_SHORT: - { - minCns->EvaluateBroadcastInPlace(INT8_MIN); - maxCns->EvaluateBroadcastInPlace(INT8_MAX); - - narrowSimdBaseType = TYP_BYTE; - break; - } - - case TYP_USHORT: - { - maxCns->EvaluateBroadcastInPlace(UINT8_MAX); - narrowSimdBaseType = TYP_UBYTE; - break; - } - - case TYP_INT: - { - minCns->EvaluateBroadcastInPlace(INT16_MIN); - maxCns->EvaluateBroadcastInPlace(INT16_MAX); - - narrowSimdBaseType = TYP_SHORT; - break; - } - - case TYP_UINT: - { - maxCns->EvaluateBroadcastInPlace(UINT16_MAX); - narrowSimdBaseType = TYP_USHORT; - break; - } - - case TYP_LONG: - { - minCns->EvaluateBroadcastInPlace(INT32_MIN); - maxCns->EvaluateBroadcastInPlace(INT32_MAX); - - narrowSimdBaseType = TYP_INT; - break; - } - - case TYP_ULONG: - { - maxCns->EvaluateBroadcastInPlace(UINT32_MAX); - narrowSimdBaseType = TYP_UINT; - break; - } - - default: - { - unreached(); - } - } - - // This does a clamp which is defined as: Min(Max(value, min), max) - // which means that we do a max computation if a minimum constant is specified - // There will be none specified for unsigned to unsigned narrowing since - // they share a lower bound (0) and will already be correct. - - if (minCns != nullptr) - { - op1 = gtNewSimdMinMaxNode(retType, op1, minCns, simdBaseType, simdSize, /* isMax */ true, - /* isMagnitude */ false, /* isNumber */ false); - op2 = gtNewSimdMinMaxNode(retType, op2, gtCloneExpr(minCns), simdBaseType, simdSize, - /* isMax */ true, /* isMagnitude */ false, /* isNumber */ false); - } - - op1 = gtNewSimdMinMaxNode(retType, op1, maxCns, simdBaseType, simdSize, /* isMax */ false, - /* isMagnitude */ false, /* isNumber */ false); - op2 = gtNewSimdMinMaxNode(retType, op2, gtCloneExpr(maxCns), simdBaseType, simdSize, - /* isMax */ false, /* isMagnitude */ false, /* isNumber */ false); - - retNode = gtNewSimdNarrowNode(retType, op1, op2, narrowSimdBaseType, simdSize); + retNode = gtNewSimdNarrowWithSaturationNode(retType, op1, op2, simdBaseType, simdSize); } #elif defined(TARGET_ARM64) op2 = impSIMDPopStack(); @@ -4863,8 +4730,10 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseType, simdSize); } #elif defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_NarrowWithSaturation - return nullptr; + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + retNode = gtNewSimdNarrowWithSaturationNode(retType, op1, op2, simdBaseType, simdSize); #else unreached(); #endif @@ -4875,11 +4744,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Reverse - Need Shuffle - return nullptr; -#endif - #if defined(TARGET_XARCH) if ((simdSize == 64) && varTypeIsByte(simdBaseType)) { @@ -4925,7 +4789,8 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, } #if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ShiftLeft + // WASM SIMD only supports shift-by-scalar, not a per-lane variable shift, so fall back to + // the software path return nullptr; #endif @@ -4960,7 +4825,7 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, intrinsic = varTypeIsLong(simdBaseType) ? NI_AdvSimd_ShiftLogicalScalar : NI_AdvSimd_ShiftLogical; } #elif defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ShiftLeft + // Unreachable: WASM bails out to the software path above, before the operands are popped return nullptr; #else unreached(); @@ -4976,15 +4841,10 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert((sig->numArgs == 2) || (sig->numArgs == 3)); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Shuffle - return nullptr; -#endif - bool isShuffleNative = (intrinsic != NI_Vector_Shuffle); bool isNonDeterministic = isShuffleNative; -#if defined(TARGET_ARM64) +#if defined(TARGET_ARM64) || defined(TARGET_WASM) if (isNonDeterministic) { isNonDeterministic = genTypeSize(simdBaseType) > 1; @@ -5298,11 +5158,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Sum - Need Shuffle - return nullptr; -#endif - op1 = impSIMDPopStack(); retNode = gtNewSimdSumNode(retType, op1, simdBaseType, simdSize); break; @@ -5312,11 +5167,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_ToScalar - Need GetElement - return nullptr; -#endif - op1 = impSIMDPopStack(); retNode = gtNewSimdToScalarNode(retType, op1, simdBaseType, simdSize); break; @@ -5342,11 +5192,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Unzip - Need Shuffle - return nullptr; -#endif - #if defined(TARGET_XARCH) if (simdSize == 16) { @@ -5379,14 +5224,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 1); -#if defined(TARGET_WASM) - if (simdBaseType == TYP_FLOAT) - { - // TODO-WASM-SIMD - Implement NI_Vector_WidenUpper(float) - Need Shuffle - return nullptr; - } -#endif - op1 = impSIMDPopStack(); retNode = gtNewSimdWidenUpperNode(retType, op1, simdBaseType, simdSize); break; @@ -5396,11 +5233,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 3); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_WithElement - return nullptr; -#endif - #if defined(TARGET_X86) if (varTypeIsLong(simdBaseType)) { @@ -5493,11 +5325,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Zip - Need Shuffle - return nullptr; -#endif - op2 = impSIMDPopStack(); op1 = impSIMDPopStack(); @@ -5799,11 +5626,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_op_LeftShift - return nullptr; -#endif - op2 = impPopStack().val; op1 = impSIMDPopStack(); @@ -5842,11 +5664,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_op_RightShift - return nullptr; -#endif - genTreeOps op = varTypeIsUnsigned(simdBaseType) ? GT_RSZ : GT_RSH; op2 = impPopStack().val; @@ -5886,11 +5703,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, { assert(sig->numArgs == 2); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_op_UnsignedRightShift - return nullptr; -#endif - op2 = impPopStack().val; op1 = impSIMDPopStack(); @@ -5933,11 +5745,6 @@ GenTree* Compiler::impXplatIntrinsic(NamedIntrinsic intrinsic, assert(sig->numArgs == 2); assert(retNode == nullptr); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_Concat - Need Shuffle - return nullptr; -#endif - op2 = impSIMDPopStack(); op1 = impSIMDPopStack(); diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index d9d47e7914db7f..da4666b5e5d3e5 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -40,7 +40,22 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { case HW_Category_SIMD: { - GetEmitter()->emitIns(ins); + if ((info.id == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) + { + // A constant, fully in-range mask was lowered to an immediate i8x16.shuffle. + // genConsumeMultiOpOperands left the source on the value stack once (the mask + // operand is contained, so no v128.const was materialized). i8x16.shuffle + // selects from two vectors, so push the source a second time and encode the + // mask as the 16-byte shuffle immediate. + GenTree* src = node->Op(1); + regNumber srcReg = GetMultiUseOperandReg(src); + GetEmitter()->emitIns_I(INS_local_get, emitActualTypeSize(src), WasmRegToIndex(srcReg)); + GetEmitter()->emitIns_V128Imm(INS_i8x16_shuffle, node->Op(2)->AsVecCon()->gtSimdVal.u8); + } + else + { + GetEmitter()->emitIns(ins); + } break; } case HW_Category_IMM: @@ -63,7 +78,23 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } else { - NYI_WASM_SIMD("!codeGenIsTableDriven"); + switch (info.id) + { + case NI_Vector_AsVector128Unsafe: + case NI_Vector_AsVector2: + case NI_Vector_AsVector3: + { + // These are pure reinterprets between SIMD widths. Every SIMD type occupies a + // full v128 on the value stack, so the consumed operand already is the result and + // there is nothing to emit. + break; + } + + default: + { + NYI_WASM_SIMD("!codeGenIsTableDriven"); + } + } } WasmProduceReg(node); diff --git a/src/coreclr/jit/hwintrinsiclist.h b/src/coreclr/jit/hwintrinsiclist.h index e9ffeba17c0411..d838afa24e8f74 100644 --- a/src/coreclr/jit/hwintrinsiclist.h +++ b/src/coreclr/jit/hwintrinsiclist.h @@ -62,9 +62,9 @@ HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #else -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #endif HARDWARE_INTRINSIC(Vector, AsVector4, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index b1322a05ef7c76..bbc7360ce121f6 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -5279,29 +5279,19 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, else if (!isNative || !BlockNonDeterministicIntrinsics(mustExpand)) { #if defined(FEATURE_HW_INTRINSICS) -#if !defined(TARGET_WASM) GenTree* op2 = impImplicitR4orR8Cast(impPopStack().val, callType); GenTree* op1 = impImplicitR4orR8Cast(impPopStack().val, callType); -#endif if (isNative) { assert(!isMagnitude && !isNumber); -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_MinMax - Need GetElement -#else retNode = gtNewSimdMinMaxNativeNode(callType, op1, op2, JitType2PreciseVarType(callJitType), 0, isMax); -#endif } else { -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Implement NI_Vector_MinMax - Need GetElement -#else retNode = gtNewSimdMinMaxNode(callType, op1, op2, JitType2PreciseVarType(callJitType), 0, isMax, isMagnitude, isNumber); -#endif } #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index 5b2a6225e841f5..41925ba6faed85 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -1961,11 +1961,6 @@ class LocalAddressVisitor final : public GenTreeVisitor return IndirTransform::LclFld; } -#if defined(TARGET_WASM) - // TODO-WASM-SIMD: Handle once GetElement and WithElement are supported - return IndirTransform::LclFld; -#endif - #ifdef FEATURE_HW_INTRINSICS if (varTypeIsSIMD(varDsc)) { diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 2926e1c2477562..0313bf4871087d 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -523,6 +523,7 @@ class Lowering final : public Phase #elif defined(TARGET_WASM) GenTree* LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node); + void LowerHWIntrinsicSwizzle(GenTreeHWIntrinsic* node); #endif // !TARGET_XARCH && !TARGET_ARM64 GenTree* InsertNewSimdCreateScalarUnsafeNode(var_types type, GenTree* op1, diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index da8e304283aed5..18227b108bf8d2 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -842,6 +842,14 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicCreate(node); } + case NI_Vector_CreateScalarUnsafe: + { + // CreateScalarUnsafe leaves the upper elements undefined, so we can broadcast the value + // across every lane. This maps directly to a single splat instruction. + node->ChangeHWIntrinsicId(NI_PackedSimd_Splat); + return LowerNode(node); + } + case NI_Vector_op_Equality: { assert(category == HW_Category_Helper); @@ -866,6 +874,31 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Vector_GetElement: + { + // GetElement(vector, index) maps directly to ExtractScalar(vector, imm). + node->ChangeHWIntrinsicId(NI_PackedSimd_ExtractScalar); + return LowerHWIntrinsicWithImm(node); + } + + case NI_Vector_ToScalar: + { + // ToScalar(vector) is GetElement(vector, 0), i.e. ExtractScalar(vector, 0). + GenTree* idx = m_compiler->gtNewIconNode(0); + BlockRange().InsertBefore(node, idx); + LowerNode(idx); + + node->ResetHWIntrinsicId(NI_PackedSimd_ExtractScalar, m_compiler, node->Op(1), idx); + return LowerHWIntrinsicWithImm(node); + } + + case NI_Vector_WithElement: + { + // WithElement(vector, index, value) maps directly to ReplaceScalar(vector, imm, value). + node->ChangeHWIntrinsicId(NI_PackedSimd_ReplaceScalar); + return LowerHWIntrinsicWithImm(node); + } + case NI_PackedSimd_ExtractScalar: case NI_PackedSimd_ReplaceScalar: { @@ -873,6 +906,13 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_Swizzle: + { + assert(category == HW_Category_SIMD); + LowerHWIntrinsicSwizzle(node); + return node->gtNext; + } + default: { assert(category == HW_Category_SIMD); @@ -914,6 +954,56 @@ GenTree* Lowering::LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node) return node->gtNext; } +//---------------------------------------------------------------------------------------------- +// LowerHWIntrinsicSwizzle: Rewrite a constant-mask PackedSimd Swizzle into an i8x16.shuffle. +// +// Wasm's i8x16.swizzle takes a runtime byte-index mask, whereas i8x16.shuffle encodes the 16 +// lane selectors as an immediate. When the mask is a compile-time constant with no out-of-range +// lanes, emitting i8x16.shuffle lets the underlying wasm engine pattern-match the known +// permutation into an optimal native sequence instead of a generic table lookup. +// +// i8x16.shuffle selects from two vectors (lanes 0-15 from the first, 16-31 from the second), so +// for a single-source swizzle we feed the source vector as both operands. On the wasm value +// stack that requires the source twice, so it is marked multiply-used (materialized into a local) +// and the codegen side emits the extra local.get. Out-of-range masks are left as a swizzle, whose +// native "index >= 16 -> 0" behavior i8x16.shuffle cannot reproduce without a zero operand. +// +// Arguments: +// node - The PackedSimd Swizzle node. +// +void Lowering::LowerHWIntrinsicSwizzle(GenTreeHWIntrinsic* node) +{ + assert(node->GetHWIntrinsicId() == NI_PackedSimd_Swizzle); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + + if (op2->IsCnsVec()) + { + const simd_t& mask = op2->AsVecCon()->gtSimdVal; + bool allInRange = true; + + for (int i = 0; i < 16; i++) + { + if (mask.u8[i] >= 16) + { + allInRange = false; + break; + } + } + + if (allInRange) + { + // The mask becomes the i8x16.shuffle immediate; the source is consumed as both + // shuffle operands, so it must be available twice on the value stack. + MakeSrcContained(node, op2); + SetMultiplyUsed(op1 DEBUGARG("i8x16.shuffle reuses the source as both operands")); + } + } + + ContainCheckHWIntrinsic(node); +} + //---------------------------------------------------------------------------------------------- // LowerHWIntrinsicCompareUnsignedLong: Rewrite a PackedSimd ordered ulong compare into a // signed compare on sign-bit-flipped operands. diff --git a/src/coreclr/jit/regallocwasm.cpp b/src/coreclr/jit/regallocwasm.cpp index 031f1cf229e202..860f8d8dbd0dc6 100644 --- a/src/coreclr/jit/regallocwasm.cpp +++ b/src/coreclr/jit/regallocwasm.cpp @@ -711,6 +711,15 @@ void WasmRegAlloc::CollectReferencesForLclVar(GenTreeLclVar* lclVar) // temporary registers for its operands. void WasmRegAlloc::CollectReferencesForHardwareIntrinsic(GenTreeHWIntrinsic* node) { + // A constant, in-range mask Swizzle is lowered to an immediate i8x16.shuffle, which reuses the + // source operand as both shuffle inputs. Lowering marked the source multiply-used (and contained + // the mask), so release its temporary register here. + if ((node->GetHWIntrinsicId() == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) + { + ConsumeTemporaryRegForOperand(node->Op(1) DEBUGARG("i8x16.shuffle source reuse")); + return; + } + // Only intrinsics with an immediate operand can need the jump-table fallback. if (!HWIntrinsicInfo::HasImmediateOperand(node->GetHWIntrinsicId())) { diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 99080376b6d95b..a9862e5571276f 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -8291,7 +8291,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunUnary(GenTreeHWIntrinsic* tree, #endif { -#ifdef FEATURE_MASKED_HW_INTRINSICS +#if defined(FEATURE_MASKED_HW_INTRINSICS) || defined(TARGET_WASM) simdmask_t simdMaskVal; switch (simdSize) @@ -8332,10 +8332,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunUnary(GenTreeHWIntrinsic* tree, assert(elemCount <= 32); return VNForIntCon(static_cast(mask)); -#elif defined(TARGET_WASM) - NYI_WASM_SIMD("Vector128_ExtractMostSignificantBits"); - break; -#endif // FEATURE_MASKED_HW_INTRINSICS +#endif // FEATURE_MASKED_HW_INTRINSICS || TARGET_WASM } #ifdef TARGET_XARCH @@ -9605,9 +9602,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( switch (ni) { -#ifndef TARGET_WASM -// TODO-WASM: Implement bitwise select case -#if defined(TARGET_XARCH) +#if defined(TARGET_XARCH) || defined(TARGET_WASM) case NI_Vector_ConditionalSelect: #elif defined(TARGET_ARM64) case NI_AdvSimd_BitwiseSelect: @@ -9673,7 +9668,6 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( } break; } -#endif // !defined(TARGET_WASM) case NI_Vector_WithElement: { diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index db66eb3b89a36c..cb08687fe338be 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -3,7 +3,9 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Numerics; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Wasm; using Xunit; @@ -250,6 +252,587 @@ public static unsafe void ExtractInsertScalarTest() Assert.Equal(Vector128.Create(1, 2, 10, 4), modified); } + [Fact] + public static unsafe void Vector128GetWithElementTest() + { + var vi = Vector128.Create(10, 20, 30, 40); + + // GetElement/WithElement with a constant index, and ToScalar (GetElement(0)). + Assert.Equal(10, vi.GetElement(0)); + Assert.Equal(30, vi.GetElement(2)); + Assert.Equal(10, vi.ToScalar()); + Assert.Equal(Vector128.Create(10, 20, 99, 40), vi.WithElement(2, 99)); + + // Small elements must sign/zero-extend based on the element type. + var vsb = Vector128.Create((sbyte)-1, -2, -3, -4, -5, -6, -7, -8, + -9, -10, -11, -12, -13, -14, -15, -16); + Assert.Equal((sbyte)-3, vsb.GetElement(2)); + + var vb = Vector128.Create((byte)255, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15); + Assert.Equal((byte)255, vb.GetElement(0)); + + // 64-bit and floating-point lanes. + var vl = Vector128.Create(1L, 2L); + Assert.Equal(2L, vl.GetElement(1)); + Assert.Equal(Vector128.Create(1L, 42L), vl.WithElement(1, 42L)); + + var vf = Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f); + Assert.Equal(3.0f, vf.GetElement(2)); + Assert.Equal(Vector128.Create(1.0f, 2.0f, 7.0f, 4.0f), vf.WithElement(2, 7.0f)); + + var vd = Vector128.Create(1.0, 2.0); + Assert.Equal(2.0, vd.GetElement(1)); + Assert.Equal(9.0, vd.WithElement(0, 9.0).ToScalar()); + + // A non-constant index exercises the jump-table fallback in codegen. + for (int i = 0; i < 4; i++) + { + Assert.Equal((i + 1) * 10, vi.GetElement(i)); + Assert.Equal(7, vi.WithElement(i, 7).GetElement(i)); + } + } + + [Fact] + public static unsafe void Vector128ShiftTest() + { + // Left shift (<<) with a constant count across element widths. + Assert.Equal(Vector128.Create(4, 8, 12, 16), Vector128.Create(1, 2, 3, 4) << 2); + Assert.Equal(Vector128.Create(2L, 4L), Vector128.Create(1L, 2L) << 1); + Assert.Equal(Vector128.Create((short)8, 16, 24, 32, 40, 48, 56, 64), + Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8) << 3); + + // Arithmetic right shift (>>) preserves sign for signed types. + Assert.Equal(Vector128.Create(-2, -1, 1, 2), Vector128.Create(-8, -4, 4, 8) >> 2); + Assert.Equal(Vector128.Create((sbyte)-1, -1, 0, 1, -2, 2, -4, 4, -1, -1, 0, 1, -2, 2, -4, 4), + Vector128.Create((sbyte)-2, -1, 1, 2, -4, 4, -8, 8, -2, -1, 1, 2, -4, 4, -8, 8) >> 1); + + // Unsigned/logical right shift (>>>) zero-fills. + Assert.Equal(Vector128.Create(0x3FFFFFFFu, 1u, 2u, 3u), + Vector128.Create(0xFFFFFFFFu, 4u, 8u, 12u) >>> 2); + Assert.Equal(Vector128.Create(unchecked((int)0x3FFFFFFF), 1, 2, 3), + Vector128.Create(unchecked((int)0xFFFFFFFF), 4, 8, 12) >>> 2); + + // Non-constant counts exercise the scalar-amount path. + var v = Vector128.Create(1, 2, 3, 4); + for (int i = 0; i < 4; i++) + { + Assert.Equal(Vector128.Create(1 << i, 2 << i, 3 << i, 4 << i), v << i); + Assert.Equal(Vector128.Create(16 >> i, 32 >> i, 48 >> i, 64 >> i), + Vector128.Create(16, 32, 48, 64) >> i); + Assert.Equal(Vector128.Create(16u >>> i, 32u >>> i, 48u >>> i, 64u >>> i), + Vector128.Create(16u, 32u, 48u, 64u) >>> i); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T Opaque(T value) => value; + + [Fact] + public static unsafe void Vector128CreateScalarTest() + { + // Non-constant operands force the CreateScalar/CreateScalarUnsafe lowering rather + // than constant folding to a vector constant. + + // CreateScalar zero-fills the upper elements. + Assert.Equal(Vector128.Create(42, 0, 0, 0), Vector128.CreateScalar(Opaque(42))); + Assert.Equal(Vector128.Create(7L, 0L), Vector128.CreateScalar(Opaque(7L))); + Assert.Equal(Vector128.Create((short)9, 0, 0, 0, 0, 0, 0, 0), + Vector128.CreateScalar(Opaque((short)9))); + Assert.Equal(Vector128.Create((byte)200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + Vector128.CreateScalar(Opaque((byte)200))); + Assert.Equal(Vector128.Create(3.5f, 0.0f, 0.0f, 0.0f), Vector128.CreateScalar(Opaque(3.5f))); + Assert.Equal(Vector128.Create(6.25, 0.0), Vector128.CreateScalar(Opaque(6.25))); + + // CreateScalarUnsafe leaves the upper elements undefined, so only lane 0 is guaranteed. + Assert.Equal(42, Vector128.CreateScalarUnsafe(Opaque(42)).ToScalar()); + Assert.Equal(7L, Vector128.CreateScalarUnsafe(Opaque(7L)).ToScalar()); + Assert.Equal((short)9, Vector128.CreateScalarUnsafe(Opaque((short)9)).ToScalar()); + Assert.Equal((byte)200, Vector128.CreateScalarUnsafe(Opaque((byte)200)).ToScalar()); + Assert.Equal(3.5f, Vector128.CreateScalarUnsafe(Opaque(3.5f)).ToScalar()); + Assert.Equal(6.25, Vector128.CreateScalarUnsafe(Opaque(6.25)).ToScalar()); + } + + [Fact] + public static unsafe void Vector2And3ConversionTest() + { + // Non-constant operands force the reinterpret nodes rather than constant folding. + + Vector128 v = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + + // Narrowing keeps the lower lanes and drops the rest. + Assert.Equal(new Vector2(1.0f, 2.0f), v.AsVector2()); + Assert.Equal(new Vector3(1.0f, 2.0f, 3.0f), v.AsVector3()); + + // AsVector128 widens and zero-fills the upper lanes. + Assert.Equal(Vector128.Create(1.0f, 2.0f, 0.0f, 0.0f), Opaque(new Vector2(1.0f, 2.0f)).AsVector128()); + Assert.Equal(Vector128.Create(1.0f, 2.0f, 3.0f, 0.0f), Opaque(new Vector3(1.0f, 2.0f, 3.0f)).AsVector128()); + + // AsVector128Unsafe leaves the upper lanes undefined, so only the lower lanes are guaranteed. + Vector128 u2 = Opaque(new Vector2(1.0f, 2.0f)).AsVector128Unsafe(); + Assert.Equal(1.0f, u2.GetElement(0)); + Assert.Equal(2.0f, u2.GetElement(1)); + + Vector128 u3 = Opaque(new Vector3(1.0f, 2.0f, 3.0f)).AsVector128Unsafe(); + Assert.Equal(1.0f, u3.GetElement(0)); + Assert.Equal(2.0f, u3.GetElement(1)); + Assert.Equal(3.0f, u3.GetElement(2)); + } + + [Fact] + public static unsafe void ConstantShuffleTest() + { + // Opaque data with constant indices exercises the Swizzle-based constant-shuffle path. + + Vector128 vi = Opaque(Vector128.Create(1, 2, 3, 4)); + Assert.Equal(Vector128.Create(4, 3, 2, 1), Vector128.Shuffle(vi, Vector128.Create(3, 2, 1, 0))); + // Out-of-range indices zero the corresponding element. + Assert.Equal(Vector128.Create(1, 2, 3, 0), Vector128.Shuffle(vi, Vector128.Create(0, 1, 2, 99))); + + Vector128 vf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Assert.Equal(Vector128.Create(4.0f, 3.0f, 2.0f, 1.0f), Vector128.Shuffle(vf, Vector128.Create(3, 2, 1, 0))); + + Vector128 vl = Opaque(Vector128.Create(10L, 20L)); + Assert.Equal(Vector128.Create(20L, 10L), Vector128.Shuffle(vl, Vector128.Create(1L, 0L))); + + Vector128 vs = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Assert.Equal(Vector128.Create((short)8, 7, 6, 5, 4, 3, 2, 1), + Vector128.Shuffle(vs, Vector128.Create((short)7, 6, 5, 4, 3, 2, 1, 0))); + + Vector128 vb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Assert.Equal(Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + Vector128.Shuffle(vb, Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0))); + // Out-of-range byte indices zero their element. + Assert.Equal(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0), + Vector128.Shuffle(vb, Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 200))); + } + + [Fact] + public static unsafe void VariableShuffleTest() + { + // Opaque indices force the variable-index path (byte-index expansion + i8x16.swizzle). + + Vector128 vi = Opaque(Vector128.Create(1, 2, 3, 4)); + Vector128 ii = Opaque(Vector128.Create(3, 2, 1, 0)); + Assert.Equal(Vector128.Create(4, 3, 2, 1), Vector128.Shuffle(vi, ii)); + // Out-of-range indices zero the corresponding element. + Assert.Equal(Vector128.Create(1, 2, 3, 0), Vector128.Shuffle(vi, Opaque(Vector128.Create(0, 1, 2, 99)))); + + Vector128 vf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Assert.Equal(Vector128.Create(4.0f, 3.0f, 2.0f, 1.0f), + Vector128.Shuffle(vf, Opaque(Vector128.Create(3, 2, 1, 0)))); + + Vector128 vl = Opaque(Vector128.Create(10L, 20L)); + Assert.Equal(Vector128.Create(20L, 10L), Vector128.Shuffle(vl, Opaque(Vector128.Create(1L, 0L)))); + // Out-of-range long index zeroes its element. + Assert.Equal(Vector128.Create(10L, 0L), Vector128.Shuffle(vl, Opaque(Vector128.Create(0L, 5L)))); + + Vector128 vs = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Assert.Equal(Vector128.Create((short)8, 7, 6, 5, 4, 3, 2, 1), + Vector128.Shuffle(vs, Opaque(Vector128.Create((short)7, 6, 5, 4, 3, 2, 1, 0)))); + // Out-of-range short index zeroes its element. + Assert.Equal(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 0), + Vector128.Shuffle(vs, Opaque(Vector128.Create((short)0, 1, 2, 3, 4, 5, 6, 99)))); + + Vector128 vb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Assert.Equal(Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + Vector128.Shuffle(vb, Opaque(Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)))); + // Out-of-range byte index zeroes its element. + Assert.Equal(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0), + Vector128.Shuffle(vb, Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 200)))); + } + + [Fact] + public static unsafe void ReverseTest() + { + // Reverse lowers to a constant Shuffle; Opaque data forces the shuffle rather than folding. + + Vector128 vi = Opaque(Vector128.Create(1, 2, 3, 4)); + Assert.Equal(Vector128.Create(4, 3, 2, 1), Vector128.Reverse(vi)); + + Vector128 vf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Assert.Equal(Vector128.Create(4.0f, 3.0f, 2.0f, 1.0f), Vector128.Reverse(vf)); + + Vector128 vl = Opaque(Vector128.Create(10L, 20L)); + Assert.Equal(Vector128.Create(20L, 10L), Vector128.Reverse(vl)); + + Vector128 vd = Opaque(Vector128.Create(1.5, 2.5)); + Assert.Equal(Vector128.Create(2.5, 1.5), Vector128.Reverse(vd)); + + Vector128 vs = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Assert.Equal(Vector128.Create((short)8, 7, 6, 5, 4, 3, 2, 1), Vector128.Reverse(vs)); + + Vector128 vb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Assert.Equal(Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), Vector128.Reverse(vb)); + } + + [Fact] + public static unsafe void SumTest() + { + // Opaque data forces the shuffle + add reduction rather than constant folding. + + Assert.Equal(10, Vector128.Sum(Opaque(Vector128.Create(1, 2, 3, 4)))); + Assert.Equal(10.0f, Vector128.Sum(Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)))); + Assert.Equal(30L, Vector128.Sum(Opaque(Vector128.Create(10L, 20L)))); + Assert.Equal(4.0, Vector128.Sum(Opaque(Vector128.Create(1.5, 2.5)))); + Assert.Equal((short)36, Vector128.Sum(Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)))); + Assert.Equal((byte)120, + Vector128.Sum(Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))); + } + + [Fact] + public static unsafe void ZipTest() + { + // Opaque operands force the shuffle + OR interleave rather than constant folding. + + Vector128 li = Opaque(Vector128.Create(1, 2, 3, 4)); + Vector128 ri = Opaque(Vector128.Create(5, 6, 7, 8)); + Assert.Equal(Vector128.Create(1, 5, 2, 6), Vector128.ZipLower(li, ri)); + Assert.Equal(Vector128.Create(3, 7, 4, 8), Vector128.ZipUpper(li, ri)); + + Vector128 lf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Vector128 rf = Opaque(Vector128.Create(5.0f, 6.0f, 7.0f, 8.0f)); + Assert.Equal(Vector128.Create(1.0f, 5.0f, 2.0f, 6.0f), Vector128.ZipLower(lf, rf)); + Assert.Equal(Vector128.Create(3.0f, 7.0f, 4.0f, 8.0f), Vector128.ZipUpper(lf, rf)); + + Vector128 ll = Opaque(Vector128.Create(10L, 20L)); + Vector128 rl = Opaque(Vector128.Create(30L, 40L)); + Assert.Equal(Vector128.Create(10L, 30L), Vector128.ZipLower(ll, rl)); + Assert.Equal(Vector128.Create(20L, 40L), Vector128.ZipUpper(ll, rl)); + + Vector128 ls = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Vector128 rs = Opaque(Vector128.Create((short)9, 10, 11, 12, 13, 14, 15, 16)); + Assert.Equal(Vector128.Create((short)1, 9, 2, 10, 3, 11, 4, 12), Vector128.ZipLower(ls, rs)); + Assert.Equal(Vector128.Create((short)5, 13, 6, 14, 7, 15, 8, 16), Vector128.ZipUpper(ls, rs)); + + Vector128 lb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Vector128 rb = Opaque(Vector128.Create((byte)16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)); + Assert.Equal(Vector128.Create((byte)0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + Vector128.ZipLower(lb, rb)); + Assert.Equal(Vector128.Create((byte)8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + Vector128.ZipUpper(lb, rb)); + } + + [Fact] + public static unsafe void UnzipTest() + { + // Opaque operands force the shuffle + OR deinterleave rather than constant folding. + + Vector128 li = Opaque(Vector128.Create(1, 2, 3, 4)); + Vector128 ri = Opaque(Vector128.Create(5, 6, 7, 8)); + Assert.Equal(Vector128.Create(1, 3, 5, 7), Vector128.UnzipEven(li, ri)); + Assert.Equal(Vector128.Create(2, 4, 6, 8), Vector128.UnzipOdd(li, ri)); + + Vector128 lf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Vector128 rf = Opaque(Vector128.Create(5.0f, 6.0f, 7.0f, 8.0f)); + Assert.Equal(Vector128.Create(1.0f, 3.0f, 5.0f, 7.0f), Vector128.UnzipEven(lf, rf)); + Assert.Equal(Vector128.Create(2.0f, 4.0f, 6.0f, 8.0f), Vector128.UnzipOdd(lf, rf)); + + Vector128 ll = Opaque(Vector128.Create(10L, 20L)); + Vector128 rl = Opaque(Vector128.Create(30L, 40L)); + Assert.Equal(Vector128.Create(10L, 30L), Vector128.UnzipEven(ll, rl)); + Assert.Equal(Vector128.Create(20L, 40L), Vector128.UnzipOdd(ll, rl)); + + Vector128 ls = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Vector128 rs = Opaque(Vector128.Create((short)9, 10, 11, 12, 13, 14, 15, 16)); + Assert.Equal(Vector128.Create((short)1, 3, 5, 7, 9, 11, 13, 15), Vector128.UnzipEven(ls, rs)); + Assert.Equal(Vector128.Create((short)2, 4, 6, 8, 10, 12, 14, 16), Vector128.UnzipOdd(ls, rs)); + + Vector128 lb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Vector128 rb = Opaque(Vector128.Create((byte)16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)); + Assert.Equal(Vector128.Create((byte)0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + Vector128.UnzipEven(lb, rb)); + Assert.Equal(Vector128.Create((byte)1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + Vector128.UnzipOdd(lb, rb)); + } + + [Fact] + public static unsafe void ConcatTest() + { + // Opaque operands force the shuffle + OR concat rather than constant folding. + + Vector128 li = Opaque(Vector128.Create(1, 2, 3, 4)); + Vector128 ri = Opaque(Vector128.Create(5, 6, 7, 8)); + Assert.Equal(Vector128.Create(1, 2, 5, 6), Vector128.ConcatLowerLower(li, ri)); + Assert.Equal(Vector128.Create(3, 4, 5, 6), Vector128.ConcatUpperLower(li, ri)); + Assert.Equal(Vector128.Create(1, 2, 7, 8), Vector128.ConcatLowerUpper(li, ri)); + Assert.Equal(Vector128.Create(3, 4, 7, 8), Vector128.ConcatUpperUpper(li, ri)); + + Vector128 lf = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Vector128 rf = Opaque(Vector128.Create(5.0f, 6.0f, 7.0f, 8.0f)); + Assert.Equal(Vector128.Create(1.0f, 2.0f, 5.0f, 6.0f), Vector128.ConcatLowerLower(lf, rf)); + Assert.Equal(Vector128.Create(3.0f, 4.0f, 7.0f, 8.0f), Vector128.ConcatUpperUpper(lf, rf)); + + Vector128 ll = Opaque(Vector128.Create(10L, 20L)); + Vector128 rl = Opaque(Vector128.Create(30L, 40L)); + Assert.Equal(Vector128.Create(10L, 30L), Vector128.ConcatLowerLower(ll, rl)); + Assert.Equal(Vector128.Create(20L, 40L), Vector128.ConcatUpperUpper(ll, rl)); + Assert.Equal(Vector128.Create(20L, 30L), Vector128.ConcatUpperLower(ll, rl)); + + Vector128 ls = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Vector128 rs = Opaque(Vector128.Create((short)9, 10, 11, 12, 13, 14, 15, 16)); + Assert.Equal(Vector128.Create((short)1, 2, 3, 4, 9, 10, 11, 12), Vector128.ConcatLowerLower(ls, rs)); + Assert.Equal(Vector128.Create((short)5, 6, 7, 8, 13, 14, 15, 16), Vector128.ConcatUpperUpper(ls, rs)); + + Vector128 lb = Opaque(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + Vector128 rb = Opaque(Vector128.Create((byte)16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)); + Assert.Equal(Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23), + Vector128.ConcatLowerLower(lb, rb)); + Assert.Equal(Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31), + Vector128.ConcatUpperUpper(lb, rb)); + } + + [Fact] + public static unsafe void CreateAlternatingSequenceTest() + { + // Opaque operands force the broadcast + zip path rather than a constant vector. + + int ei = Opaque(3); + int oi = Opaque(7); + Assert.Equal(Vector128.Create(3, 7, 3, 7), Vector128.CreateAlternatingSequence(ei, oi)); + + float ef = Opaque(1.5f); + float of = Opaque(-2.5f); + Assert.Equal(Vector128.Create(1.5f, -2.5f, 1.5f, -2.5f), Vector128.CreateAlternatingSequence(ef, of)); + + long el = Opaque(10L); + long ol = Opaque(-20L); + Assert.Equal(Vector128.Create(10L, -20L), Vector128.CreateAlternatingSequence(el, ol)); + + double ed = Opaque(4.25); + double od = Opaque(8.5); + Assert.Equal(Vector128.Create(4.25, 8.5), Vector128.CreateAlternatingSequence(ed, od)); + + short es = Opaque((short)5); + short os = Opaque((short)-6); + Assert.Equal(Vector128.Create((short)5, -6, 5, -6, 5, -6, 5, -6), Vector128.CreateAlternatingSequence(es, os)); + + byte eb = Opaque((byte)1); + byte ob = Opaque((byte)2); + Assert.Equal(Vector128.Create((byte)1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2), + Vector128.CreateAlternatingSequence(eb, ob)); + } + + [Fact] + public static unsafe void DotTest() + { + // Opaque operands force the Sum(left * right) reduction rather than constant folding. + + Vector128 ai = Opaque(Vector128.Create(1, 2, 3, 4)); + Vector128 bi = Opaque(Vector128.Create(5, 6, 7, 8)); + Assert.Equal(1 * 5 + 2 * 6 + 3 * 7 + 4 * 8, Vector128.Dot(ai, bi)); + + Vector128 af = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Vector128 bf = Opaque(Vector128.Create(5.0f, 6.0f, 7.0f, 8.0f)); + Assert.Equal(1.0f * 5 + 2.0f * 6 + 3.0f * 7 + 4.0f * 8, Vector128.Dot(af, bf)); + + Vector128 al = Opaque(Vector128.Create(2L, 3L)); + Vector128 bl = Opaque(Vector128.Create(4L, 5L)); + Assert.Equal(2L * 4 + 3L * 5, Vector128.Dot(al, bl)); + + Vector128 ad = Opaque(Vector128.Create(1.5, 2.5)); + Vector128 bd = Opaque(Vector128.Create(3.5, 4.5)); + Assert.Equal(1.5 * 3.5 + 2.5 * 4.5, Vector128.Dot(ad, bd)); + + Vector128 ash = Opaque(Vector128.Create((short)1, 2, 3, 4, 5, 6, 7, 8)); + Vector128 bsh = Opaque(Vector128.Create((short)1, 1, 1, 1, 1, 1, 1, 1)); + Assert.Equal((short)(1 + 2 + 3 + 4 + 5 + 6 + 7 + 8), Vector128.Dot(ash, bsh)); + + Vector128 ab = Opaque(Vector128.Create((byte)1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); + Vector128 bb = Opaque(Vector128.Create((byte)1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + Assert.Equal((byte)((1 + 2 + 3 + 4 + 5 + 6 + 7 + 8) * 2), Vector128.Dot(ab, bb)); + } + + [Fact] + public static unsafe void WidenTest() + { + // Opaque operands force the widening intrinsics rather than constant folding. The float + // WidenUpper is the newly-enabled case (shuffle + promote); the integer forms exercise the + // sign/zero-extend widening instructions. + + Vector128 f = Opaque(Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f)); + Assert.Equal(Vector128.Create(1.0, 2.0), Vector128.WidenLower(f)); + Assert.Equal(Vector128.Create(3.0, 4.0), Vector128.WidenUpper(f)); + + Vector128 i = Opaque(Vector128.Create(-1, 2, -3, 4)); + Assert.Equal(Vector128.Create(-1L, 2L), Vector128.WidenLower(i)); + Assert.Equal(Vector128.Create(-3L, 4L), Vector128.WidenUpper(i)); + + Vector128 s = Opaque(Vector128.Create((short)-1, 2, -3, 4, -5, 6, -7, 8)); + Assert.Equal(Vector128.Create(-1, 2, -3, 4), Vector128.WidenLower(s)); + Assert.Equal(Vector128.Create(-5, 6, -7, 8), Vector128.WidenUpper(s)); + + Vector128 b = Opaque(Vector128.Create((byte)1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)); + Assert.Equal(Vector128.Create((ushort)1, 2, 3, 4, 5, 6, 7, 8), Vector128.WidenLower(b)); + Assert.Equal(Vector128.Create((ushort)9, 10, 11, 12, 13, 14, 15, 16), Vector128.WidenUpper(b)); + } + + [Fact] + public static unsafe void NarrowTest() + { + // Opaque operands force the narrowing builder (byte-granularity shuffle for integers, + // demote + shuffle for double->float) rather than constant folding. + + Vector128 sh1 = Opaque(Vector128.Create((short)0x0100, 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E)); + Vector128 sh2 = Opaque(Vector128.Create(unchecked((short)0x1110), 0x1312, 0x1514, 0x1716, 0x1918, 0x1B1A, 0x1D1C, 0x1F1E)); + Assert.Equal(Vector128.Create((byte)0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1A, 0x1C, 0x1E), Vector128.Narrow(sh1, sh2)); + + Vector128 i1 = Opaque(Vector128.Create(0x00030002, 0x00050004, 0x00070006, 0x00090008)); + Vector128 i2 = Opaque(Vector128.Create(0x000B000A, 0x000D000C, 0x000F000E, 0x00110010)); + Assert.Equal(Vector128.Create((short)2, 4, 6, 8, 10, 12, 14, 16), Vector128.Narrow(i1, i2)); + + Vector128 l1 = Opaque(Vector128.Create(0x0000000200000001L, 0x0000000400000003L)); + Vector128 l2 = Opaque(Vector128.Create(0x0000000600000005L, 0x0000000800000007L)); + Assert.Equal(Vector128.Create(1, 3, 5, 7), Vector128.Narrow(l1, l2)); + + Vector128 d1 = Opaque(Vector128.Create(1.5, 2.5)); + Vector128 d2 = Opaque(Vector128.Create(3.5, 4.5)); + Assert.Equal(Vector128.Create(1.5f, 2.5f, 3.5f, 4.5f), Vector128.Narrow(d1, d2)); + } + + [Fact] + public static unsafe void NarrowWithSaturationTest() + { + // Opaque operands force the saturating-narrow builder (clamp to the narrow range then narrow). + + Vector128 sh1 = Opaque(Vector128.Create((short)-200, 200, -100, 100, 0, 50, -50, 127)); + Vector128 sh2 = Opaque(Vector128.Create((short)128, 300, -300, 1, -1, 2, -2, 3)); + Assert.Equal(Vector128.Create((sbyte)-128, 127, -100, 100, 0, 50, -50, 127, 127, 127, -128, 1, -1, 2, -2, 3), Vector128.NarrowWithSaturation(sh1, sh2)); + + // Unsigned source values above 0x7FFF must clamp as unsigned magnitude (not be treated as negative). + Vector128 ush1 = Opaque(Vector128.Create((ushort)0, 255, 256, 40000, 0x8000, 0xFFFF, 100, 200)); + Vector128 ush2 = Opaque(Vector128.Create((ushort)1, 2, 3, 4, 5, 6, 7, 8)); + Assert.Equal(Vector128.Create((byte)0, 255, 255, 255, 255, 255, 100, 200, 1, 2, 3, 4, 5, 6, 7, 8), Vector128.NarrowWithSaturation(ush1, ush2)); + + Vector128 i1 = Opaque(Vector128.Create(-40000, 40000, 100, -100)); + Vector128 i2 = Opaque(Vector128.Create(32767, -32768, 32768, -32769)); + Assert.Equal(Vector128.Create((short)-32768, 32767, 100, -100, 32767, -32768, 32767, -32768), Vector128.NarrowWithSaturation(i1, i2)); + + Vector128 l1 = Opaque(Vector128.Create(long.MinValue, long.MaxValue)); + Vector128 l2 = Opaque(Vector128.Create(5L, -5L)); + Assert.Equal(Vector128.Create(int.MinValue, int.MaxValue, 5, -5), Vector128.NarrowWithSaturation(l1, l2)); + + Vector128 ul1 = Opaque(Vector128.Create(0UL, 0xFFFFFFFFFFFFFFFFUL)); + Vector128 ul2 = Opaque(Vector128.Create(0x1_0000_0000UL, 7UL)); + Assert.Equal(Vector128.Create(0u, uint.MaxValue, uint.MaxValue, 7u), Vector128.NarrowWithSaturation(ul1, ul2)); + + Vector128 d1 = Opaque(Vector128.Create(1.5, 2.5)); + Vector128 d2 = Opaque(Vector128.Create(3.5, 4.5)); + Assert.Equal(Vector128.Create(1.5f, 2.5f, 3.5f, 4.5f), Vector128.NarrowWithSaturation(d1, d2)); + } + + [Fact] + public static unsafe void MinMaxScalarTest() + { + // Scalar Math.Min/Max and the Number/Magnitude variants now lower through the SIMD builders on + // WASM by wrapping the operands in a single-element vector and extracting the result. + + Assert.Equal(5.0, Math.Max(Opaque(3.0), Opaque(5.0))); + Assert.Equal(3.0, Math.Min(Opaque(3.0), Opaque(5.0))); + Assert.Equal(5.0f, Math.Max(Opaque(3.0f), Opaque(5.0f))); + Assert.Equal(3.0f, Math.Min(Opaque(3.0f), Opaque(5.0f))); + + // Max/Min propagate NaN. + Assert.True(double.IsNaN(Math.Max(Opaque(double.NaN), Opaque(5.0)))); + Assert.True(double.IsNaN(Math.Min(Opaque(5.0), Opaque(double.NaN)))); + + // The Number variants ignore NaN. + Assert.Equal(5.0, double.MaxNumber(Opaque(double.NaN), Opaque(5.0))); + Assert.Equal(5.0, double.MinNumber(Opaque(double.NaN), Opaque(5.0))); + Assert.Equal(5.0, double.MaxNumber(Opaque(3.0), Opaque(5.0))); + Assert.Equal(3.0, double.MinNumber(Opaque(3.0), Opaque(5.0))); + + // Magnitude variants compare by absolute value. + Assert.Equal(-5.0, double.MaxMagnitude(Opaque(-5.0), Opaque(3.0))); + Assert.Equal(3.0, double.MinMagnitude(Opaque(-5.0), Opaque(3.0))); + + // -0.0 is less than +0.0. + Assert.False(double.IsNegative(Math.Max(Opaque(-0.0), Opaque(0.0)))); + Assert.True(double.IsNegative(Math.Min(Opaque(-0.0), Opaque(0.0)))); + } + + [Fact] + public static unsafe void PromotedSimdFieldAccessTest() + { + // Individually reading and writing Vector2/3/4 fields promotes the local and lowers the field + // accesses through GetElement/WithElement rather than memory-based field loads and stores. + + Vector4 v4 = Opaque(new Vector4(1.0f, 2.0f, 3.0f, 4.0f)); + v4.X += 10.0f; + v4.W = v4.Y + v4.Z; + Assert.Equal(11.0f, v4.X); + Assert.Equal(2.0f, v4.Y); + Assert.Equal(3.0f, v4.Z); + Assert.Equal(5.0f, v4.W); + + Vector3 v3 = Opaque(new Vector3(1.0f, 2.0f, 3.0f)); + v3.Y = v3.X + v3.Z; + Assert.Equal(1.0f, v3.X); + Assert.Equal(4.0f, v3.Y); + Assert.Equal(3.0f, v3.Z); + + Vector2 v2 = Opaque(new Vector2(1.0f, 2.0f)); + v2.X = v2.Y * 2.0f; + Assert.Equal(4.0f, v2.X); + Assert.Equal(2.0f, v2.Y); + + // Plane embeds a Vector3 Normal in a 16-byte value, exercising the SIMD12 field get/set path. + Plane plane = Opaque(new Plane(1.0f, 2.0f, 3.0f, 4.0f)); + plane.Normal = new Vector3(plane.Normal.X + 10.0f, plane.Normal.Y, plane.Normal.Z); + Assert.Equal(new Vector3(11.0f, 2.0f, 3.0f), plane.Normal); + Assert.Equal(4.0f, plane.D); + } + + [Fact] + public static unsafe void ExtractMostSignificantBitsConstantFoldTest() + { + // A constant input lets value numbering fold ExtractMostSignificantBits to an integer constant; + // the Opaque overloads keep the runtime path covered so both agree. + + Assert.Equal(0b1010u, Vector128.Create(0, -1, 0, -1).ExtractMostSignificantBits()); + Assert.Equal(0b1010u, Opaque(Vector128.Create(0, -1, 0, -1)).ExtractMostSignificantBits()); + + Assert.Equal(0b0101u, Vector128.Create(-1, 0, -1, 0).ExtractMostSignificantBits()); + + var bytes = Vector128.Create((byte)0x80, 0, 0x80, 0, 0x80, 0, 0x80, 0, 0x80, 0, 0x80, 0, 0x80, 0, 0x80, 0); + Assert.Equal(0b0101010101010101u, bytes.ExtractMostSignificantBits()); + Assert.Equal(0b0101010101010101u, Opaque(bytes).ExtractMostSignificantBits()); + + Assert.Equal(0b10u, Vector128.Create(0.0, -1.0).ExtractMostSignificantBits()); + Assert.Equal(0b01u, Vector128.Create(-1.0, 0.0).ExtractMostSignificantBits()); + } + + [Fact] + public static unsafe void ConditionalSelectConstantFoldTest() + { + // A constant mask lets value numbering / gtFoldExprHWIntrinsic fold ConditionalSelect to a + // constant vector as (trueValue & mask) | (falseValue & ~mask); the Opaque overloads keep the + // runtime BitwiseSelect path covered so both agree. + + var trueValue = Vector128.Create(1, 2, 3, 4); + var falseValue = Vector128.Create(5, 6, 7, 8); + + // All-bits-set mask selects trueValue + Assert.Equal(trueValue, Vector128.ConditionalSelect(Vector128.AllBitsSet, trueValue, falseValue)); + + // Zero mask selects falseValue + Assert.Equal(falseValue, Vector128.ConditionalSelect(Vector128.Zero, trueValue, falseValue)); + + // Per-lane mix + var mask = Vector128.Create(-1, 0, -1, 0); + var expected = Vector128.Create(1, 6, 3, 8); + Assert.Equal(expected, Vector128.ConditionalSelect(mask, trueValue, falseValue)); + Assert.Equal(expected, Vector128.ConditionalSelect(Opaque(mask), Opaque(trueValue), Opaque(falseValue))); + + // Sub-lane granularity with bytes + var byteMask = Vector128.Create((byte)0xF0, 0x0F, 0xFF, 0x00, 0xF0, 0x0F, 0xFF, 0x00, 0xF0, 0x0F, 0xFF, 0x00, 0xF0, 0x0F, 0xFF, 0x00); + var byteTrue = Vector128.Create((byte)0xAA); + var byteFalse = Vector128.Create((byte)0x55); + var byteExpected = Vector128.Create((byte)0xA5, 0x5A, 0xAA, 0x55, 0xA5, 0x5A, 0xAA, 0x55, 0xA5, 0x5A, 0xAA, 0x55, 0xA5, 0x5A, 0xAA, 0x55); + Assert.Equal(byteExpected, Vector128.ConditionalSelect(byteMask, byteTrue, byteFalse)); + Assert.Equal(byteExpected, Vector128.ConditionalSelect(Opaque(byteMask), Opaque(byteTrue), Opaque(byteFalse))); + } + [Fact] public static unsafe void SaturatingArithmeticTest() {