Vectorize BitIncrement and BitDecrement for float/double in TensorPri…

…mitives (#123610)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com>
Co-authored-by: tannergooding <10487869+tannergooding@users.noreply.github.com>
Co-authored-by: Stephen Toub <stoub@microsoft.com>

src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
 
 namespace System.Numerics.Tensors
@@ -26,11 +27,162 @@ public static void BitDecrement<T>(ReadOnlySpan<T> x, Span<T> destination)
         private readonly struct BitDecrementOperator<T> : IUnaryOperator<T, T>
             where T : IFloatingPointIeee754<T>
         {
-            public static bool Vectorizable => false; // TODO: Vectorize
+            public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);
+
             public static T Invoke(T x) => T.BitDecrement(x);
-            public static Vector128<T> Invoke(Vector128<T> x) => throw new NotSupportedException();
-            public static Vector256<T> Invoke(Vector256<T> x) => throw new NotSupportedException();
-            public static Vector512<T> Invoke(Vector512<T> x) => throw new NotSupportedException();
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector128<T> Invoke(Vector128<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector128<float> xFloat = x.AsSingle();
+                    Vector128<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector128<uint> result = Vector128.ConditionalSelect(
+                        Vector128.IsNegative(xFloat).AsUInt32(),
+                        bits + Vector128<uint>.One,
+                        bits - Vector128<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector128<uint> isPositiveZero = Vector128.IsZero(xFloat).AsUInt32();
+                    Vector128<uint> specialValue = Vector128.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;
+
+                    Vector128<uint> isNaNOrNegInf = (Vector128.IsNaN(xFloat) | Vector128.IsNegativeInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector128<uint> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector128<double> xDouble = x.AsDouble();
+                    Vector128<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector128<ulong> result = Vector128.ConditionalSelect(
+                        Vector128.IsNegative(xDouble).AsUInt64(),
+                        bits + Vector128<ulong>.One,
+                        bits - Vector128<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector128<ulong> isPositiveZero = Vector128.IsZero(xDouble).AsUInt64();
+                    Vector128<ulong> specialValue = Vector128.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;
+
+                    Vector128<ulong> isNaNOrNegInf = (Vector128.IsNaN(xDouble) | Vector128.IsNegativeInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector128<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector256<T> Invoke(Vector256<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector256<float> xFloat = x.AsSingle();
+                    Vector256<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector256<uint> result = Vector256.ConditionalSelect(
+                        Vector256.IsNegative(xFloat).AsUInt32(),
+                        bits + Vector256<uint>.One,
+                        bits - Vector256<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector256<uint> isPositiveZero = Vector256.IsZero(xFloat).AsUInt32();
+                    Vector256<uint> specialValue = Vector256.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;
+
+                    Vector256<uint> isNaNOrNegInf = (Vector256.IsNaN(xFloat) | Vector256.IsNegativeInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector256<uint> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector256<double> xDouble = x.AsDouble();
+                    Vector256<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector256<ulong> result = Vector256.ConditionalSelect(
+                        Vector256.IsNegative(xDouble).AsUInt64(),
+                        bits + Vector256<ulong>.One,
+                        bits - Vector256<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector256<ulong> isPositiveZero = Vector256.IsZero(xDouble).AsUInt64();
+                    Vector256<ulong> specialValue = Vector256.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;
+
+                    Vector256<ulong> isNaNOrNegInf = (Vector256.IsNaN(xDouble) | Vector256.IsNegativeInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector256<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector512<T> Invoke(Vector512<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector512<float> xFloat = x.AsSingle();
+                    Vector512<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector512<uint> result = Vector512.ConditionalSelect(
+                        Vector512.IsNegative(xFloat).AsUInt32(),
+                        bits + Vector512<uint>.One,
+                        bits - Vector512<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector512<uint> isPositiveZero = Vector512.IsZero(xFloat).AsUInt32();
+                    Vector512<uint> specialValue = Vector512.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;
+
+                    Vector512<uint> isNaNOrNegInf = (Vector512.IsNaN(xFloat) | Vector512.IsNegativeInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector512<uint> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector512<double> xDouble = x.AsDouble();
+                    Vector512<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> increment, positive -> decrement
+                    Vector512<ulong> result = Vector512.ConditionalSelect(
+                        Vector512.IsNegative(xDouble).AsUInt64(),
+                        bits + Vector512<ulong>.One,
+                        bits - Vector512<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector512<ulong> isPositiveZero = Vector512.IsZero(xDouble).AsUInt64();
+                    Vector512<ulong> specialValue = Vector512.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;
+
+                    Vector512<ulong> isNaNOrNegInf = (Vector512.IsNaN(xDouble) | Vector512.IsNegativeInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrNegInf;
+
+                    Vector512<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
+                    return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
         }
     }
 }

src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
 
 namespace System.Numerics.Tensors
@@ -26,11 +27,162 @@ public static void BitIncrement<T>(ReadOnlySpan<T> x, Span<T> destination)
         private readonly struct BitIncrementOperator<T> : IUnaryOperator<T, T>
             where T : IFloatingPointIeee754<T>
         {
-            public static bool Vectorizable => false; // TODO: Vectorize
+            public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);
+
             public static T Invoke(T x) => T.BitIncrement(x);
-            public static Vector128<T> Invoke(Vector128<T> x) => throw new NotSupportedException();
-            public static Vector256<T> Invoke(Vector256<T> x) => throw new NotSupportedException();
-            public static Vector512<T> Invoke(Vector512<T> x) => throw new NotSupportedException();
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector128<T> Invoke(Vector128<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector128<float> xFloat = x.AsSingle();
+                    Vector128<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector128<uint> result = Vector128.ConditionalSelect(
+                        Vector128.IsNegative(xFloat).AsUInt32(),
+                        bits - Vector128<uint>.One,
+                        bits + Vector128<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector128<uint> isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000u));
+                    Vector128<uint> specialValue = Vector128.Create(0x0000_0001u) & isNegativeZero;
+
+                    Vector128<uint> isNaNOrPosInf = (Vector128.IsNaN(xFloat) | Vector128.IsPositiveInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector128<uint> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector128<double> xDouble = x.AsDouble();
+                    Vector128<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector128<ulong> result = Vector128.ConditionalSelect(
+                        Vector128.IsNegative(xDouble).AsUInt64(),
+                        bits - Vector128<ulong>.One,
+                        bits + Vector128<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector128<ulong> isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000_0000_0000ul));
+                    Vector128<ulong> specialValue = Vector128.Create(0x0000_0000_0000_0001ul) & isNegativeZero;
+
+                    Vector128<ulong> isNaNOrPosInf = (Vector128.IsNaN(xDouble) | Vector128.IsPositiveInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector128<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector256<T> Invoke(Vector256<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector256<float> xFloat = x.AsSingle();
+                    Vector256<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector256<uint> result = Vector256.ConditionalSelect(
+                        Vector256.IsNegative(xFloat).AsUInt32(),
+                        bits - Vector256<uint>.One,
+                        bits + Vector256<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector256<uint> isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000u));
+                    Vector256<uint> specialValue = Vector256.Create(0x0000_0001u) & isNegativeZero;
+
+                    Vector256<uint> isNaNOrPosInf = (Vector256.IsNaN(xFloat) | Vector256.IsPositiveInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector256<uint> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector256<double> xDouble = x.AsDouble();
+                    Vector256<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector256<ulong> result = Vector256.ConditionalSelect(
+                        Vector256.IsNegative(xDouble).AsUInt64(),
+                        bits - Vector256<ulong>.One,
+                        bits + Vector256<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector256<ulong> isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000_0000_0000ul));
+                    Vector256<ulong> specialValue = Vector256.Create(0x0000_0000_0000_0001ul) & isNegativeZero;
+
+                    Vector256<ulong> isNaNOrPosInf = (Vector256.IsNaN(xDouble) | Vector256.IsPositiveInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector256<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static Vector512<T> Invoke(Vector512<T> x)
+            {
+                if (typeof(T) == typeof(float))
+                {
+                    Vector512<float> xFloat = x.AsSingle();
+                    Vector512<uint> bits = xFloat.AsUInt32();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector512<uint> result = Vector512.ConditionalSelect(
+                        Vector512.IsNegative(xFloat).AsUInt32(),
+                        bits - Vector512<uint>.One,
+                        bits + Vector512<uint>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector512<uint> isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000u));
+                    Vector512<uint> specialValue = Vector512.Create(0x0000_0001u) & isNegativeZero;
+
+                    Vector512<uint> isNaNOrPosInf = (Vector512.IsNaN(xFloat) | Vector512.IsPositiveInfinity(xFloat)).AsUInt32();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector512<uint> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
+                }
+
+                if (typeof(T) == typeof(double))
+                {
+                    Vector512<double> xDouble = x.AsDouble();
+                    Vector512<ulong> bits = xDouble.AsUInt64();
+
+                    // General case: negative -> decrement, positive -> increment
+                    Vector512<ulong> result = Vector512.ConditionalSelect(
+                        Vector512.IsNegative(xDouble).AsUInt64(),
+                        bits - Vector512<ulong>.One,
+                        bits + Vector512<ulong>.One);
+
+                    // Handle special cases with a single conditional select
+                    Vector512<ulong> isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000_0000_0000ul));
+                    Vector512<ulong> specialValue = Vector512.Create(0x0000_0000_0000_0001ul) & isNegativeZero;
+
+                    Vector512<ulong> isNaNOrPosInf = (Vector512.IsNaN(xDouble) | Vector512.IsPositiveInfinity(xDouble)).AsUInt64();
+                    specialValue |= bits & isNaNOrPosInf;
+
+                    Vector512<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
+                    return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
+                }
+
+                // Fallback for unsupported types - should not be reached since Vectorizable returns false
+                throw new NotSupportedException();
+            }
         }
     }
 }