From 7f11eff9ec5ef78e6e5df6db9b5882e03804aaf2 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 24 Jul 2026 14:18:06 +0200 Subject: [PATCH] add cast_value --- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 14 + .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 + .../v3/codec/core/CastValueCodec.java | 232 ++++++++ .../v3/codec/core/CastValueConverter.java | 503 ++++++++++++++++++ .../dev/zarr/zarrjava/CastValueCodecTest.java | 188 +++++++ 5 files changed, 938 insertions(+) create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueCodec.java create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueConverter.java create mode 100644 src/test/java/dev/zarr/zarrjava/CastValueCodecTest.java diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5c3487ce..7c12041d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -64,6 +64,20 @@ public CodecBuilder withTranspose(int[] order) { return this; } + public CodecBuilder withCastValue(CastValueCodec.Configuration configuration) { + codecs.add(new CastValueCodec(configuration)); + return this; + } + + public CodecBuilder withCastValue(DataType dataType) { + return withCastValue(new CastValueCodec.Configuration(dataType, null, null, null)); + } + + public CodecBuilder withCastValue(DataType dataType, CastValueCodec.Rounding rounding, + CastValueCodec.OutOfRange outOfRange) { + return withCastValue(new CastValueCodec.Configuration(dataType, rounding, outOfRange, null)); + } + public CodecBuilder withBytes(Endian endian) { if (dataType.getByteCount() <= 1) codecs.add(new BytesCodec()); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index ad249e08..ada09755 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -12,6 +12,7 @@ public class CodecRegistry { static { addType("transpose", TransposeCodec.class); + addType("cast_value", CastValueCodec.class); addType("bytes", BytesCodec.class); addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueCodec.java new file mode 100644 index 00000000..c543ad6c --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueCodec.java @@ -0,0 +1,232 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v3.codec.core.CastValueConverter.ScalarEntry; +import ucar.ma2.Array; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.List; + +/** + * The {@code cast_value} codec converts (casts) the numeric value of every array element to a + * different data type. It is an {@code array -> array} codec: it changes the stored data type while + * leaving every other array property intact, and it does not reinterpret binary representations. + * + *

On encode the values are cast from the array data type to {@code configuration.data_type}; on + * decode the same procedure runs with the input and output data types swapped. The actual numeric + * conversion lives in {@link CastValueConverter}; this class only handles the Zarr codec integration: + * configuration parsing, propagating the new data type to downstream codecs, and casting the fill + * value. + * + *

Supported data types are the real-number types this library models: {@code int8/16/32/64}, + * {@code uint8/16/32/64}, {@code float32} and {@code float64}. Other data types from the codec + * specification (e.g. {@code float8_*}, {@code bfloat16}, {@code int2}) are not modelled here. + */ +public class CastValueCodec extends ArrayArrayCodec implements Codec { + + @JsonIgnore + @Nonnull + public final String name = "cast_value"; + @Nonnull + public final Configuration configuration; + + // Set up in resolveArrayMetadata (once, before any encode/decode call), then only read. + @JsonIgnore + private CastValueConverter converter; + @JsonIgnore + private List encodeEntries = new ArrayList<>(); + @JsonIgnore + private List decodeEntries = new ArrayList<>(); + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public CastValueCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; + } + + // ===== Codec pipeline integration ======================================================== + + @Override + public Array encode(Array chunkArray) throws ZarrException { + return converter.castArray(chunkArray, arrayDataType(), configuration.dataType, encodeEntries); + } + + @Override + public Array decode(Array chunkArray) throws ZarrException { + return converter.castArray(chunkArray, configuration.dataType, arrayDataType(), decodeEntries); + } + + @Override + public long computeEncodedSize(long inputByteLength, CoreArrayMetadata arrayMetadata) + throws ZarrException { + long numElements = inputByteLength / arrayMetadata.dataType.getByteCount(); + return numElements * configuration.dataType.getByteCount(); + } + + /** + * Runs once when the codec pipeline is built. It validates the configuration, prepares the + * converter and scalar_map lookups, casts the fill value to the target type, and reports the new + * data type (and cast fill value) to the codecs downstream of this one. + */ + @Override + public CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + super.resolveArrayMetadata(); + DataType inputType = arrayDataType(); + DataType outputType = configuration.dataType; + CastValueConverter.requireSupported(inputType); + CastValueConverter.requireSupported(outputType); + if (configuration.outOfRange == OutOfRange.WRAP && CastValueConverter.isFloatTarget(outputType)) { + throw new ZarrException( + "The cast_value 'out_of_range' value 'wrap' is only permitted for integral target data types."); + } + + this.converter = new CastValueConverter(configuration.rounding, configuration.outOfRange); + this.encodeEntries = converter.buildEntries( + configuration.scalarMap == null ? null : configuration.scalarMap.encode, inputType, outputType); + this.decodeEntries = converter.buildEntries( + configuration.scalarMap == null ? null : configuration.scalarMap.decode, outputType, inputType); + + Object srcFillValue = arrayMetadata.parsedFillValue; + Object castFillValue = converter.castFillValue(srcFillValue, inputType, outputType, encodeEntries); + if (srcFillValue != null + && !converter.fillValueRoundTrips(castFillValue, srcFillValue, outputType, inputType, decodeEntries)) { + throw new ZarrException( + "The cast_value fill value '" + srcFillValue + "' does not survive a round-trip cast."); + } + + return new CoreArrayMetadata( + arrayMetadata.shape, arrayMetadata.chunkShape, outputType, castFillValue); + } + + private DataType arrayDataType() throws ZarrException { + if (!(arrayMetadata.dataType instanceof DataType)) { + throw new ZarrException("The cast_value codec requires a Zarr v3 data type."); + } + return (DataType) arrayMetadata.dataType; + } + + // ===== Configuration ===================================================================== + + /** How values are rounded when the target data type cannot exactly represent a value. */ + public enum Rounding { + NEAREST_EVEN("nearest-even", RoundingMode.HALF_EVEN), + TOWARDS_ZERO("towards-zero", RoundingMode.DOWN), + TOWARDS_POSITIVE("towards-positive", RoundingMode.CEILING), + TOWARDS_NEGATIVE("towards-negative", RoundingMode.FLOOR), + NEAREST_AWAY("nearest-away", RoundingMode.HALF_UP); + + private final String value; + final RoundingMode mode; + + Rounding(String value, RoundingMode mode) { + this.value = value; + this.mode = mode; + } + + @JsonValue + public String getValue() { + return value; + } + + @JsonCreator + public static Rounding fromValue(String value) { + for (Rounding rounding : values()) { + if (rounding.value.equals(value)) { + return rounding; + } + } + throw new IllegalArgumentException("Unknown cast_value rounding: '" + value + "'."); + } + } + + /** How values outside the representable range of the target data type are handled. */ + public enum OutOfRange { + CLAMP("clamp"), + WRAP("wrap"); + + private final String value; + + OutOfRange(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + + @JsonCreator + public static OutOfRange fromValue(String value) { + for (OutOfRange outOfRange : values()) { + if (outOfRange.value.equals(value)) { + return outOfRange; + } + } + throw new IllegalArgumentException("Unknown cast_value out_of_range: '" + value + "'."); + } + } + + /** Explicit input-to-output scalar mappings, evaluated before any other casting rule. */ + public static final class ScalarMap { + @Nullable + @JsonProperty("encode") + public final Object[][] encode; + @Nullable + @JsonProperty("decode") + public final Object[][] decode; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ScalarMap( + @Nullable @JsonProperty("encode") Object[][] encode, + @Nullable @JsonProperty("decode") Object[][] decode) { + this.encode = encode; + this.decode = decode; + } + } + + public static final class Configuration { + + @Nonnull + @JsonProperty("data_type") + public final DataType dataType; + + @Nonnull + @JsonProperty("rounding") + public final Rounding rounding; + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("out_of_range") + public final OutOfRange outOfRange; + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("scalar_map") + public final ScalarMap scalarMap; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration( + @Nonnull @JsonProperty(value = "data_type", required = true) DataType dataType, + @Nullable @JsonProperty("rounding") Rounding rounding, + @Nullable @JsonProperty("out_of_range") OutOfRange outOfRange, + @Nullable @JsonProperty("scalar_map") ScalarMap scalarMap) { + this.dataType = dataType; + this.rounding = rounding == null ? Rounding.NEAREST_EVEN : rounding; + this.outOfRange = outOfRange; + this.scalarMap = scalarMap; + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueConverter.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueConverter.java new file mode 100644 index 00000000..741b3005 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/CastValueConverter.java @@ -0,0 +1,503 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec.OutOfRange; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec.Rounding; +import ucar.ma2.Array; +import ucar.ma2.IndexIterator; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +/** + * The pure value-casting math behind the {@link CastValueCodec}. This class knows nothing about the + * Zarr codec pipeline; it only converts numeric values from one {@link DataType} to another following + * the {@code cast_value} procedure (scalar_map, then exact representability, then rounding and + * out-of-range handling). Keeping it separate from the codec makes the arithmetic easy to read and + * test in isolation. + * + *

Values are carried through an exact {@link BigDecimal} / {@link BigInteger} intermediate so that + * 64-bit integer exactness and directed rounding stay correct. + */ +final class CastValueConverter { + + private final Rounding rounding; + private final OutOfRange outOfRange; + + CastValueConverter(Rounding rounding, OutOfRange outOfRange) { + this.rounding = rounding; + this.outOfRange = outOfRange; + } + + // ===== Public API used by the codec ====================================================== + + /** + * Casts every element of {@code input} (a chunk in {@code inputType}) into a new array of + * {@code outputType}. + */ + Array castArray(Array input, DataType inputType, DataType outputType, List entries) + throws ZarrException { + int[] shape = input.getShape(); + Array output = Array.factory(outputType.getMA2DataType(), shape); + IndexIterator inputIt = input.getIndexIterator(); + IndexIterator outputIt = output.getIndexIterator(); + while (inputIt.hasNext()) { + Scalar scalar = readScalar(inputIt, inputType); + Object result = castScalar(scalar, outputType, entries); + writeScalar(outputIt, outputType, result); + } + return output; + } + + /** Casts a single (boxed) fill value into the boxed primitive of {@code outputType}. */ + Object castFillValue(Object fillValue, DataType inputType, DataType outputType, + List entries) throws ZarrException { + if (fillValue == null) { + return null; + } + Object result = castScalar(scalarFromBoxed(fillValue, inputType), outputType, entries); + return toBoxedForType(result, outputType); + } + + /** + * Returns whether a cast fill value decodes back to the original fill value. Used to validate that + * the fill value survives a round-trip cast (a {@code cast_value} specification requirement). + */ + boolean fillValueRoundTrips(Object castFillValue, Object originalFillValue, DataType outputType, + DataType inputType, List decodeEntries) + throws ZarrException { + Object decoded = castScalar(scalarFromBoxed(castFillValue, outputType), inputType, decodeEntries); + Object roundTripped = toBoxedForType(decoded, inputType); + return scalarsEqual(scalarFromBoxed(roundTripped, inputType), + scalarFromBoxed(originalFillValue, inputType)); + } + + /** + * Turns a {@code scalar_map} JSON table into a list of parsed lookup entries. The input scalars are + * parsed using {@code inputType}'s fill-value encoding, the output scalars using {@code outputType}'s. + */ + List buildEntries(Object[][] pairs, DataType inputType, DataType outputType) + throws ZarrException { + List entries = new ArrayList<>(); + if (pairs == null) { + return entries; + } + for (Object[] pair : pairs) { + if (pair.length != 2) { + throw new ZarrException("Each cast_value scalar_map entry must be a length-2 array."); + } + Object keyBoxed = ArrayMetadata.parseFillValue(pair[0], inputType); + Object outputBoxed = ArrayMetadata.parseFillValue(pair[1], outputType); + entries.add(new ScalarEntry(scalarFromBoxed(keyBoxed, inputType), outputBoxed)); + } + return entries; + } + + static void requireSupported(DataType type) throws ZarrException { + if (!isFloat(type) && !isSigned(type) && !isUnsigned(type)) { + throw new ZarrException( + "The cast_value codec does not support the data type '" + type.getValue() + + "'. Supported types are the integral and floating-point real-number types."); + } + } + + static boolean isFloatTarget(DataType type) { + return isFloat(type); + } + + // ===== The ordered casting procedure (one scalar) ======================================== + + private Object castScalar(Scalar scalar, DataType outputType, List entries) + throws ZarrException { + // 1. explicit scalar_map mapping (first match wins) + for (ScalarEntry entry : entries) { + if (scalarsEqual(entry.key, scalar)) { + return entry.output; + } + } + // 2. + 3. exact representability, then rounding and out-of-range handling + if (isFloat(outputType)) { + return castToFloat(scalar, outputType); + } + return castToInteger(scalar, outputType); + } + + private Object castToInteger(Scalar scalar, DataType outputType) throws ZarrException { + if (scalar.isNaN || scalar.isPositiveInfinity || scalar.isNegativeInfinity) { + throw new ZarrException( + "Cannot cast a NaN or infinite value to the integer data type '" + outputType.getValue() + + "' without an explicit scalar_map mapping."); + } + BigInteger min = integerMin(outputType); + BigInteger max = integerMax(outputType); + BigDecimal value = scalar.value; + + BigInteger candidate; + if (value.stripTrailingZeros().scale() <= 0) { + candidate = value.toBigInteger(); // already an integer value; no rounding needed + } else { + candidate = value.setScale(0, rounding.mode).toBigInteger(); + } + + if (candidate.compareTo(min) >= 0 && candidate.compareTo(max) <= 0) { + return candidate; + } + return handleOutOfRange(candidate, outputType, min, max); + } + + private Object handleOutOfRange(BigInteger candidate, DataType outputType, BigInteger min, + BigInteger max) throws ZarrException { + if (outOfRange == null) { + throw new ZarrException( + "The value '" + candidate + "' is out of range for the target data type '" + + outputType.getValue() + "' and no 'out_of_range' rule is configured."); + } + if (outOfRange == OutOfRange.CLAMP) { + return candidate.compareTo(min) < 0 ? min : max; + } + // WRAP: map to the value congruent modulo 2^N inside the representable range. + int bits = integerBits(outputType); + BigInteger modulus = BigInteger.ONE.shiftLeft(bits); + BigInteger wrapped = candidate.mod(modulus); // in [0, 2^N - 1] + if (isSigned(outputType) && wrapped.compareTo(max) > 0) { + wrapped = wrapped.subtract(modulus); + } + return wrapped; + } + + private Object castToFloat(Scalar scalar, DataType outputType) throws ZarrException { + boolean isFloat32 = outputType == DataType.FLOAT32; + if (scalar.isNaN) { + return isFloat32 ? (Object) Float.NaN : (Object) Double.NaN; + } + if (scalar.isPositiveInfinity) { + return isFloat32 ? (Object) Float.POSITIVE_INFINITY : (Object) Double.POSITIVE_INFINITY; + } + if (scalar.isNegativeInfinity) { + return isFloat32 ? (Object) Float.NEGATIVE_INFINITY : (Object) Double.NEGATIVE_INFINITY; + } + BigDecimal value = scalar.value; + if (value.signum() == 0) { + if (scalar.isNegativeZero) { + return isFloat32 ? (Object) (-0.0f) : (Object) (-0.0d); + } + return isFloat32 ? (Object) 0.0f : (Object) 0.0d; + } + + if (isFloat32) { + float nearest = value.floatValue(); + if (!Float.isInfinite(nearest) && new BigDecimal((double) nearest).compareTo(value) == 0) { + return nearest; // exactly representable + } + float result = roundToFloat(value); + if (Float.isInfinite(result)) { + return handleFloatOverflow(result, outputType, value); + } + return result; + } else { + double nearest = value.doubleValue(); + if (!Double.isInfinite(nearest) && new BigDecimal(nearest).compareTo(value) == 0) { + return nearest; // exactly representable + } + double result = roundToDouble(value); + if (Double.isInfinite(result)) { + return handleFloatOverflow(result, outputType, value); + } + return result; + } + } + + private Object handleFloatOverflow(double infinite, DataType outputType, BigDecimal value) + throws ZarrException { + if (outOfRange == OutOfRange.CLAMP) { + // Data types with +-Infinity map out-of-finite-range values to +-Infinity. + return outputType == DataType.FLOAT32 ? (Object) (float) infinite : (Object) infinite; + } + throw new ZarrException( + "The value '" + value.toPlainString() + "' exceeds the finite range of the target data type '" + + outputType.getValue() + "' and no 'clamp' out_of_range rule is configured."); + } + + /** Rounds an exact value to the nearest float allowed by the configured rounding mode. */ + private float roundToFloat(BigDecimal value) { + float nearest = value.floatValue(); + if (Float.isInfinite(nearest)) { + return nearest; + } + float down; + float up; + if (new BigDecimal((double) nearest).compareTo(value) > 0) { + up = nearest; + down = Math.nextDown(nearest); + } else { + down = nearest; + up = Math.nextUp(nearest); + } + switch (rounding) { + case TOWARDS_ZERO: + return value.signum() > 0 ? down : up; + case TOWARDS_POSITIVE: + return up; + case TOWARDS_NEGATIVE: + return down; + default: // NEAREST_EVEN, NEAREST_AWAY + return nearest; + } + } + + private double roundToDouble(BigDecimal value) { + double nearest = value.doubleValue(); + if (Double.isInfinite(nearest)) { + return nearest; + } + double down; + double up; + if (new BigDecimal(nearest).compareTo(value) > 0) { + up = nearest; + down = Math.nextDown(nearest); + } else { + down = nearest; + up = Math.nextUp(nearest); + } + switch (rounding) { + case TOWARDS_ZERO: + return value.signum() > 0 ? down : up; + case TOWARDS_POSITIVE: + return up; + case TOWARDS_NEGATIVE: + return down; + default: // NEAREST_EVEN, NEAREST_AWAY + return nearest; + } + } + + // ===== Reading / writing array elements ================================================== + + private static Scalar readScalar(IndexIterator it, DataType type) { + switch (type) { + case FLOAT32: + return Scalar.ofFloat(it.getFloatNext()); + case FLOAT64: + return Scalar.ofDouble(it.getDoubleNext()); + case INT8: + return Scalar.ofInteger(BigInteger.valueOf(it.getByteNext())); + case UINT8: + return Scalar.ofInteger(BigInteger.valueOf(it.getByteNext() & 0xFFL)); + case INT16: + return Scalar.ofInteger(BigInteger.valueOf(it.getShortNext())); + case UINT16: + return Scalar.ofInteger(BigInteger.valueOf(it.getShortNext() & 0xFFFFL)); + case INT32: + return Scalar.ofInteger(BigInteger.valueOf(it.getIntNext())); + case UINT32: + return Scalar.ofInteger(BigInteger.valueOf(it.getIntNext() & 0xFFFFFFFFL)); + case INT64: + return Scalar.ofInteger(BigInteger.valueOf(it.getLongNext())); + case UINT64: + return Scalar.ofInteger(new BigInteger(Long.toUnsignedString(it.getLongNext()))); + default: + throw new IllegalStateException("Unsupported cast_value data type: " + type); + } + } + + private static void writeScalar(IndexIterator it, DataType type, Object value) { + Number number = (Number) value; + switch (type) { + case FLOAT32: + it.setFloatNext(number.floatValue()); + break; + case FLOAT64: + it.setDoubleNext(number.doubleValue()); + break; + case INT8: + case UINT8: + it.setByteNext(number.byteValue()); + break; + case INT16: + case UINT16: + it.setShortNext(number.shortValue()); + break; + case INT32: + case UINT32: + it.setIntNext(number.intValue()); + break; + case INT64: + case UINT64: + it.setLongNext(number.longValue()); + break; + default: + throw new IllegalStateException("Unsupported cast_value data type: " + type); + } + } + + /** Reconstructs a {@link Scalar} from a boxed value (a fill value or a scalar_map key). */ + private static Scalar scalarFromBoxed(Object value, DataType type) { + Number number = (Number) value; + switch (type) { + case FLOAT32: + return Scalar.ofFloat(number.floatValue()); + case FLOAT64: + return Scalar.ofDouble(number.doubleValue()); + case INT8: + return Scalar.ofInteger(BigInteger.valueOf(number.byteValue())); + case UINT8: + return Scalar.ofInteger(BigInteger.valueOf(number.longValue() & 0xFFL)); + case INT16: + return Scalar.ofInteger(BigInteger.valueOf(number.shortValue())); + case UINT16: + return Scalar.ofInteger(BigInteger.valueOf(number.longValue() & 0xFFFFL)); + case INT32: + return Scalar.ofInteger(BigInteger.valueOf(number.intValue())); + case UINT32: + return Scalar.ofInteger(BigInteger.valueOf(number.longValue() & 0xFFFFFFFFL)); + case INT64: + return Scalar.ofInteger(BigInteger.valueOf(number.longValue())); + case UINT64: + return Scalar.ofInteger(new BigInteger(Long.toUnsignedString(number.longValue()))); + default: + throw new IllegalStateException("Unsupported cast_value data type: " + type); + } + } + + /** + * Converts a cast result (a {@link BigInteger} for integral targets, or a boxed float/double) into + * the exact boxed primitive type expected for a fill value of {@code type}. + */ + private static Object toBoxedForType(Object value, DataType type) { + Number number = (Number) value; + switch (type) { + case FLOAT32: + return number.floatValue(); + case FLOAT64: + return number.doubleValue(); + case INT8: + case UINT8: + return number.byteValue(); + case INT16: + case UINT16: + return number.shortValue(); + case INT32: + case UINT32: + return number.intValue(); + case INT64: + case UINT64: + return number.longValue(); + default: + throw new IllegalStateException("Unsupported cast_value data type: " + type); + } + } + + // ===== Value model & comparison ========================================================== + + private static boolean scalarsEqual(Scalar a, Scalar b) { + if (a.isNaN || b.isNaN) { + return a.isNaN && b.isNaN; + } + if (a.isPositiveInfinity || b.isPositiveInfinity) { + return a.isPositiveInfinity && b.isPositiveInfinity; + } + if (a.isNegativeInfinity || b.isNegativeInfinity) { + return a.isNegativeInfinity && b.isNegativeInfinity; + } + return a.value.compareTo(b.value) == 0; + } + + /** An exact numeric value, or one of the special IEEE-754 values (NaN, +-Infinity). */ + private static final class Scalar { + final boolean isNaN; + final boolean isPositiveInfinity; + final boolean isNegativeInfinity; + final boolean isNegativeZero; + final BigDecimal value; // null for the special values above + + private Scalar(boolean isNaN, boolean isPositiveInfinity, boolean isNegativeInfinity, + boolean isNegativeZero, BigDecimal value) { + this.isNaN = isNaN; + this.isPositiveInfinity = isPositiveInfinity; + this.isNegativeInfinity = isNegativeInfinity; + this.isNegativeZero = isNegativeZero; + this.value = value; + } + + static Scalar ofDouble(double d) { + if (Double.isNaN(d)) { + return new Scalar(true, false, false, false, null); + } + if (d == Double.POSITIVE_INFINITY) { + return new Scalar(false, true, false, false, null); + } + if (d == Double.NEGATIVE_INFINITY) { + return new Scalar(false, false, true, false, null); + } + boolean negZero = (d == 0.0d) && (Double.doubleToRawLongBits(d) != 0L); + return new Scalar(false, false, false, negZero, new BigDecimal(d)); + } + + static Scalar ofFloat(float f) { + if (Float.isNaN(f)) { + return new Scalar(true, false, false, false, null); + } + if (f == Float.POSITIVE_INFINITY) { + return new Scalar(false, true, false, false, null); + } + if (f == Float.NEGATIVE_INFINITY) { + return new Scalar(false, false, true, false, null); + } + boolean negZero = (f == 0.0f) && (Float.floatToRawIntBits(f) != 0); + return new Scalar(false, false, false, negZero, new BigDecimal((double) f)); + } + + static Scalar ofInteger(BigInteger i) { + return new Scalar(false, false, false, false, new BigDecimal(i)); + } + } + + /** A single {@code scalar_map} lookup entry: match {@link #key}, emit {@link #output}. */ + static final class ScalarEntry { + final Scalar key; + final Object output; // boxed primitive of the output data type + + private ScalarEntry(Scalar key, Object output) { + this.key = key; + this.output = output; + } + } + + // ===== Data type facts =================================================================== + + private static boolean isFloat(DataType type) { + return type == DataType.FLOAT32 || type == DataType.FLOAT64; + } + + private static boolean isSigned(DataType type) { + return type == DataType.INT8 || type == DataType.INT16 || type == DataType.INT32 + || type == DataType.INT64; + } + + private static boolean isUnsigned(DataType type) { + return type == DataType.UINT8 || type == DataType.UINT16 || type == DataType.UINT32 + || type == DataType.UINT64; + } + + private static int integerBits(DataType type) { + return type.getByteCount() * 8; + } + + private static BigInteger integerMin(DataType type) { + if (isUnsigned(type)) { + return BigInteger.ZERO; + } + return BigInteger.ONE.shiftLeft(integerBits(type) - 1).negate(); + } + + private static BigInteger integerMax(DataType type) { + if (isUnsigned(type)) { + return BigInteger.ONE.shiftLeft(integerBits(type)).subtract(BigInteger.ONE); + } + return BigInteger.ONE.shiftLeft(integerBits(type) - 1).subtract(BigInteger.ONE); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/CastValueCodecTest.java b/src/test/java/dev/zarr/zarrjava/CastValueCodecTest.java new file mode 100644 index 00000000..f131d711 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/CastValueCodecTest.java @@ -0,0 +1,188 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec.OutOfRange; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec.Rounding; +import dev.zarr.zarrjava.v3.codec.core.CastValueCodec.ScalarMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertThrows; + +public class CastValueCodecTest extends ZarrTest { + + private StoreHandle handle(String name) { + return new FilesystemStore(TESTOUTPUT).resolve("cast_value", name); + } + + private double[] roundTrip(String name, DataType target, CastValueCodec.Configuration config, + double[] data) throws ZarrException, IOException { + return roundTrip(name, target, config, data, 0); + } + + private double[] roundTrip(String name, DataType target, CastValueCodec.Configuration config, + double[] data, Object fillValue) throws ZarrException, IOException { + StoreHandle storeHandle = handle(name); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(data.length) + .withDataType(DataType.FLOAT64) + .withChunkShape(data.length) + .withFillValue(fillValue) + .withCodecs(c -> c.withCastValue(config).withBytes()) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, new int[]{data.length}, data)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + return (double[]) result.get1DJavaArray(ucar.ma2.DataType.DOUBLE); + } + + private static CastValueCodec.Configuration config(DataType target, Rounding rounding, + OutOfRange outOfRange, ScalarMap scalarMap) { + return new CastValueCodec.Configuration(target, rounding, outOfRange, scalarMap); + } + + @Test + public void testExactRoundTrip() throws Exception { + double[] data = {0, 1, 2, -3, 127, -128}; + double[] result = roundTrip("exact_int8", DataType.INT8, + config(DataType.INT8, null, null, null), data); + Assertions.assertArrayEquals(data, result); + } + + @Test + public void testClamp() throws Exception { + double[] data = {200, -200, 50}; + double[] result = roundTrip("clamp_int8", DataType.INT8, + config(DataType.INT8, null, OutOfRange.CLAMP, null), data); + Assertions.assertArrayEquals(new double[]{127, -128, 50}, result); + } + + @Test + public void testWrap() throws Exception { + double[] data = {130, -129, 261}; + double[] result = roundTrip("wrap_int8", DataType.INT8, + config(DataType.INT8, null, OutOfRange.WRAP, null), data); + Assertions.assertArrayEquals(new double[]{-126, 127, 5}, result); + } + + @Test + public void testRoundingNearestEven() throws Exception { + double[] data = {2.5, 3.5, -2.5, 0.5}; + double[] result = roundTrip("round_even", DataType.INT32, + config(DataType.INT32, Rounding.NEAREST_EVEN, null, null), data); + Assertions.assertArrayEquals(new double[]{2, 4, -2, 0}, result); + } + + @Test + public void testRoundingTowardsZero() throws Exception { + double[] data = {2.5, 3.5, -2.5, -3.9}; + double[] result = roundTrip("round_zero", DataType.INT32, + config(DataType.INT32, Rounding.TOWARDS_ZERO, null, null), data); + Assertions.assertArrayEquals(new double[]{2, 3, -2, -3}, result); + } + + @Test + public void testRoundingFloorAndCeil() throws Exception { + double[] data = {2.1, -2.1}; + double[] floor = roundTrip("round_floor", DataType.INT32, + config(DataType.INT32, Rounding.TOWARDS_NEGATIVE, null, null), data); + Assertions.assertArrayEquals(new double[]{2, -3}, floor); + double[] ceil = roundTrip("round_ceil", DataType.INT32, + config(DataType.INT32, Rounding.TOWARDS_POSITIVE, null, null), data); + Assertions.assertArrayEquals(new double[]{3, -2}, ceil); + } + + @Test + public void testNumpyCompatibilityExample() throws Exception { + // Mirrors the specification's NumPy-compatibility example: float64 -> uint8, round towards + // zero, wrap out-of-range values, and map NaN / +-Infinity to 0. + ScalarMap scalarMap = new ScalarMap( + new Object[][]{{"NaN", 0}, {"+Infinity", 0}, {"-Infinity", 0}}, null); + double[] data = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 3.9, 300, -1}; + double[] result = roundTrip("numpy", DataType.UINT8, + config(DataType.UINT8, Rounding.TOWARDS_ZERO, OutOfRange.WRAP, scalarMap), data); + Assertions.assertArrayEquals(new double[]{0, 0, 0, 3, 44, 255}, result); + } + + @Test + public void testScalarMapDecode() throws Exception { + // Round-trip a NaN fill/value: encode NaN -> 0 and decode 0 -> NaN. + ScalarMap scalarMap = new ScalarMap( + new Object[][]{{"NaN", 0}}, + new Object[][]{{0, "NaN"}}); + double[] data = {Double.NaN, 5}; + double[] result = roundTrip("scalarmap_decode", DataType.UINT8, + config(DataType.UINT8, Rounding.TOWARDS_ZERO, OutOfRange.WRAP, scalarMap), data, 7); + Assertions.assertTrue(Double.isNaN(result[0])); + Assertions.assertEquals(5.0, result[1]); + } + + @Test + public void testOutOfRangeWithoutRuleFailsOnWrite() { + double[] data = {200}; + // The write path runs on a parallel stream, so the codec's ZarrException surfaces wrapped. + assertCastError(() -> roundTrip("oor_fail", DataType.INT8, + config(DataType.INT8, null, null, null), data)); + } + + @Test + public void testNaNToIntegerWithoutMappingFailsOnWrite() { + double[] data = {Double.NaN}; + assertCastError(() -> roundTrip("nan_fail", DataType.INT8, + config(DataType.INT8, null, OutOfRange.CLAMP, null), data)); + } + + private static void assertCastError(org.junit.function.ThrowingRunnable runnable) { + Throwable thrown = assertThrows(Throwable.class, runnable); + for (Throwable t = thrown; t != null; t = t.getCause()) { + if (t instanceof ZarrException) { + return; + } + } + Assertions.fail("Expected a ZarrException in the cause chain, got: " + thrown); + } + + @Test + public void testFillValueRoundTripValidation() { + // Fill value NaN encodes to 0 but decodes back to 0.0 (no decode mapping), so it cannot + // survive a round trip -> array creation must fail. + StoreHandle storeHandle = handle("fill_roundtrip_fail"); + ScalarMap scalarMap = new ScalarMap(new Object[][]{{"NaN", 0}}, null); + assertThrows(ZarrException.class, () -> { + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(4) + .withDataType(DataType.FLOAT64) + .withChunkShape(4) + .withFillValue("NaN") + .withCodecs(c -> c.withCastValue( + config(DataType.UINT8, Rounding.TOWARDS_ZERO, OutOfRange.WRAP, scalarMap)).withBytes()) + .build(); + Array.create(storeHandle, metadata); + }); + } + + @Test + public void testFloat64ToFloat32RoundTrip() throws Exception { + // Values that are exactly representable in float32 survive the round trip unchanged. + double[] data = {0, 0.5, -0.25, 1024, -2048}; + double[] result = roundTrip("float32", DataType.FLOAT32, + config(DataType.FLOAT32, null, null, null), data); + Assertions.assertArrayEquals(data, result); + } + + @Test + public void testUnsupportedBoolTargetFails() { + double[] data = {0, 1}; + assertThrows(ZarrException.class, () -> roundTrip("bool_fail", DataType.BOOL, + config(DataType.BOOL, null, null, null), data)); + } +}