From 55b5e240505768d52b80b9a748aafbad31c01ba1 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 02:47:48 +0200 Subject: [PATCH 1/7] feat: stop allocating byte arrays --- .../slime/reader/LimitedInputStream.java | 42 +++++++ .../impl/v13/v13SlimeWorldDeSerializer.java | 106 +++++++++--------- 2 files changed, 98 insertions(+), 50 deletions(-) create mode 100644 core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java new file mode 100644 index 000000000..8a7b93159 --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java @@ -0,0 +1,42 @@ +package com.infernalsuite.asp.serialization.slime.reader; + +import java.io.IOException; +import java.io.InputStream; + +public class LimitedInputStream extends InputStream { + private final InputStream in; + private int remaining; + + public LimitedInputStream(InputStream in, int limit) { + this.in = in; + this.remaining = limit; + } + + @Override + public int read() throws IOException { + if (remaining <= 0) return -1; + int b = in.read(); + if (b != -1) remaining--; + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (remaining <= 0) return -1; + len = Math.min(len, remaining); + int read = in.read(b, off, len); + if (read > 0) remaining -= read; + return read; + } + + public void drainRemaining() { + byte[] buffer = new byte[512]; + try { + while (remaining > 0) { + int read = read(buffer, 0, Math.min(buffer.length, remaining)); + if (read == -1) break; + } + } catch (IOException ignored) { + } + } +} diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index 38cbac883..27610d5fc 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -1,6 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v13; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.infernalsuite.asp.SlimeLogger; import com.infernalsuite.asp.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; @@ -10,8 +11,10 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; +import com.infernalsuite.asp.skeleton.SkeletonSlimeWorld; +import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; @@ -19,9 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; +import java.io.*; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -38,11 +39,10 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S int worldVersion = dataStream.readInt(); byte additionalWorldData = dataStream.readByte(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, additionalWorldData, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -54,12 +54,13 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S worldPropertyMap.merge(propertyMap); } - return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); + + dataStream.close(); + return new SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte additionalWorldData, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte additionalWorldData, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -77,7 +78,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray blockLightArray; if ((sectionFlags & 1) == 1) { byte[] blockLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(blockLightByteArray); + chunkData.readFully(blockLightByteArray); blockLightArray = new NibbleArray(blockLightByteArray); } else { blockLightArray = null; @@ -87,49 +88,37 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray skyLightArray; if (((sectionFlags >> 1) & 1) == 1) { byte[] skyLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(skyLightByteArray); + chunkData.readFully(skyLightByteArray); skyLightArray = new NibbleArray(skyLightByteArray); } else { skyLightArray = null; } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); - chunkSections[sectionId] = new com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); + chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); CompoundBinaryTag poiChunk = null; if(v13AdditionalWorldData.POI_CHUNKS.isSet(additionalWorldData)) { - byte[] poiData = new byte[chunkData.readInt()]; - chunkData.read(poiData); - poiChunk = readCompound(poiData); + poiChunk = readLimitedCompound(chunkData); } ListBinaryTag blockTicks = null; if(v13AdditionalWorldData.BLOCK_TICKS.isSet(additionalWorldData)) { - byte[] blockTickData = new byte[chunkData.readInt()]; - chunkData.read(blockTickData); - CompoundBinaryTag tag = readCompound(blockTickData); + CompoundBinaryTag tag = readLimitedCompound(chunkData); blockTicks = tag.getList("block_ticks", BinaryTagTypes.COMPOUND); } ListBinaryTag fluidTicks = null; if(v13AdditionalWorldData.FLUID_TICKS.isSet(additionalWorldData)) { - byte[] fluidTickData = new byte[chunkData.readInt()]; - chunkData.read(fluidTickData); - CompoundBinaryTag tag = readCompound(fluidTickData); + CompoundBinaryTag tag = readLimitedCompound(chunkData); fluidTicks = tag.getList("fluid_ticks", BinaryTagTypes.COMPOUND); } @@ -144,9 +133,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Tile Entities - byte[] tileEntitiesRaw = read(chunkData); List tileEntities; - CompoundBinaryTag tileEntitiesCompound = readCompound(tileEntitiesRaw); + CompoundBinaryTag tileEntitiesCompound = readLimitedCompound(chunkData); if (tileEntitiesCompound.isEmpty()) { tileEntities = Collections.emptyList(); } else { @@ -157,9 +145,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - byte[] entitiesRaw = read(chunkData); List entities; - CompoundBinaryTag entitiesCompound = readCompound(entitiesRaw); + CompoundBinaryTag entitiesCompound = readLimitedCompound(chunkData); if (entitiesCompound.isEmpty()) { entities = Collections.emptyList(); } else { @@ -169,8 +156,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Extra Tag - byte[] rawExtra = read(chunkData); - CompoundBinaryTag extra = readCompound(rawExtra); + CompoundBinaryTag extra = readLimitedCompound(chunkData); Map extraData = new HashMap<>(); extra.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -180,21 +166,13 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; - } + stream.readInt(); //Decompressed length, legacy - private static byte[] read(DataInputStream stream) throws IOException { - int length = stream.readInt(); - byte[] data = new byte[length]; - stream.read(data); - return data; + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(inputStream); } private static @NotNull CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { @@ -202,4 +180,32 @@ private static byte[] read(DataInputStream stream) throws IOException { return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); } + + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { + int length = stream.readInt(); + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read(limitedInputStream); + limitedInputStream.drainRemaining(); + return tag; + } + + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read(zstd); + + //binary tag reading does not guarantee that the buffer is fully read + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} + + return tag; + } + } From db533fc2661d01be8929c31f00543550704773c5 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 03:00:19 +0200 Subject: [PATCH 2/7] fix: buffered streams make memory explode up to gigabytes in allocations --- .../impl/v13/v13SlimeWorldDeSerializer.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index 27610d5fc..0dd83b824 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -186,7 +186,13 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro if(length == 0) return CompoundBinaryTag.empty(); LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); - CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read(limitedInputStream); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later limitedInputStream.drainRemaining(); return tag; } @@ -199,9 +205,13 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); - CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read(zstd); - //binary tag reading does not guarantee that the buffer is fully read + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later byte[] buffer = new byte[512]; while (zstd.read(buffer) != -1) {} From bc69220268e712fd6d631c8c8e3bffebbd4cd115 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 03:05:33 +0200 Subject: [PATCH 3/7] feat: backport to v12 and v11 (untested) --- .../impl/v11/v11SlimeWorldDeSerializer.java | 95 +++++++++++-------- .../impl/v12/v12SlimeWorldDeSerializer.java | 87 ++++++++++------- .../impl/v13/v13SlimeWorldDeSerializer.java | 6 -- 3 files changed, 105 insertions(+), 83 deletions(-) diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java index fcb25b70d..4a0172fe8 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java @@ -1,6 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v11; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.infernalsuite.asp.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; @@ -11,13 +12,17 @@ import com.infernalsuite.asp.api.world.SlimeWorld; import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; +import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import net.kyori.adventure.nbt.*; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.ByteArrayInputStream; +import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; @@ -35,11 +40,10 @@ public class v11SlimeWorldDeSerializer implements com.infernalsuite.asp.serializ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException { int worldVersion = dataStream.readInt(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); SlimePropertyMap worldPropertyMap = propertyMap; CompoundBinaryTag propertiesMap = extraTag.get("properties") != null @@ -60,9 +64,8 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -97,33 +100,20 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); - chunkSections[sectionId] = new com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); + chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); // Tile Entities - int compressedTileEntitiesLength = chunkData.readInt(); - int decompressedTileEntitiesLength = chunkData.readInt(); - byte[] compressedTileEntitiesData = new byte[compressedTileEntitiesLength]; - byte[] decompressedTileEntitiesData = new byte[decompressedTileEntitiesLength]; - chunkData.read(compressedTileEntitiesData); - Zstd.decompress(decompressedTileEntitiesData, compressedTileEntitiesData); - - CompoundBinaryTag tileEntitiesCompound = readCompound(decompressedTileEntitiesData); + CompoundBinaryTag tileEntitiesCompound = readCompressedCompound(chunkData); ListBinaryTag tileEntitiesTag = tileEntitiesCompound.getList("tileEntities", BinaryTagTypes.COMPOUND); List serializedTileEntities = new ArrayList<>(tileEntitiesTag.size()); @@ -133,14 +123,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - int compressedEntitiesLength = chunkData.readInt(); - int decompressedEntitiesLength = chunkData.readInt(); - byte[] compressedEntitiesData = new byte[compressedEntitiesLength]; - byte[] decompressedEntitiesData = new byte[decompressedEntitiesLength]; - chunkData.read(compressedEntitiesData); - Zstd.decompress(decompressedEntitiesData, compressedEntitiesData); - - CompoundBinaryTag entitiesCompound = readCompound(decompressedEntitiesData); + CompoundBinaryTag entitiesCompound = readCompressedCompound(chunkData); ListBinaryTag entitiesTag = entitiesCompound.getList("entities", BinaryTagTypes.COMPOUND); List serializedEntities = new ArrayList<>(entitiesTag.size()); for (BinaryTag binaryTag : entitiesTag) { @@ -153,19 +136,49 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; + stream.readInt(); //Decompressed length, legacy + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(inputStream); } - private static CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { + int length = stream.readInt(); + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + limitedInputStream.drainRemaining(); + return tag; + } + + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); + return tag; } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java index 71516aa23..084b06402 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java @@ -1,6 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v12; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.infernalsuite.asp.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; @@ -11,6 +12,7 @@ import com.infernalsuite.asp.api.world.SlimeWorld; import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -23,6 +25,7 @@ import org.jetbrains.annotations.Nullable; import java.io.ByteArrayInputStream; +import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.util.Collections; @@ -40,11 +43,10 @@ public class v12SlimeWorldDeSerializer implements com.infernalsuite.asp.serializ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException { int worldVersion = dataStream.readInt(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -59,9 +61,8 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -79,7 +80,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray blockLightArray; if (chunkData.readBoolean()) { byte[] blockLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(blockLightByteArray); + chunkData.readFully(blockLightByteArray); blockLightArray = new NibbleArray(blockLightByteArray); } else { blockLightArray = null; @@ -89,35 +90,28 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray skyLightArray; if (chunkData.readBoolean()) { byte[] skyLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(skyLightByteArray); + chunkData.readFully(skyLightByteArray); skyLightArray = new NibbleArray(skyLightByteArray); } else { skyLightArray = null; } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); // Tile Entities - byte[] tileEntitiesRaw = read(chunkData); List tileEntities; - CompoundBinaryTag tileEntitiesCompound = readCompound(tileEntitiesRaw); + CompoundBinaryTag tileEntitiesCompound = readLimitedCompound(chunkData); if (tileEntitiesCompound.isEmpty()) { tileEntities = Collections.emptyList(); } else { @@ -128,9 +122,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - byte[] entitiesRaw = read(chunkData); List entities; - CompoundBinaryTag entitiesCompound = readCompound(entitiesRaw); + CompoundBinaryTag entitiesCompound = readLimitedCompound(chunkData); if (entitiesCompound.isEmpty()) { entities = Collections.emptyList(); } else { @@ -140,8 +133,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Extra Tag - byte[] rawExtra = read(chunkData); - CompoundBinaryTag extra = readCompound(rawExtra); + CompoundBinaryTag extra = readLimitedCompound(chunkData); Map extraData = new HashMap<>(); extra.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -151,26 +143,49 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; + stream.readInt(); //Decompressed length, legacy + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(inputStream); } - private static byte[] read(DataInputStream stream) throws IOException { + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { int length = stream.readInt(); - byte[] data = new byte[length]; - stream.read(data); - return data; + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + limitedInputStream.drainRemaining(); + return tag; } - private static @NotNull CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); + return tag; } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index 0dd83b824..b96c21807 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -175,12 +175,6 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro return new DataInputStream(inputStream); } - private static @NotNull CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); - - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); - } - private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { int length = stream.readInt(); if(length == 0) return CompoundBinaryTag.empty(); From b01c05ef28bf9f727ff3c325b3a04443030482e9 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 03:08:45 +0200 Subject: [PATCH 4/7] fix: buffered output stream on serialization --- .../asp/serialization/slime/SlimeSerializer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java index a11791c7f..fb58755ae 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.util.*; @@ -175,7 +176,7 @@ static byte[] serializeChunks(SlimeWorld world, Collection chunks, E // Extra Tag if (chunk.getExtraData() == null) { - LOGGER.warn("Chunk at " + chunk.getX() + ", " + chunk.getZ() + " from world " + world.getName() + " has no extra data! When deserialized, this chunk will have an empty extra data tag!"); + LOGGER.warn("Chunk at {}, {} from world {} has no extra data! When deserialized, this chunk will have an empty extra data tag!", chunk.getX(), chunk.getZ(), world.getName()); } byte[] extra = serializeCompoundTag(CompoundBinaryTag.from(chunk.getExtraData())); @@ -197,7 +198,9 @@ protected static byte[] serializeCompoundTag(CompoundBinaryTag tag) throws IOExc if (tag == null || tag.isEmpty()) return new byte[0]; ByteArrayOutputStream outByteStream = new ByteArrayOutputStream(); - BinaryTagIO.writer().write(tag, outByteStream); + + //Avoid a buffered output stream by casting to DataOutput. Buffered Output Streams make the memory usage explode + BinaryTagIO.writer().write(tag, (DataOutput) new DataOutputStream(outByteStream)); return outByteStream.toByteArray(); } From 4ec1afd6a0448ffb3814224681e2dc8207549138 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 03:45:35 +0200 Subject: [PATCH 5/7] feat: optimize saving (also cutting about 1gb on specific testing world) --- .../asp/SimpleDataFixerConverter.java | 1 + .../asp/level/SlimeInMemoryWorld.java | 2 +- .../serialization/anvil/AnvilWorldReader.java | 2 +- .../serialization/slime/SlimeSerializer.java | 51 +++++++++++-------- .../impl/v10/v10SlimeWorldDeSerializer.java | 3 +- .../impl/v11/v11SlimeWorldDeSerializer.java | 7 +-- .../impl/v12/v12SlimeWorldDeSerializer.java | 7 +-- .../impl/v13/v13SlimeWorldDeSerializer.java | 5 +- .../impl/v1_9/v1_9SlimeWorldDeserializer.java | 2 +- .../asp/skeleton/SkeletonCloning.java | 2 +- .../asp/skeleton/SkeletonSlimeWorld.java | 2 +- .../asp/util/CountingOutputStream.java | 29 +++++++++++ .../reader => util}/LimitedInputStream.java | 2 +- .../asp/util/ThrowingConsumer.java | 7 +++ .../infernalsuite/asp/{ => util}/Util.java | 2 +- 15 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java rename core/src/main/java/com/infernalsuite/asp/{serialization/slime/reader => util}/LimitedInputStream.java (94%) create mode 100644 core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java rename core/src/main/java/com/infernalsuite/asp/{ => util}/Util.java (85%) diff --git a/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java b/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java index 1fa4802dd..c900ef914 100644 --- a/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java +++ b/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java @@ -15,6 +15,7 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; +import com.infernalsuite.asp.util.Util; import net.kyori.adventure.nbt.CompoundBinaryTag; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; diff --git a/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java b/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java index 7fb69fc6c..b4b7b5359 100644 --- a/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java +++ b/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java @@ -3,7 +3,7 @@ import ca.spottedleaf.moonrise.patches.chunk_system.level.entity.ChunkEntitySlices; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.PoiChunk; import com.infernalsuite.asp.Converter; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.WorldAlreadyExistsException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.world.properties.SlimeProperties; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java b/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java index dfd6c0729..91dafa14b 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.serialization.anvil; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.InvalidWorldException; import com.infernalsuite.asp.api.utils.NibbleArray; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java index fb58755ae..fd0194f9a 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.serialization.slime; -import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdOutputStream; import com.infernalsuite.asp.api.utils.SlimeFormat; import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; @@ -8,6 +8,8 @@ import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; import com.infernalsuite.asp.serialization.slime.reader.impl.v13.v13AdditionalWorldData; +import com.infernalsuite.asp.util.CountingOutputStream; +import com.infernalsuite.asp.util.ThrowingConsumer; import net.kyori.adventure.nbt.BinaryTag; import net.kyori.adventure.nbt.BinaryTagIO; import net.kyori.adventure.nbt.BinaryTagTypes; @@ -16,10 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; +import java.io.*; import java.util.*; public class SlimeSerializer { @@ -61,20 +60,13 @@ public static byte[] serialize(SlimeWorld world) { outStream.writeByte(v13AdditionalWorldData.fromSet(additionalWorldData)); // Chunks - byte[] chunkData = serializeChunks(world, world.getChunkStorage(), additionalWorldData); - byte[] compressedChunkData = Zstd.compress(chunkData); - outStream.writeInt(compressedChunkData.length); - outStream.writeInt(chunkData.length); - outStream.write(compressedChunkData); - - // Extra Tag - byte[] extra = serializeCompoundTag(CompoundBinaryTag.builder().put(extraData).build()); - byte[] compressedExtra = Zstd.compress(extra); + writeCompressed(outStream, value -> serializeChunks(value, world, world.getChunkStorage(), additionalWorldData)); - outStream.writeInt(compressedExtra.length); - outStream.writeInt(extra.length); - outStream.write(compressedExtra); + writeCompressed(outStream, value -> { + //Avoid a buffered output stream by casting to DataOutput. Buffered Output Streams make the memory usage explode + BinaryTagIO.writer().write(CompoundBinaryTag.builder().put(extraData).build(), (DataOutput) new DataOutputStream(value)); + }); } catch (Exception e) { throw new RuntimeException(e); @@ -83,9 +75,7 @@ public static byte[] serialize(SlimeWorld world) { return outByteStream.toByteArray(); } - static byte[] serializeChunks(SlimeWorld world, Collection chunks, EnumSet data) throws IOException { - ByteArrayOutputStream outByteStream = new ByteArrayOutputStream(16384); - DataOutputStream outStream = new DataOutputStream(outByteStream); + static void serializeChunks(DataOutputStream outStream, SlimeWorld world, Collection chunks, EnumSet data) throws IOException { // Prune chunks List chunksToSave = chunks.stream() @@ -183,8 +173,26 @@ static byte[] serializeChunks(SlimeWorld world, Collection chunks, E outStream.writeInt(extra.length); outStream.write(extra); } + } - return outByteStream.toByteArray(); + private static void writeCompressed(DataOutputStream out, ThrowingConsumer writer) throws Exception { + ByteArrayOutputStream compressedOut = new ByteArrayOutputStream(); + ZstdOutputStream zstd = new ZstdOutputStream(compressedOut); + DataOutputStream dataOut = new DataOutputStream(zstd); + + CountingOutputStream counting = new CountingOutputStream(dataOut); + + // write uncompressed data into zstd stream + writer.accept(new DataOutputStream(counting)); + + dataOut.flush(); + zstd.close(); + + byte[] compressed = compressedOut.toByteArray(); + + out.writeInt(compressed.length); + out.writeInt((int) counting.getCount()); + out.write(compressed); } private static CompoundBinaryTag wrap(String key, ListBinaryTag list) { @@ -198,7 +206,6 @@ protected static byte[] serializeCompoundTag(CompoundBinaryTag tag) throws IOExc if (tag == null || tag.isEmpty()) return new byte[0]; ByteArrayOutputStream outByteStream = new ByteArrayOutputStream(); - //Avoid a buffered output stream by casting to DataOutput. Buffered Output Streams make the memory usage explode BinaryTagIO.writer().write(tag, (DataOutput) new DataOutputStream(outByteStream)); diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java index c8d366512..a8afb31c0 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java @@ -1,7 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v10; import com.github.luben.zstd.Zstd; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.serialization.slime.reader.VersionedByteSlimeWorldReader; @@ -9,7 +9,6 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java index 4a0172fe8..20713f30a 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java @@ -1,8 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v11; -import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -10,9 +9,8 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; -import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; +import com.infernalsuite.asp.util.LimitedInputStream; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -21,7 +19,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java index 084b06402..3bbfe970b 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java @@ -1,8 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v12; -import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -10,9 +9,8 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; -import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; +import com.infernalsuite.asp.util.LimitedInputStream; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -24,7 +22,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index b96c21807..932dc99d7 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -1,9 +1,8 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v13; -import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; import com.infernalsuite.asp.SlimeLogger; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -12,7 +11,7 @@ import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; -import com.infernalsuite.asp.serialization.slime.reader.LimitedInputStream; +import com.infernalsuite.asp.util.LimitedInputStream; import com.infernalsuite.asp.skeleton.SkeletonSlimeWorld; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java index 1086e2fd0..8c5196d1b 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java @@ -2,7 +2,7 @@ import com.github.luben.zstd.Zstd; import com.infernalsuite.asp.SlimeLogger; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.utils.NibbleArray; diff --git a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java index aa2a32fd9..08263ad2e 100644 --- a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java +++ b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.skeleton; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.utils.NibbleArray; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java index fdb7b85f9..edca3892f 100644 --- a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java +++ b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.skeleton; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.WorldAlreadyExistsException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java b/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java new file mode 100644 index 000000000..9279e564a --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java @@ -0,0 +1,29 @@ +package com.infernalsuite.asp.util; + +import java.io.IOException; +import java.io.OutputStream; + +public class CountingOutputStream extends OutputStream { + private final OutputStream out; + private long count = 0; + + public CountingOutputStream(OutputStream out) { + this.out = out; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + count++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + count += len; + } + + public long getCount() { + return count; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java similarity index 94% rename from core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java rename to core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java index 8a7b93159..b74476273 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/LimitedInputStream.java +++ b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java @@ -1,4 +1,4 @@ -package com.infernalsuite.asp.serialization.slime.reader; +package com.infernalsuite.asp.util; import java.io.IOException; import java.io.InputStream; diff --git a/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java b/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java new file mode 100644 index 000000000..0516ba4f8 --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java @@ -0,0 +1,7 @@ +package com.infernalsuite.asp.util; + +public interface ThrowingConsumer { + + public void accept(T value) throws Exception; + +} diff --git a/core/src/main/java/com/infernalsuite/asp/Util.java b/core/src/main/java/com/infernalsuite/asp/util/Util.java similarity index 85% rename from core/src/main/java/com/infernalsuite/asp/Util.java rename to core/src/main/java/com/infernalsuite/asp/util/Util.java index 1a3652ffe..0e1ed0dee 100644 --- a/core/src/main/java/com/infernalsuite/asp/Util.java +++ b/core/src/main/java/com/infernalsuite/asp/util/Util.java @@ -1,4 +1,4 @@ -package com.infernalsuite.asp; +package com.infernalsuite.asp.util; public final class Util { From 2cfdefce3e99885407c0e33bde251ae36cf613fc Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 03:55:57 +0200 Subject: [PATCH 6/7] fix: regression in loading speed --- .../slime/reader/impl/v11/v11SlimeWorldDeSerializer.java | 3 ++- .../slime/reader/impl/v12/v12SlimeWorldDeSerializer.java | 3 ++- .../slime/reader/impl/v13/v13SlimeWorldDeSerializer.java | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java index 20713f30a..ad125d624 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.BufferedInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; @@ -139,7 +140,7 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); - return new DataInputStream(inputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); } private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java index 3bbfe970b..254b82aad 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java @@ -22,6 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.BufferedInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; @@ -146,7 +147,7 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); - return new DataInputStream(inputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); } private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index 932dc99d7..e3bcd90ed 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -171,7 +171,7 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); - return new DataInputStream(inputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); } private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { From 54c51ccf105cf1e87d91c8dfdf88a7caf728b96c Mon Sep 17 00:00:00 2001 From: David Date: Sun, 5 Apr 2026 18:24:45 +0200 Subject: [PATCH 7/7] feat: close zstd streams, dont close parent stream on limited stream --- .../impl/v11/v11SlimeWorldDeSerializer.java | 21 +++++++++++-------- .../impl/v12/v12SlimeWorldDeSerializer.java | 21 +++++++++++-------- .../impl/v13/v13SlimeWorldDeSerializer.java | 20 ++++++++++-------- .../asp/util/LimitedInputStream.java | 5 +++++ 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java index ad125d624..95b2a0ec5 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java @@ -59,6 +59,8 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); + chunkBytes.close(); + dataStream.close(); return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } @@ -166,17 +168,18 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro if(decompressedLength == 0) return CompoundBinaryTag.empty(); LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); - ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { - //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory - //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) - CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); - //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, - //we might error out later - byte[] buffer = new byte[512]; - while (zstd.read(buffer) != -1) {} + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} - return tag; + return tag; + } } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java index 254b82aad..9a8422171 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java @@ -56,6 +56,8 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S worldPropertyMap.merge(propertyMap); } + chunkBytes.close(); + dataStream.close(); return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } @@ -173,17 +175,18 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro if(decompressedLength == 0) return CompoundBinaryTag.empty(); LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); - ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { - //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory - //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) - CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); - //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, - //we might error out later - byte[] buffer = new byte[512]; - while (zstd.read(buffer) != -1) {} + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} - return tag; + return tag; + } } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index e3bcd90ed..d3e76524c 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -54,6 +54,7 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S } + chunkBytes.close(); dataStream.close(); return new SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } @@ -197,18 +198,19 @@ private static DataInputStream openCompressedStream(DataInputStream stream) thro if(decompressedLength == 0) return CompoundBinaryTag.empty(); LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); - ZstdInputStream zstd = new ZstdInputStream(limitedInputStream); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { - //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory - //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) - CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); - //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, - //we might error out later - byte[] buffer = new byte[512]; - while (zstd.read(buffer) != -1) {} + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} - return tag; + return tag; + } } } diff --git a/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java index b74476273..88ad6d549 100644 --- a/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java +++ b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java @@ -39,4 +39,9 @@ public void drainRemaining() { } catch (IOException ignored) { } } + + @Override + public void close() throws IOException { + drainRemaining(); + } }