diff --git a/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java b/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java index 1fa4802dd..c900ef914 100644 --- a/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java +++ b/aspaper-server/src/main/java/com/infernalsuite/asp/SimpleDataFixerConverter.java @@ -15,6 +15,7 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; +import com.infernalsuite.asp.util.Util; import net.kyori.adventure.nbt.CompoundBinaryTag; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; diff --git a/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java b/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java index 7fb69fc6c..b4b7b5359 100644 --- a/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java +++ b/aspaper-server/src/main/java/com/infernalsuite/asp/level/SlimeInMemoryWorld.java @@ -3,7 +3,7 @@ import ca.spottedleaf.moonrise.patches.chunk_system.level.entity.ChunkEntitySlices; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.PoiChunk; import com.infernalsuite.asp.Converter; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.WorldAlreadyExistsException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.world.properties.SlimeProperties; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java b/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java index dfd6c0729..91dafa14b 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.serialization.anvil; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.InvalidWorldException; import com.infernalsuite.asp.api.utils.NibbleArray; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java index a11791c7f..fd0194f9a 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/SlimeSerializer.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.serialization.slime; -import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdOutputStream; import com.infernalsuite.asp.api.utils.SlimeFormat; import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; @@ -8,6 +8,8 @@ import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; import com.infernalsuite.asp.serialization.slime.reader.impl.v13.v13AdditionalWorldData; +import com.infernalsuite.asp.util.CountingOutputStream; +import com.infernalsuite.asp.util.ThrowingConsumer; import net.kyori.adventure.nbt.BinaryTag; import net.kyori.adventure.nbt.BinaryTagIO; import net.kyori.adventure.nbt.BinaryTagTypes; @@ -16,9 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; +import java.io.*; import java.util.*; public class SlimeSerializer { @@ -60,20 +60,13 @@ public static byte[] serialize(SlimeWorld world) { outStream.writeByte(v13AdditionalWorldData.fromSet(additionalWorldData)); // Chunks - byte[] chunkData = serializeChunks(world, world.getChunkStorage(), additionalWorldData); - byte[] compressedChunkData = Zstd.compress(chunkData); - outStream.writeInt(compressedChunkData.length); - outStream.writeInt(chunkData.length); - outStream.write(compressedChunkData); - - // Extra Tag - byte[] extra = serializeCompoundTag(CompoundBinaryTag.builder().put(extraData).build()); - byte[] compressedExtra = Zstd.compress(extra); + writeCompressed(outStream, value -> serializeChunks(value, world, world.getChunkStorage(), additionalWorldData)); - outStream.writeInt(compressedExtra.length); - outStream.writeInt(extra.length); - outStream.write(compressedExtra); + writeCompressed(outStream, value -> { + //Avoid a buffered output stream by casting to DataOutput. Buffered Output Streams make the memory usage explode + BinaryTagIO.writer().write(CompoundBinaryTag.builder().put(extraData).build(), (DataOutput) new DataOutputStream(value)); + }); } catch (Exception e) { throw new RuntimeException(e); @@ -82,9 +75,7 @@ public static byte[] serialize(SlimeWorld world) { return outByteStream.toByteArray(); } - static byte[] serializeChunks(SlimeWorld world, Collection chunks, EnumSet data) throws IOException { - ByteArrayOutputStream outByteStream = new ByteArrayOutputStream(16384); - DataOutputStream outStream = new DataOutputStream(outByteStream); + static void serializeChunks(DataOutputStream outStream, SlimeWorld world, Collection chunks, EnumSet data) throws IOException { // Prune chunks List chunksToSave = chunks.stream() @@ -175,15 +166,33 @@ static byte[] serializeChunks(SlimeWorld world, Collection chunks, E // Extra Tag if (chunk.getExtraData() == null) { - LOGGER.warn("Chunk at " + chunk.getX() + ", " + chunk.getZ() + " from world " + world.getName() + " has no extra data! When deserialized, this chunk will have an empty extra data tag!"); + LOGGER.warn("Chunk at {}, {} from world {} has no extra data! When deserialized, this chunk will have an empty extra data tag!", chunk.getX(), chunk.getZ(), world.getName()); } byte[] extra = serializeCompoundTag(CompoundBinaryTag.from(chunk.getExtraData())); outStream.writeInt(extra.length); outStream.write(extra); } + } - return outByteStream.toByteArray(); + private static void writeCompressed(DataOutputStream out, ThrowingConsumer writer) throws Exception { + ByteArrayOutputStream compressedOut = new ByteArrayOutputStream(); + ZstdOutputStream zstd = new ZstdOutputStream(compressedOut); + DataOutputStream dataOut = new DataOutputStream(zstd); + + CountingOutputStream counting = new CountingOutputStream(dataOut); + + // write uncompressed data into zstd stream + writer.accept(new DataOutputStream(counting)); + + dataOut.flush(); + zstd.close(); + + byte[] compressed = compressedOut.toByteArray(); + + out.writeInt(compressed.length); + out.writeInt((int) counting.getCount()); + out.write(compressed); } private static CompoundBinaryTag wrap(String key, ListBinaryTag list) { @@ -197,7 +206,8 @@ protected static byte[] serializeCompoundTag(CompoundBinaryTag tag) throws IOExc if (tag == null || tag.isEmpty()) return new byte[0]; ByteArrayOutputStream outByteStream = new ByteArrayOutputStream(); - BinaryTagIO.writer().write(tag, outByteStream); + //Avoid a buffered output stream by casting to DataOutput. Buffered Output Streams make the memory usage explode + BinaryTagIO.writer().write(tag, (DataOutput) new DataOutputStream(outByteStream)); return outByteStream.toByteArray(); } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java index c8d366512..a8afb31c0 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java @@ -1,7 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v10; import com.github.luben.zstd.Zstd; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.serialization.slime.reader.VersionedByteSlimeWorldReader; @@ -9,7 +9,6 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java index fcb25b70d..95b2a0ec5 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java @@ -1,7 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v11; -import com.github.luben.zstd.Zstd; -import com.infernalsuite.asp.Util; +import com.github.luben.zstd.ZstdInputStream; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -9,15 +9,18 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.util.LimitedInputStream; +import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import net.kyori.adventure.nbt.*; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; +import java.io.BufferedInputStream; +import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; @@ -35,11 +38,10 @@ public class v11SlimeWorldDeSerializer implements com.infernalsuite.asp.serializ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException { int worldVersion = dataStream.readInt(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); SlimePropertyMap worldPropertyMap = propertyMap; CompoundBinaryTag propertiesMap = extraTag.get("properties") != null @@ -57,12 +59,13 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); + chunkBytes.close(); + dataStream.close(); return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -97,33 +100,20 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); - chunkSections[sectionId] = new com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); + chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); // Tile Entities - int compressedTileEntitiesLength = chunkData.readInt(); - int decompressedTileEntitiesLength = chunkData.readInt(); - byte[] compressedTileEntitiesData = new byte[compressedTileEntitiesLength]; - byte[] decompressedTileEntitiesData = new byte[decompressedTileEntitiesLength]; - chunkData.read(compressedTileEntitiesData); - Zstd.decompress(decompressedTileEntitiesData, compressedTileEntitiesData); - - CompoundBinaryTag tileEntitiesCompound = readCompound(decompressedTileEntitiesData); + CompoundBinaryTag tileEntitiesCompound = readCompressedCompound(chunkData); ListBinaryTag tileEntitiesTag = tileEntitiesCompound.getList("tileEntities", BinaryTagTypes.COMPOUND); List serializedTileEntities = new ArrayList<>(tileEntitiesTag.size()); @@ -133,14 +123,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - int compressedEntitiesLength = chunkData.readInt(); - int decompressedEntitiesLength = chunkData.readInt(); - byte[] compressedEntitiesData = new byte[compressedEntitiesLength]; - byte[] decompressedEntitiesData = new byte[decompressedEntitiesLength]; - chunkData.read(compressedEntitiesData); - Zstd.decompress(decompressedEntitiesData, compressedEntitiesData); - - CompoundBinaryTag entitiesCompound = readCompound(decompressedEntitiesData); + CompoundBinaryTag entitiesCompound = readCompressedCompound(chunkData); ListBinaryTag entitiesTag = entitiesCompound.getList("entities", BinaryTagTypes.COMPOUND); List serializedEntities = new ArrayList<>(entitiesTag.size()); for (BinaryTag binaryTag : entitiesTag) { @@ -153,19 +136,50 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; + stream.readInt(); //Decompressed length, legacy + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); + } + + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { + int length = stream.readInt(); + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + limitedInputStream.drainRemaining(); + return tag; } - private static CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} + + return tag; + } } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java index 71516aa23..9a8422171 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java @@ -1,7 +1,7 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v12; -import com.github.luben.zstd.Zstd; -import com.infernalsuite.asp.Util; +import com.github.luben.zstd.ZstdInputStream; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -9,8 +9,8 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.util.LimitedInputStream; import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -22,7 +22,8 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; +import java.io.BufferedInputStream; +import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.util.Collections; @@ -40,11 +41,10 @@ public class v12SlimeWorldDeSerializer implements com.infernalsuite.asp.serializ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException { int worldVersion = dataStream.readInt(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -56,12 +56,13 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S worldPropertyMap.merge(propertyMap); } + chunkBytes.close(); + dataStream.close(); return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -79,7 +80,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray blockLightArray; if (chunkData.readBoolean()) { byte[] blockLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(blockLightByteArray); + chunkData.readFully(blockLightByteArray); blockLightArray = new NibbleArray(blockLightByteArray); } else { blockLightArray = null; @@ -89,35 +90,28 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray skyLightArray; if (chunkData.readBoolean()) { byte[] skyLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(skyLightByteArray); + chunkData.readFully(skyLightByteArray); skyLightArray = new NibbleArray(skyLightByteArray); } else { skyLightArray = null; } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); // Tile Entities - byte[] tileEntitiesRaw = read(chunkData); List tileEntities; - CompoundBinaryTag tileEntitiesCompound = readCompound(tileEntitiesRaw); + CompoundBinaryTag tileEntitiesCompound = readLimitedCompound(chunkData); if (tileEntitiesCompound.isEmpty()) { tileEntities = Collections.emptyList(); } else { @@ -128,9 +122,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - byte[] entitiesRaw = read(chunkData); List entities; - CompoundBinaryTag entitiesCompound = readCompound(entitiesRaw); + CompoundBinaryTag entitiesCompound = readLimitedCompound(chunkData); if (entitiesCompound.isEmpty()) { entities = Collections.emptyList(); } else { @@ -140,8 +133,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Extra Tag - byte[] rawExtra = read(chunkData); - CompoundBinaryTag extra = readCompound(rawExtra); + CompoundBinaryTag extra = readLimitedCompound(chunkData); Map extraData = new HashMap<>(); extra.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -151,26 +143,50 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; + stream.readInt(); //Decompressed length, legacy + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); } - private static byte[] read(DataInputStream stream) throws IOException { + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { int length = stream.readInt(); - byte[] data = new byte[length]; - stream.read(data); - return data; + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + limitedInputStream.drainRemaining(); + return tag; } - private static @NotNull CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} + + return tag; + } } } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java index 38cbac883..d3e76524c 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v13/v13SlimeWorldDeSerializer.java @@ -1,8 +1,8 @@ package com.infernalsuite.asp.serialization.slime.reader.impl.v13; -import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.infernalsuite.asp.SlimeLogger; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.exceptions.NewerFormatException; import com.infernalsuite.asp.api.loaders.SlimeLoader; @@ -10,8 +10,10 @@ import com.infernalsuite.asp.api.world.SlimeChunk; import com.infernalsuite.asp.api.world.SlimeChunkSection; import com.infernalsuite.asp.api.world.SlimeWorld; -import com.infernalsuite.asp.api.world.properties.SlimeProperties; import com.infernalsuite.asp.api.world.properties.SlimePropertyMap; +import com.infernalsuite.asp.util.LimitedInputStream; +import com.infernalsuite.asp.skeleton.SkeletonSlimeWorld; +import com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton; import com.infernalsuite.asp.skeleton.SlimeChunkSkeleton; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; @@ -19,9 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; +import java.io.*; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -38,11 +38,10 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S int worldVersion = dataStream.readInt(); byte additionalWorldData = dataStream.readByte(); - byte[] chunkBytes = readCompressed(dataStream); + DataInputStream chunkBytes = openCompressedStream(dataStream); Long2ObjectMap chunks = readChunks(propertyMap, additionalWorldData, chunkBytes); - byte[] extraTagBytes = readCompressed(dataStream); - CompoundBinaryTag extraTag = readCompound(extraTagBytes); + CompoundBinaryTag extraTag = readCompressedCompound(dataStream); ConcurrentMap extraData = new ConcurrentHashMap<>(); extraTag.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -54,12 +53,14 @@ public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, S worldPropertyMap.merge(propertyMap); } - return new com.infernalsuite.asp.skeleton.SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); + + chunkBytes.close(); + dataStream.close(); + return new SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraData, worldPropertyMap, worldVersion); } - private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte additionalWorldData, byte[] chunkBytes) throws IOException { + private static Long2ObjectMap readChunks(SlimePropertyMap slimePropertyMap, byte additionalWorldData, DataInputStream chunkData) throws IOException { Long2ObjectMap chunkMap = new Long2ObjectOpenHashMap<>(); - DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes)); int chunks = chunkData.readInt(); for (int i = 0; i < chunks; i++) { @@ -77,7 +78,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray blockLightArray; if ((sectionFlags & 1) == 1) { byte[] blockLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(blockLightByteArray); + chunkData.readFully(blockLightByteArray); blockLightArray = new NibbleArray(blockLightByteArray); } else { blockLightArray = null; @@ -87,49 +88,37 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope NibbleArray skyLightArray; if (((sectionFlags >> 1) & 1) == 1) { byte[] skyLightByteArray = new byte[ARRAY_SIZE]; - chunkData.read(skyLightByteArray); + chunkData.readFully(skyLightByteArray); skyLightArray = new NibbleArray(skyLightByteArray); } else { skyLightArray = null; } // Block Data - byte[] blockStateData = new byte[chunkData.readInt()]; - chunkData.read(blockStateData); - CompoundBinaryTag blockStateTag = readCompound(blockStateData); + CompoundBinaryTag blockStateTag = readLimitedCompound(chunkData); // Biome Data - byte[] biomeData = new byte[chunkData.readInt()]; - chunkData.read(biomeData); - CompoundBinaryTag biomeTag = readCompound(biomeData); + CompoundBinaryTag biomeTag = readLimitedCompound(chunkData); - chunkSections[sectionId] = new com.infernalsuite.asp.skeleton.SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); + chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray); } // HeightMaps - byte[] heightMapData = new byte[chunkData.readInt()]; - chunkData.read(heightMapData); - CompoundBinaryTag heightMaps = readCompound(heightMapData); + CompoundBinaryTag heightMaps = readLimitedCompound(chunkData); CompoundBinaryTag poiChunk = null; if(v13AdditionalWorldData.POI_CHUNKS.isSet(additionalWorldData)) { - byte[] poiData = new byte[chunkData.readInt()]; - chunkData.read(poiData); - poiChunk = readCompound(poiData); + poiChunk = readLimitedCompound(chunkData); } ListBinaryTag blockTicks = null; if(v13AdditionalWorldData.BLOCK_TICKS.isSet(additionalWorldData)) { - byte[] blockTickData = new byte[chunkData.readInt()]; - chunkData.read(blockTickData); - CompoundBinaryTag tag = readCompound(blockTickData); + CompoundBinaryTag tag = readLimitedCompound(chunkData); blockTicks = tag.getList("block_ticks", BinaryTagTypes.COMPOUND); } ListBinaryTag fluidTicks = null; if(v13AdditionalWorldData.FLUID_TICKS.isSet(additionalWorldData)) { - byte[] fluidTickData = new byte[chunkData.readInt()]; - chunkData.read(fluidTickData); - CompoundBinaryTag tag = readCompound(fluidTickData); + CompoundBinaryTag tag = readLimitedCompound(chunkData); fluidTicks = tag.getList("fluid_ticks", BinaryTagTypes.COMPOUND); } @@ -144,9 +133,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Tile Entities - byte[] tileEntitiesRaw = read(chunkData); List tileEntities; - CompoundBinaryTag tileEntitiesCompound = readCompound(tileEntitiesRaw); + CompoundBinaryTag tileEntitiesCompound = readLimitedCompound(chunkData); if (tileEntitiesCompound.isEmpty()) { tileEntities = Collections.emptyList(); } else { @@ -157,9 +145,8 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope // Entities - byte[] entitiesRaw = read(chunkData); List entities; - CompoundBinaryTag entitiesCompound = readCompound(entitiesRaw); + CompoundBinaryTag entitiesCompound = readLimitedCompound(chunkData); if (entitiesCompound.isEmpty()) { entities = Collections.emptyList(); } else { @@ -169,8 +156,7 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope } // Extra Tag - byte[] rawExtra = read(chunkData); - CompoundBinaryTag extra = readCompound(rawExtra); + CompoundBinaryTag extra = readLimitedCompound(chunkData); Map extraData = new HashMap<>(); extra.forEach(entry -> extraData.put(entry.getKey(), entry.getValue())); @@ -180,26 +166,51 @@ private static Long2ObjectMap readChunks(SlimePropertyMap slimePrope return chunkMap; } - private static byte[] readCompressed(DataInputStream stream) throws IOException { + private static DataInputStream openCompressedStream(DataInputStream stream) throws IOException { int compressedLength = stream.readInt(); - int decompressedLength = stream.readInt(); - byte[] compressedData = new byte[compressedLength]; - byte[] decompressedData = new byte[decompressedLength]; - stream.read(compressedData); - Zstd.decompress(decompressedData, compressedData); - return decompressedData; + stream.readInt(); //Decompressed length, legacy + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + ZstdInputStream inputStream = new ZstdInputStream(limitedInputStream); + return new DataInputStream(new BufferedInputStream(inputStream)); } - private static byte[] read(DataInputStream stream) throws IOException { + private static @NotNull CompoundBinaryTag readLimitedCompound(DataInputStream stream) throws IOException { int length = stream.readInt(); - byte[] data = new byte[length]; - stream.read(data); - return data; + if(length == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, length); + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(limitedInputStream)); + + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + limitedInputStream.drainRemaining(); + return tag; } - private static @NotNull CompoundBinaryTag readCompound(byte[] tagBytes) throws IOException { - if (tagBytes.length == 0) return CompoundBinaryTag.empty(); + private static @NotNull CompoundBinaryTag readCompressedCompound(DataInputStream stream) throws IOException { + int compressedLength = stream.readInt(); + int decompressedLength = stream.readInt(); + + if(decompressedLength == 0) return CompoundBinaryTag.empty(); + + LimitedInputStream limitedInputStream = new LimitedInputStream(stream, compressedLength); + try(ZstdInputStream zstd = new ZstdInputStream(limitedInputStream)) { + + //Avoid a buffered input stream by casting to DataInput. Buffered Input Streams make the memory + //usage explode (e.g. with buffered streams here 1,3gb; with a data input directly: 300mb) + CompoundBinaryTag tag = BinaryTagIO.unlimitedReader().read((DataInput) new DataInputStream(zstd)); - return BinaryTagIO.unlimitedReader().read(new ByteArrayInputStream(tagBytes)); + //binary tag reading does not guarantee that the buffer is fully read. If we don't do this, + //we might error out later + byte[] buffer = new byte[512]; + while (zstd.read(buffer) != -1) {} + + return tag; + } } + } diff --git a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java index 1086e2fd0..8c5196d1b 100644 --- a/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java +++ b/core/src/main/java/com/infernalsuite/asp/serialization/slime/reader/impl/v1_9/v1_9SlimeWorldDeserializer.java @@ -2,7 +2,7 @@ import com.github.luben.zstd.Zstd; import com.infernalsuite.asp.SlimeLogger; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.CorruptedWorldException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.utils.NibbleArray; diff --git a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java index aa2a32fd9..08263ad2e 100644 --- a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java +++ b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonCloning.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.skeleton; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.utils.NibbleArray; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java index fdb7b85f9..edca3892f 100644 --- a/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java +++ b/core/src/main/java/com/infernalsuite/asp/skeleton/SkeletonSlimeWorld.java @@ -1,6 +1,6 @@ package com.infernalsuite.asp.skeleton; -import com.infernalsuite.asp.Util; +import com.infernalsuite.asp.util.Util; import com.infernalsuite.asp.api.exceptions.WorldAlreadyExistsException; import com.infernalsuite.asp.api.loaders.SlimeLoader; import com.infernalsuite.asp.api.world.SlimeChunk; diff --git a/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java b/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java new file mode 100644 index 000000000..9279e564a --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/util/CountingOutputStream.java @@ -0,0 +1,29 @@ +package com.infernalsuite.asp.util; + +import java.io.IOException; +import java.io.OutputStream; + +public class CountingOutputStream extends OutputStream { + private final OutputStream out; + private long count = 0; + + public CountingOutputStream(OutputStream out) { + this.out = out; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + count++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + count += len; + } + + public long getCount() { + return count; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java new file mode 100644 index 000000000..88ad6d549 --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/util/LimitedInputStream.java @@ -0,0 +1,47 @@ +package com.infernalsuite.asp.util; + +import java.io.IOException; +import java.io.InputStream; + +public class LimitedInputStream extends InputStream { + private final InputStream in; + private int remaining; + + public LimitedInputStream(InputStream in, int limit) { + this.in = in; + this.remaining = limit; + } + + @Override + public int read() throws IOException { + if (remaining <= 0) return -1; + int b = in.read(); + if (b != -1) remaining--; + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (remaining <= 0) return -1; + len = Math.min(len, remaining); + int read = in.read(b, off, len); + if (read > 0) remaining -= read; + return read; + } + + public void drainRemaining() { + byte[] buffer = new byte[512]; + try { + while (remaining > 0) { + int read = read(buffer, 0, Math.min(buffer.length, remaining)); + if (read == -1) break; + } + } catch (IOException ignored) { + } + } + + @Override + public void close() throws IOException { + drainRemaining(); + } +} diff --git a/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java b/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java new file mode 100644 index 000000000..0516ba4f8 --- /dev/null +++ b/core/src/main/java/com/infernalsuite/asp/util/ThrowingConsumer.java @@ -0,0 +1,7 @@ +package com.infernalsuite.asp.util; + +public interface ThrowingConsumer { + + public void accept(T value) throws Exception; + +} diff --git a/core/src/main/java/com/infernalsuite/asp/Util.java b/core/src/main/java/com/infernalsuite/asp/util/Util.java similarity index 85% rename from core/src/main/java/com/infernalsuite/asp/Util.java rename to core/src/main/java/com/infernalsuite/asp/util/Util.java index 1a3652ffe..0e1ed0dee 100644 --- a/core/src/main/java/com/infernalsuite/asp/Util.java +++ b/core/src/main/java/com/infernalsuite/asp/util/Util.java @@ -1,4 +1,4 @@ -package com.infernalsuite.asp; +package com.infernalsuite.asp.util; public final class Util {