diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/MCAWorld.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/MCAWorld.java index f79e2531..b4a70032 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/MCAWorld.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/MCAWorld.java @@ -53,7 +53,7 @@ public class MCAWorld implements World { private final Path dimensionFolder; private final Path regionFolder; - private final ChunkLoader chunkLoader = new ChunkLoader(); + private final ChunkLoader chunkLoader = new ChunkLoader(this); private final LoadingCache regionCache = Caffeine.newBuilder() .executor(BlueMap.THREAD_POOL) .maximumSize(64) diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/ChunkLoader.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/ChunkLoader.java index d0d1c986..cc5f088a 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/ChunkLoader.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/ChunkLoader.java @@ -2,7 +2,7 @@ package de.bluecolored.bluemap.core.world.mca.chunk; import de.bluecolored.bluemap.core.storage.Compression; import de.bluecolored.bluemap.core.world.mca.MCAUtil; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.Nullable; @@ -16,6 +16,12 @@ import java.util.function.BiFunction; public class ChunkLoader { + private final MCAWorld world; + + public ChunkLoader(MCAWorld world) { + this.world = world; + } + // sorted list of chunk-versions, loaders at the start of the list are preferred over loaders at the end private static final List> CHUNK_VERSION_LOADERS = List.of( new ChunkVersionLoader<>(Chunk_1_18.Data.class, Chunk_1_18::new, 2844), @@ -26,7 +32,7 @@ public class ChunkLoader { private ChunkVersionLoader lastUsedLoader = CHUNK_VERSION_LOADERS.get(0); - public MCAChunk load(MCARegion region, byte[] data, int offset, int length, Compression compression) throws IOException { + public MCAChunk load(byte[] data, int offset, int length, Compression compression) throws IOException { InputStream in = new ByteArrayInputStream(data, offset, length); in.mark(-1); @@ -34,7 +40,7 @@ public class ChunkLoader { ChunkVersionLoader usedLoader = lastUsedLoader; MCAChunk chunk; try (InputStream decompressedIn = new BufferedInputStream(compression.decompress(in))) { - chunk = usedLoader.load(region, decompressedIn); + chunk = usedLoader.load(world, decompressedIn); } // check version and reload chunk if the wrong loader has been used and a better one has been found @@ -42,7 +48,7 @@ public class ChunkLoader { if (actualLoader != null && usedLoader != actualLoader) { in.reset(); // reset read position try (InputStream decompressedIn = new BufferedInputStream(compression.decompress(in))) { - chunk = actualLoader.load(region, decompressedIn); + chunk = actualLoader.load(world, decompressedIn); } lastUsedLoader = actualLoader; } @@ -62,12 +68,12 @@ public class ChunkLoader { private static class ChunkVersionLoader { private final Class dataType; - private final BiFunction constructor; + private final BiFunction constructor; private final int dataVersion; - public MCAChunk load(MCARegion region, InputStream in) throws IOException { + public MCAChunk load(MCAWorld world, InputStream in) throws IOException { D data = MCAUtil.BLUENBT.read(in, dataType); - return mightSupport(data.getDataVersion()) ? constructor.apply(region, data) : new MCAChunk(region, data) {}; + return mightSupport(data.getDataVersion()) ? constructor.apply(world, data) : new MCAChunk(world, data) {}; } public boolean mightSupport(int dataVersion) { diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_13.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_13.java index 083b3a5c..80e0a49a 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_13.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_13.java @@ -7,7 +7,7 @@ import de.bluecolored.bluemap.core.world.BlockState; import de.bluecolored.bluemap.core.world.DimensionType; import de.bluecolored.bluemap.core.world.LightData; import de.bluecolored.bluemap.core.world.mca.MCAUtil; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; import de.bluecolored.bluenbt.NBTName; import lombok.Getter; import org.jetbrains.annotations.Nullable; @@ -38,8 +38,8 @@ public class Chunk_1_13 extends MCAChunk { final int[] biomes; - public Chunk_1_13(MCARegion region, Data data) { - super(region, data); + public Chunk_1_13(MCAWorld world, Data data) { + super(world, data); Level level = data.level; @@ -50,7 +50,7 @@ public class Chunk_1_13 extends MCAChunk { STATUS_POSTPROCESSED.equals(level.status); this.inhabitedTime = level.inhabitedTime; - DimensionType dimensionType = getRegion().getWorld().getDimensionType(); + DimensionType dimensionType = getWorld().getDimensionType(); this.skyLight = dimensionType.hasSkylight() ? 16 : 0; this.worldSurfaceHeights = level.heightmaps.worldSurface; diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_15.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_15.java index 474ef263..99c347a9 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_15.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_15.java @@ -1,12 +1,12 @@ package de.bluecolored.bluemap.core.world.mca.chunk; import de.bluecolored.bluemap.core.world.Biome; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; public class Chunk_1_15 extends Chunk_1_13 { - public Chunk_1_15(MCARegion region, Data data) { - super(region, data); + public Chunk_1_15(MCAWorld world, Data data) { + super(world, data); } @Override diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_16.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_16.java index 1fbffba7..deee5462 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_16.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_16.java @@ -7,8 +7,8 @@ import de.bluecolored.bluemap.core.world.BlockState; import de.bluecolored.bluemap.core.world.DimensionType; import de.bluecolored.bluemap.core.world.LightData; import de.bluecolored.bluemap.core.world.mca.MCAUtil; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; import de.bluecolored.bluemap.core.world.mca.PackedIntArrayAccess; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; import de.bluecolored.bluenbt.NBTName; import lombok.Getter; import org.jetbrains.annotations.Nullable; @@ -37,8 +37,8 @@ public class Chunk_1_16 extends MCAChunk { private final int[] biomes; - public Chunk_1_16(MCARegion region, Data data) { - super(region, data); + public Chunk_1_16(MCAWorld world, Data data) { + super(world, data); Level level = data.level; @@ -46,7 +46,7 @@ public class Chunk_1_16 extends MCAChunk { this.hasLightData = STATUS_FULL.equals(level.status); this.inhabitedTime = level.inhabitedTime; - DimensionType dimensionType = getRegion().getWorld().getDimensionType(); + DimensionType dimensionType = getWorld().getDimensionType(); this.skyLight = dimensionType.hasSkylight() ? 16 : 0; int worldHeight = dimensionType.getHeight(); diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_18.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_18.java index 20dc9221..8dadce34 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_18.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_18.java @@ -7,8 +7,8 @@ import de.bluecolored.bluemap.core.world.BlockState; import de.bluecolored.bluemap.core.world.DimensionType; import de.bluecolored.bluemap.core.world.LightData; import de.bluecolored.bluemap.core.world.mca.MCAUtil; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; import de.bluecolored.bluemap.core.world.mca.PackedIntArrayAccess; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; import de.bluecolored.bluenbt.NBTName; import lombok.Getter; import org.jetbrains.annotations.Nullable; @@ -37,14 +37,14 @@ public class Chunk_1_18 extends MCAChunk { private final Section[] sections; private final int sectionMin, sectionMax; - public Chunk_1_18(MCARegion region, Data data) { - super(region, data); + public Chunk_1_18(MCAWorld world, Data data) { + super(world, data); this.generated = !STATUS_EMPTY.equals(data.status); this.hasLightData = STATUS_FULL.equals(data.status); this.inhabitedTime = data.inhabitedTime; - DimensionType dimensionType = getRegion().getWorld().getDimensionType(); + DimensionType dimensionType = getWorld().getDimensionType(); this.worldMinY = dimensionType.getMinY(); this.skyLight = dimensionType.hasSkylight() ? 16 : 0; diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/MCAChunk.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/MCAChunk.java index 77274236..d7fa66bc 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/MCAChunk.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/MCAChunk.java @@ -2,7 +2,7 @@ package de.bluecolored.bluemap.core.world.mca.chunk; import de.bluecolored.bluemap.core.world.BlockState; import de.bluecolored.bluemap.core.world.Chunk; -import de.bluecolored.bluemap.core.world.mca.region.MCARegion; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; import lombok.Getter; import lombok.ToString; @@ -20,11 +20,11 @@ public abstract class MCAChunk implements Chunk { protected static final String[] EMPTY_STRING_ARRAY = new String[0]; protected static final BlockState[] EMPTY_BLOCKSTATE_ARRAY = new BlockState[0]; - private final MCARegion region; + private final MCAWorld world; private final int dataVersion; - public MCAChunk(MCARegion region, Data chunkData) { - this.region = region; + public MCAChunk(MCAWorld world, Data chunkData) { + this.world = world; this.dataVersion = chunkData.getDataVersion(); } diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/LinearRegion.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/LinearRegion.java new file mode 100644 index 00000000..9ff42c3d --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/LinearRegion.java @@ -0,0 +1,211 @@ +/* + * This file is part of BlueMap, licensed under the MIT License (MIT). + * + * Copyright (c) Blue (Lukas Rieger) + * Copyright (c) contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package de.bluecolored.bluemap.core.world.mca.region; + +import com.flowpowered.math.vector.Vector2i; +import de.bluecolored.bluemap.core.storage.Compression; +import de.bluecolored.bluemap.core.world.ChunkConsumer; +import de.bluecolored.bluemap.core.world.Region; +import de.bluecolored.bluemap.core.world.mca.MCAWorld; +import de.bluecolored.bluemap.core.world.mca.chunk.MCAChunk; +import io.airlift.compress.zstd.ZstdInputStream; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/* + * LinearFormat: + * + * REGION-FILE: + * 8 byte - MAGIC value + * 1 byte - version + * 8 byte - region timestamp + * 1 byte - compression level + * 2 byte - chunk count + * 4 byte - data-length in bytes + * 8 byte - data-hash + * ? byte - data + * 8 byte - MAGIC value + * + * DATA: (zstd compressed) + * 32 * 32 * 8 - header: + * 4 byte - chunk-data-length + * 4 byte - timestamp + * ? - chunks + * + */ + +public class LinearRegion implements Region { + + public static final String FILE_SUFFIX = ".linear"; + + private static final long MAGIC = 0xc3ff13183cca9d9aL; + + private final MCAWorld world; + private final Path regionFile; + private final Vector2i regionPos; + + public LinearRegion(MCAWorld world, Path regionFile) throws IllegalArgumentException { + this.world = world; + this.regionFile = regionFile; + + String[] filenameParts = regionFile.getFileName().toString().split("\\."); + int rX = Integer.parseInt(filenameParts[1]); + int rZ = Integer.parseInt(filenameParts[2]); + + this.regionPos = new Vector2i(rX, rZ); + } + + public LinearRegion(MCAWorld world, Vector2i regionPos) throws IllegalArgumentException { + this.world = world; + this.regionPos = regionPos; + this.regionFile = world.getRegionFolder().resolve(getRegionFileName(regionPos.getX(), regionPos.getY())); + } + + @Override + public void iterateAllChunks(ChunkConsumer consumer) throws IOException { + if (Files.notExists(regionFile)) return; + + long fileLength = Files.size(regionFile); + if (fileLength == 0) return; + + int chunkStartX = regionPos.getX() * 32; + int chunkStartZ = regionPos.getY() * 32; + + byte[] chunkDataBuffer = null; + byte[] compressedData; + + byte version; + long newestTimestamp; + byte compressionLevel; + short chunkCount; + int dataLength; + long dataHash; + + try ( + InputStream in = Files.newInputStream(regionFile, StandardOpenOption.READ); + BufferedInputStream bIn = new BufferedInputStream(in); + DataInputStream dIn = new DataInputStream(bIn) + ) { + if (dIn.readLong() != MAGIC) + throw new IOException("Linear region-file format: invalid header magic"); + + // read the header + version = dIn.readByte(); + newestTimestamp = dIn.readLong(); + compressionLevel = dIn.readByte(); + chunkCount = dIn.readShort(); + dataLength = dIn.readInt(); + dataHash = dIn.readLong(); + + if (version < 1 || version > 2) + throw new IOException("Linear region-file format: Unsupported version: " + version); + + if (fileLength != dataLength + 40) // 40 = header + footer + throw new IOException("Linear region-file format: Invalid file length. Expected " + (dataLength + 40) + " but got " + fileLength); + + compressedData = new byte[dataLength]; + dIn.readFully(compressedData, 0, dataLength); + + if (dIn.readLong() != MAGIC) + throw new IOException("Linear region-file format: invalid footer magic"); + + } + + try ( + InputStream in = new ZstdInputStream(new ByteArrayInputStream(compressedData)); + BufferedInputStream bIn = new BufferedInputStream(in); + DataInputStream dIn = new DataInputStream(bIn) + ) { + int[] chunkDataLengths = new int[1024]; + int[] chunkTimestamps = new int[1024]; + for (int i = 0 ; i < 1024 ; i++) { + chunkDataLengths[i] = dIn.readInt(); + chunkTimestamps[i] = dIn.readInt(); + } + + int i = 0; + int toBeSkipped = 0; + for (int z = 0; z < 32; z++) { + for (int x = 0; x < 32; x++) { + int length = chunkDataLengths[i]; + if (length > 0) { + int chunkX = chunkStartX + x; + int chunkZ = chunkStartZ + z; + long timestamp = version == 2 ? chunkTimestamps[i] : newestTimestamp; + + if (consumer.filter(chunkX, chunkZ, timestamp)) { + if (toBeSkipped > 0) skipNBytes(dIn, toBeSkipped); + + if (chunkDataBuffer == null || chunkDataBuffer.length < length) + chunkDataBuffer = new byte[length]; + dIn.readFully(chunkDataBuffer, 0, length); + + MCAChunk chunk = world.getChunkLoader().load(chunkDataBuffer, 0, length, Compression.NONE); + consumer.accept(chunkX, chunkZ, chunk); + } else { + // skip before reading the next chunk, but only if there is a next chunk + // that we actually want to read, to avoid decompressing unnecessary data + toBeSkipped += length; + } + } + + i++; + } + } + + } + } + + public static String getRegionFileName(int regionX, int regionZ) { + return "r." + regionX + "." + regionZ + FILE_SUFFIX; + } + + /** + * This method is taken here from a newer version of {@link InputStream}, + * to ensure Java 11 compatibility. + */ + private static void skipNBytes(InputStream in, long n) throws IOException { + while (n > 0) { + long ns = in.skip(n); + if (ns > 0 && ns <= n) { + // adjust number to skip + n -= ns; + } else if (ns == 0) { // no bytes skipped + // read one byte to check for EOS + if (in.read() == -1) { + throw new EOFException(); + } + // one byte read so decrement number to skip + n--; + } else { // skipped negative or too many bytes + throw new IOException("Unable to skip exactly"); + } + } + } + +} \ No newline at end of file diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/MCARegion.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/MCARegion.java index 71cedcba..12157d49 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/MCARegion.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/MCARegion.java @@ -168,7 +168,7 @@ public class MCARegion implements Region { default: throw new IOException("Unknown chunk compression-id: " + compressionTypeId); } - return world.getChunkLoader().load(this, data, 5, size - 5, compression); + return world.getChunkLoader().load(data, 5, size - 5, compression); } public static String getRegionFileName(int regionX, int regionZ) { diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/RegionType.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/RegionType.java index 434da5f0..edf91739 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/RegionType.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/RegionType.java @@ -34,8 +34,8 @@ import java.nio.file.Path; public enum RegionType { - MCA (MCARegion::new, MCARegion.FILE_SUFFIX, MCARegion::getRegionFileName); - //LINEAR (LinearRegion::new, LinearRegion.FILE_SUFFIX, LinearRegion::getRegionFileName); + MCA (MCARegion::new, MCARegion.FILE_SUFFIX, MCARegion::getRegionFileName), + LINEAR (LinearRegion::new, LinearRegion.FILE_SUFFIX, LinearRegion::getRegionFileName); // we do this to improve performance, as calling values() creates a new array each time private final static RegionType[] VALUES = values();