Upstream
/
BlueMap
mirror of https://github.com/BlueMap-Minecraft/BlueMap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Reimplement Linear region file format support

This commit is contained in:
Lukas Rieger (Blue) 2024-02-22 23:23:56 +01:00
parent ff1e38a7e1
commit dbde93c9f5
No known key found for this signature in database
GPG Key ID: AA33883B1BBA03E6
10 changed files with 247 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/MCAWorld.java
View File

@ -53,7 +53,7 @@ public class MCAWorld implements World {
private final Path dimensionFolder;
private final Path regionFolder;
private final ChunkLoader chunkLoader = new ChunkLoader();
private final ChunkLoader chunkLoader = new ChunkLoader(this);
private final LoadingCache<Vector2i, Region> regionCache = Caffeine.newBuilder()
.executor(BlueMap.THREAD_POOL)
.maximumSize(64)

20
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/ChunkLoader.java
View File

@ -2,7 +2,7 @@ package de.bluecolored.bluemap.core.world.mca.chunk;
import de.bluecolored.bluemap.core.storage.Compression;
import de.bluecolored.bluemap.core.world.mca.MCAUtil;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.Nullable;
@ -16,6 +16,12 @@ import java.util.function.BiFunction;
public class ChunkLoader {
private final MCAWorld world;
public ChunkLoader(MCAWorld world) {
this.world = world;
}
// sorted list of chunk-versions, loaders at the start of the list are preferred over loaders at the end
private static final List<ChunkVersionLoader<?>> CHUNK_VERSION_LOADERS = List.of(
new ChunkVersionLoader<>(Chunk_1_18.Data.class, Chunk_1_18::new, 2844),
@ -26,7 +32,7 @@ public class ChunkLoader {
private ChunkVersionLoader<?> lastUsedLoader = CHUNK_VERSION_LOADERS.get(0);
public MCAChunk load(MCARegion region, byte[] data, int offset, int length, Compression compression) throws IOException {
public MCAChunk load(byte[] data, int offset, int length, Compression compression) throws IOException {
InputStream in = new ByteArrayInputStream(data, offset, length);
in.mark(-1);
@ -34,7 +40,7 @@ public class ChunkLoader {
ChunkVersionLoader<?> usedLoader = lastUsedLoader;
MCAChunk chunk;
try (InputStream decompressedIn = new BufferedInputStream(compression.decompress(in))) {
chunk = usedLoader.load(region, decompressedIn);
chunk = usedLoader.load(world, decompressedIn);
}
// check version and reload chunk if the wrong loader has been used and a better one has been found
@ -42,7 +48,7 @@ public class ChunkLoader {
if (actualLoader != null && usedLoader != actualLoader) {
in.reset(); // reset read position
try (InputStream decompressedIn = new BufferedInputStream(compression.decompress(in))) {
chunk = actualLoader.load(region, decompressedIn);
chunk = actualLoader.load(world, decompressedIn);
}
lastUsedLoader = actualLoader;
}
@ -62,12 +68,12 @@ public class ChunkLoader {
private static class ChunkVersionLoader<D extends MCAChunk.Data> {
private final Class<D> dataType;
private final BiFunction<MCARegion, D, MCAChunk> constructor;
private final BiFunction<MCAWorld, D, MCAChunk> constructor;
private final int dataVersion;
public MCAChunk load(MCARegion region, InputStream in) throws IOException {
public MCAChunk load(MCAWorld world, InputStream in) throws IOException {
D data = MCAUtil.BLUENBT.read(in, dataType);
return mightSupport(data.getDataVersion()) ? constructor.apply(region, data) : new MCAChunk(region, data) {};
return mightSupport(data.getDataVersion()) ? constructor.apply(world, data) : new MCAChunk(world, data) {};
}
public boolean mightSupport(int dataVersion) {

8
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_13.java
View File

@ -7,7 +7,7 @@ import de.bluecolored.bluemap.core.world.BlockState;
import de.bluecolored.bluemap.core.world.DimensionType;
import de.bluecolored.bluemap.core.world.LightData;
import de.bluecolored.bluemap.core.world.mca.MCAUtil;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import de.bluecolored.bluenbt.NBTName;
import lombok.Getter;
import org.jetbrains.annotations.Nullable;
@ -38,8 +38,8 @@ public class Chunk_1_13 extends MCAChunk {
final int[] biomes;
public Chunk_1_13(MCARegion region, Data data) {
super(region, data);
public Chunk_1_13(MCAWorld world, Data data) {
super(world, data);
Level level = data.level;
@ -50,7 +50,7 @@ public class Chunk_1_13 extends MCAChunk {
STATUS_POSTPROCESSED.equals(level.status);
this.inhabitedTime = level.inhabitedTime;
DimensionType dimensionType = getRegion().getWorld().getDimensionType();
DimensionType dimensionType = getWorld().getDimensionType();
this.skyLight = dimensionType.hasSkylight() ? 16 : 0;
this.worldSurfaceHeights = level.heightmaps.worldSurface;

6
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_15.java
View File

@ -1,12 +1,12 @@
package de.bluecolored.bluemap.core.world.mca.chunk;
import de.bluecolored.bluemap.core.world.Biome;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
public class Chunk_1_15 extends Chunk_1_13 {
public Chunk_1_15(MCARegion region, Data data) {
super(region, data);
public Chunk_1_15(MCAWorld world, Data data) {
super(world, data);
}
@Override

8
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_16.java
View File

@ -7,8 +7,8 @@ import de.bluecolored.bluemap.core.world.BlockState;
import de.bluecolored.bluemap.core.world.DimensionType;
import de.bluecolored.bluemap.core.world.LightData;
import de.bluecolored.bluemap.core.world.mca.MCAUtil;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.mca.PackedIntArrayAccess;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluenbt.NBTName;
import lombok.Getter;
import org.jetbrains.annotations.Nullable;
@ -37,8 +37,8 @@ public class Chunk_1_16 extends MCAChunk {
private final int[] biomes;
public Chunk_1_16(MCARegion region, Data data) {
super(region, data);
public Chunk_1_16(MCAWorld world, Data data) {
super(world, data);
Level level = data.level;
@ -46,7 +46,7 @@ public class Chunk_1_16 extends MCAChunk {
this.hasLightData = STATUS_FULL.equals(level.status);
this.inhabitedTime = level.inhabitedTime;
DimensionType dimensionType = getRegion().getWorld().getDimensionType();
DimensionType dimensionType = getWorld().getDimensionType();
this.skyLight = dimensionType.hasSkylight() ? 16 : 0;
int worldHeight = dimensionType.getHeight();

8
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/Chunk_1_18.java
View File

@ -7,8 +7,8 @@ import de.bluecolored.bluemap.core.world.BlockState;
import de.bluecolored.bluemap.core.world.DimensionType;
import de.bluecolored.bluemap.core.world.LightData;
import de.bluecolored.bluemap.core.world.mca.MCAUtil;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.mca.PackedIntArrayAccess;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluenbt.NBTName;
import lombok.Getter;
import org.jetbrains.annotations.Nullable;
@ -37,14 +37,14 @@ public class Chunk_1_18 extends MCAChunk {
private final Section[] sections;
private final int sectionMin, sectionMax;
public Chunk_1_18(MCARegion region, Data data) {
super(region, data);
public Chunk_1_18(MCAWorld world, Data data) {
super(world, data);
this.generated = !STATUS_EMPTY.equals(data.status);
this.hasLightData = STATUS_FULL.equals(data.status);
this.inhabitedTime = data.inhabitedTime;
DimensionType dimensionType = getRegion().getWorld().getDimensionType();
DimensionType dimensionType = getWorld().getDimensionType();
this.worldMinY = dimensionType.getMinY();
this.skyLight = dimensionType.hasSkylight() ? 16 : 0;

8
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/chunk/MCAChunk.java
View File

@ -2,7 +2,7 @@ package de.bluecolored.bluemap.core.world.mca.chunk;
import de.bluecolored.bluemap.core.world.BlockState;
import de.bluecolored.bluemap.core.world.Chunk;
import de.bluecolored.bluemap.core.world.mca.region.MCARegion;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import lombok.Getter;
import lombok.ToString;
@ -20,11 +20,11 @@ public abstract class MCAChunk implements Chunk {
protected static final String[] EMPTY_STRING_ARRAY = new String[0];
protected static final BlockState[] EMPTY_BLOCKSTATE_ARRAY = new BlockState[0];
private final MCARegion region;
private final MCAWorld world;
private final int dataVersion;
public MCAChunk(MCARegion region, Data chunkData) {
this.region = region;
public MCAChunk(MCAWorld world, Data chunkData) {
this.world = world;
this.dataVersion = chunkData.getDataVersion();
}

211
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/LinearRegion.java Normal file
View File

@ -0,0 +1,211 @@
/*
* This file is part of BlueMap, licensed under the MIT License (MIT).
*
* Copyright (c) Blue (Lukas Rieger) <https://bluecolored.de>
* Copyright (c) contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package de.bluecolored.bluemap.core.world.mca.region;
import com.flowpowered.math.vector.Vector2i;
import de.bluecolored.bluemap.core.storage.Compression;
import de.bluecolored.bluemap.core.world.ChunkConsumer;
import de.bluecolored.bluemap.core.world.Region;
import de.bluecolored.bluemap.core.world.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.mca.chunk.MCAChunk;
import io.airlift.compress.zstd.ZstdInputStream;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
/*
* LinearFormat:
*
* REGION-FILE:
* 8 byte - MAGIC value
* 1 byte - version
* 8 byte - region timestamp
* 1 byte - compression level
* 2 byte - chunk count
* 4 byte - data-length in bytes
* 8 byte - data-hash
* ? byte - data
* 8 byte - MAGIC value
*
* DATA: (zstd compressed)
* 32 * 32 * 8 - header:
* 4 byte - chunk-data-length
* 4 byte - timestamp
* ? - chunks
*
*/
public class LinearRegion implements Region {
public static final String FILE_SUFFIX = ".linear";
private static final long MAGIC = 0xc3ff13183cca9d9aL;
private final MCAWorld world;
private final Path regionFile;
private final Vector2i regionPos;
public LinearRegion(MCAWorld world, Path regionFile) throws IllegalArgumentException {
this.world = world;
this.regionFile = regionFile;
String[] filenameParts = regionFile.getFileName().toString().split("\\.");
int rX = Integer.parseInt(filenameParts[1]);
int rZ = Integer.parseInt(filenameParts[2]);
this.regionPos = new Vector2i(rX, rZ);
}
public LinearRegion(MCAWorld world, Vector2i regionPos) throws IllegalArgumentException {
this.world = world;
this.regionPos = regionPos;
this.regionFile = world.getRegionFolder().resolve(getRegionFileName(regionPos.getX(), regionPos.getY()));
}
@Override
public void iterateAllChunks(ChunkConsumer consumer) throws IOException {
if (Files.notExists(regionFile)) return;
long fileLength = Files.size(regionFile);
if (fileLength == 0) return;
int chunkStartX = regionPos.getX() * 32;
int chunkStartZ = regionPos.getY() * 32;
byte[] chunkDataBuffer = null;
byte[] compressedData;
byte version;
long newestTimestamp;
byte compressionLevel;
short chunkCount;
int dataLength;
long dataHash;
try (
InputStream in = Files.newInputStream(regionFile, StandardOpenOption.READ);
BufferedInputStream bIn = new BufferedInputStream(in);
DataInputStream dIn = new DataInputStream(bIn)
) {
if (dIn.readLong() != MAGIC)
throw new IOException("Linear region-file format: invalid header magic");
// read the header
version = dIn.readByte();
newestTimestamp = dIn.readLong();
compressionLevel = dIn.readByte();
chunkCount = dIn.readShort();
dataLength = dIn.readInt();
dataHash = dIn.readLong();
if (version < 1 || version > 2)
throw new IOException("Linear region-file format: Unsupported version: " + version);
if (fileLength != dataLength + 40) // 40 = header + footer
throw new IOException("Linear region-file format: Invalid file length. Expected " + (dataLength + 40) + " but got " + fileLength);
compressedData = new byte[dataLength];
dIn.readFully(compressedData, 0, dataLength);
if (dIn.readLong() != MAGIC)
throw new IOException("Linear region-file format: invalid footer magic");
}
try (
InputStream in = new ZstdInputStream(new ByteArrayInputStream(compressedData));
BufferedInputStream bIn = new BufferedInputStream(in);
DataInputStream dIn = new DataInputStream(bIn)
) {
int[] chunkDataLengths = new int[1024];
int[] chunkTimestamps = new int[1024];
for (int i = 0 ; i < 1024 ; i++) {
chunkDataLengths[i] = dIn.readInt();
chunkTimestamps[i] = dIn.readInt();
}
int i = 0;
int toBeSkipped = 0;
for (int z = 0; z < 32; z++) {
for (int x = 0; x < 32; x++) {
int length = chunkDataLengths[i];
if (length > 0) {
int chunkX = chunkStartX + x;
int chunkZ = chunkStartZ + z;
long timestamp = version == 2 ? chunkTimestamps[i] : newestTimestamp;
if (consumer.filter(chunkX, chunkZ, timestamp)) {
if (toBeSkipped > 0) skipNBytes(dIn, toBeSkipped);
if (chunkDataBuffer == null || chunkDataBuffer.length < length)
chunkDataBuffer = new byte[length];
dIn.readFully(chunkDataBuffer, 0, length);
MCAChunk chunk = world.getChunkLoader().load(chunkDataBuffer, 0, length, Compression.NONE);
consumer.accept(chunkX, chunkZ, chunk);
} else {
// skip before reading the next chunk, but only if there is a next chunk
// that we actually want to read, to avoid decompressing unnecessary data
toBeSkipped += length;
}
}
i++;
}
}
}
}
public static String getRegionFileName(int regionX, int regionZ) {
return "r." + regionX + "." + regionZ + FILE_SUFFIX;
}
/**
* This method is taken here from a newer version of {@link InputStream},
* to ensure Java 11 compatibility.
*/
private static void skipNBytes(InputStream in, long n) throws IOException {
while (n > 0) {
long ns = in.skip(n);
if (ns > 0 && ns <= n) {
// adjust number to skip
n -= ns;
} else if (ns == 0) { // no bytes skipped
// read one byte to check for EOS
if (in.read() == -1) {
throw new EOFException();
}
// one byte read so decrement number to skip
n--;
} else { // skipped negative or too many bytes
throw new IOException("Unable to skip exactly");
}
}
}
}

2
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/MCARegion.java
View File

@ -168,7 +168,7 @@ public class MCARegion implements Region {
default: throw new IOException("Unknown chunk compression-id: " + compressionTypeId);
}
return world.getChunkLoader().load(this, data, 5, size - 5, compression);
return world.getChunkLoader().load(data, 5, size - 5, compression);
}
public static String getRegionFileName(int regionX, int regionZ) {

4
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/mca/region/RegionType.java
View File

@ -34,8 +34,8 @@ import java.nio.file.Path;
public enum RegionType {
MCA (MCARegion::new, MCARegion.FILE_SUFFIX, MCARegion::getRegionFileName);
//LINEAR (LinearRegion::new, LinearRegion.FILE_SUFFIX, LinearRegion::getRegionFileName);
MCA (MCARegion::new, MCARegion.FILE_SUFFIX, MCARegion::getRegionFileName),
LINEAR (LinearRegion::new, LinearRegion.FILE_SUFFIX, LinearRegion::getRegionFileName);
// we do this to improve performance, as calling values() creates a new array each time
private final static RegionType[] VALUES = values();