Upstream
/
BlueMap
mirror of https://github.com/BlueMap-Minecraft/BlueMap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: Linear region file format support (extended from #415) (#418) 

* Linear support

Fix region rendering & bitwise operators

Close streams

* Make mca region-file types extensible

* Fix file-name verification not working

---------

Co-authored-by: Sofiane H. Djerbi <46628754+kugge@users.noreply.github.com>
This commit is contained in:
Lukas Rieger 2023-04-30 11:09:36 +02:00 committed by GitHub
parent bc9e688317
commit f2355fa99b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 297 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
BlueMapCore/build.gradle.kts
View File

@ -56,6 +56,7 @@ repositories {
@Suppress("GradlePackageUpdate")
dependencies {
implementation("com.github.luben:zstd-jni:1.5.4-1")
api ("com.github.ben-manes.caffeine:caffeine:2.8.5")
api ("org.apache.commons:commons-lang3:3.6")
api ("commons-io:commons-io:2.5")

10
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/MCAWorld.java
View File

@ -31,6 +31,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import de.bluecolored.bluemap.api.debug.DebugDump;
import de.bluecolored.bluemap.core.BlueMap;
import de.bluecolored.bluemap.core.logger.Logger;
import de.bluecolored.bluemap.core.mca.region.RegionType;
import de.bluecolored.bluemap.core.util.Vector2iCache;
import de.bluecolored.bluemap.core.world.*;
import net.querz.nbt.CompoundTag;
@ -131,7 +132,7 @@ public class MCAWorld implements World {
List<Vector2i> regions = new ArrayList<>(regionFiles.length);
for (File file : regionFiles) {
if (!file.getName().endsWith(".mca")) continue;
if (RegionType.forFileName(file.getName()) == null) continue;
if (file.length() <= 0) continue;
try {
@ -213,17 +214,12 @@ public class MCAWorld implements World {
return ignoreMissingLightData;
}
private File getMCAFile(int regionX, int regionZ) {
return getRegionFolder().resolve("r." + regionX + "." + regionZ + ".mca").toFile();
}
private Region loadRegion(Vector2i regionPos) {
return loadRegion(regionPos.getX(), regionPos.getY());
}
Region loadRegion(int x, int z) {
File regionPath = getMCAFile(x, z);
return new MCARegion(this, regionPath);
return RegionType.loadRegion(this, getRegionFolder(), x, z);
}
private Chunk loadChunk(Vector2i chunkPos) {

182
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/region/LinearRegion.java Normal file
View File

@ -0,0 +1,182 @@
/*
* This file is part of BlueMap, licensed under the MIT License (MIT).
*
* Copyright (c) Blue (Lukas Rieger) <https://bluecolored.de>
* Copyright (c) contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package de.bluecolored.bluemap.core.mca.region;
import com.flowpowered.math.vector.Vector2i;
import com.github.luben.zstd.ZstdInputStream;
import de.bluecolored.bluemap.core.logger.Logger;
import de.bluecolored.bluemap.core.mca.MCAChunk;
import de.bluecolored.bluemap.core.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.Chunk;
import de.bluecolored.bluemap.core.world.EmptyChunk;
import de.bluecolored.bluemap.core.world.Region;
import net.querz.nbt.CompoundTag;
import net.querz.nbt.Tag;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public class LinearRegion implements Region {
public static final String FILE_SUFFIX = ".linear";
private static final long SUPERBLOCK = -4323716122432332390L;
private static final byte VERSION = 1;
private static final int HEADER_SIZE = 32;
private static final int FOOTER_SIZE = 8;
private final MCAWorld world;
private final Path regionFile;
private final Vector2i regionPos;
public LinearRegion(MCAWorld world, Path regionFile) throws IllegalArgumentException {
this.world = world;
this.regionFile = regionFile;
String[] filenameParts = regionFile.getFileName().toString().split("\\.");
int rX = Integer.parseInt(filenameParts[1]);
int rZ = Integer.parseInt(filenameParts[2]);
this.regionPos = new Vector2i(rX, rZ);
}
@Override
public Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) throws IOException {
if (Files.notExists(regionFile)) return EmptyChunk.INSTANCE;
long fileLength = Files.size(regionFile);
if (fileLength == 0) return EmptyChunk.INSTANCE;
try (InputStream inputStream = Files.newInputStream(regionFile);
DataInputStream rawDataStream = new DataInputStream(inputStream)) {
long superBlock = rawDataStream.readLong();
if (superBlock != SUPERBLOCK)
throw new RuntimeException("Superblock invalid: " + superBlock + " file " + regionFile);
byte version = rawDataStream.readByte();
if (version != VERSION)
throw new RuntimeException("Version invalid: " + version + " file " + regionFile);
rawDataStream.skipBytes(11); // newestTimestamp + compression level + chunk count
int dataCount = rawDataStream.readInt();
if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE)
throw new RuntimeException("File length invalid " + this.regionFile + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE));
rawDataStream.skipBytes(8); // Data Hash
byte[] rawCompressed = new byte[dataCount];
rawDataStream.readFully(rawCompressed, 0, dataCount);
superBlock = rawDataStream.readLong();
if (superBlock != SUPERBLOCK)
throw new RuntimeException("Footer superblock invalid " + this.regionFile);
try (DataInputStream dis = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) {
int x = chunkX - (regionPos.getX() << 5);
int z = chunkZ - (regionPos.getY() << 5);
int pos = (z << 5) + x;
int skip = 0;
for (int i = 0; i < pos; i++) {
skip += dis.readInt(); // size of the chunk (bytes) to skip
dis.skipBytes(4); // skip 0 (will be timestamps)
}
int size = dis.readInt();
if (size <= 0) return EmptyChunk.INSTANCE;
dis.skipBytes(((1024 - pos - 1) << 3) + 4); // Skip current chunk 0 and unneeded other chunks zero/size
dis.skipBytes(skip); // Skip unneeded chunks data
Tag<?> tag = Tag.deserialize(dis, Tag.DEFAULT_MAX_DEPTH);
if (tag instanceof CompoundTag) {
MCAChunk chunk = MCAChunk.create(world, (CompoundTag) tag);
if (!chunk.isGenerated()) return EmptyChunk.INSTANCE;
return chunk;
} else {
throw new IOException("Invalid data tag: " + (tag == null ? "null" : tag.getClass().getName()));
}
}
} catch (RuntimeException e) {
throw new IOException(e);
}
}
@Override
public Collection<Vector2i> listChunks(long modifiedSince) {
if (Files.notExists(regionFile)) return Collections.emptyList();
try {
long fileLength = Files.size(regionFile);
if (fileLength == 0) return Collections.emptyList();
} catch (IOException ex) {
Logger.global.logWarning("Failed to read file-size for file: " + regionFile);
return Collections.emptyList();
}
List<Vector2i> chunks = new ArrayList<>(1024); //1024 = 32 x 32 chunks per region-file
try (InputStream inputStream = Files.newInputStream(regionFile);
DataInputStream rawDataStream = new DataInputStream(inputStream)) {
long superBlock = rawDataStream.readLong();
if (superBlock != SUPERBLOCK) throw new RuntimeException("Superblock invalid: " + superBlock + " file " + regionFile);
byte version = rawDataStream.readByte();
if (version != VERSION) throw new RuntimeException("Version invalid: " + version + " file " + regionFile);
long newestTimestamp = rawDataStream.readLong();
// If whole region is the same - skip.
if (newestTimestamp < modifiedSince / 1000) return Collections.emptyList();
// Linear files store whole region timestamp, not chunk timestamp. We need to render the while region file.
// TODO: Add per-chunk timestamps when .linear add support for per-chunk timestamps (soon)
for(int i = 0 ; i < 1024; i++)
chunks.add(new Vector2i((regionPos.getX() << 5) + (i & 31), (regionPos.getY() << 5) + (i >> 5)));
return chunks;
} catch (RuntimeException | IOException ex) {
Logger.global.logWarning("Failed to read .linear file: " + regionFile + " (" + ex + ")");
}
return chunks;
}
@Override
public Path getRegionFile() {
return regionFile;
}
public static String getRegionFileName(int regionX, int regionZ) {
return "r." + regionX + "." + regionZ + FILE_SUFFIX;
}
}

41
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/MCARegion.java → BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/region/MCARegion.java
View File

@ -22,10 +22,12 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package de.bluecolored.bluemap.core.mca;
package de.bluecolored.bluemap.core.mca.region;
import com.flowpowered.math.vector.Vector2i;
import de.bluecolored.bluemap.core.logger.Logger;
import de.bluecolored.bluemap.core.mca.MCAChunk;
import de.bluecolored.bluemap.core.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.Chunk;
import de.bluecolored.bluemap.core.world.EmptyChunk;
import de.bluecolored.bluemap.core.world.Region;
@ -34,6 +36,8 @@ import net.querz.nbt.Tag;
import net.querz.nbt.mca.CompressionType;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -41,15 +45,17 @@ import java.util.List;
public class MCARegion implements Region {
public static final String FILE_SUFFIX = ".mca";
private final MCAWorld world;
private final File regionFile;
private final Path regionFile;
private final Vector2i regionPos;
public MCARegion(MCAWorld world, File regionFile) throws IllegalArgumentException {
public MCARegion(MCAWorld world, Path regionFile) throws IllegalArgumentException {
this.world = world;
this.regionFile = regionFile;
String[] filenameParts = regionFile.getName().split("\\.");
String[] filenameParts = regionFile.getFileName().toString().split("\\.");
int rX = Integer.parseInt(filenameParts[1]);
int rZ = Integer.parseInt(filenameParts[2]);
@ -58,9 +64,12 @@ public class MCARegion implements Region {
@Override
public Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) throws IOException {
if (!regionFile.exists() || regionFile.length() == 0) return EmptyChunk.INSTANCE;
if (Files.notExists(regionFile)) return EmptyChunk.INSTANCE;
try (RandomAccessFile raf = new RandomAccessFile(regionFile, "r")) {
long fileLength = Files.size(regionFile);
if (fileLength == 0) return EmptyChunk.INSTANCE;
try (RandomAccessFile raf = new RandomAccessFile(regionFile.toFile(), "r")) {
int xzChunk = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32);
@ -100,11 +109,19 @@ public class MCARegion implements Region {
@Override
public Collection<Vector2i> listChunks(long modifiedSince) {
if (!regionFile.exists() || regionFile.length() == 0) return Collections.emptyList();
if (Files.notExists(regionFile)) return Collections.emptyList();
try {
long fileLength = Files.size(regionFile);
if (fileLength == 0) return Collections.emptyList();
} catch (IOException ex) {
Logger.global.logWarning("Failed to read file-size for file: " + regionFile);
return Collections.emptyList();
}
List<Vector2i> chunks = new ArrayList<>(1024); //1024 = 32 x 32 chunks per region-file
try (RandomAccessFile raf = new RandomAccessFile(regionFile, "r")) {
try (RandomAccessFile raf = new RandomAccessFile(regionFile.toFile(), "r")) {
for (int x = 0; x < 32; x++) {
for (int z = 0; z < 32; z++) {
Vector2i chunk = new Vector2i(regionPos.getX() * 32 + x, regionPos.getY() * 32 + z);
@ -127,15 +144,19 @@ public class MCARegion implements Region {
}
}
} catch (RuntimeException | IOException ex) {
Logger.global.logWarning("Failed to read .mca file: " + regionFile.getAbsolutePath() + " (" + ex + ")");
Logger.global.logWarning("Failed to read .mca file: " + regionFile + " (" + ex + ")");
}
return chunks;
}
@Override
public File getRegionFile() {
public Path getRegionFile() {
return regionFile;
}
public static String getRegionFileName(int regionX, int regionZ) {
return "r." + regionX + "." + regionZ + FILE_SUFFIX;
}
}

78
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/region/RegionType.java Normal file
View File

@ -0,0 +1,78 @@
package de.bluecolored.bluemap.core.mca.region;
import de.bluecolored.bluemap.core.mca.MCAWorld;
import de.bluecolored.bluemap.core.world.Region;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.nio.file.Files;
import java.nio.file.Path;
public enum RegionType {
MCA (MCARegion::new, MCARegion.FILE_SUFFIX, MCARegion::getRegionFileName),
LINEAR (LinearRegion::new, LinearRegion.FILE_SUFFIX, LinearRegion::getRegionFileName);
// we do this to improve performance, as calling values() creates a new array each time
private final static RegionType[] VALUES = values();
private final static RegionType DEFAULT = MCA;
private final String fileSuffix;
private final RegionFactory regionFactory;
private final RegionFileNameFunction regionFileNameFunction;
RegionType(RegionFactory regionFactory, String fileSuffix, RegionFileNameFunction regionFileNameFunction) {
this.fileSuffix = fileSuffix;
this.regionFactory = regionFactory;
this.regionFileNameFunction = regionFileNameFunction;
}
public String getFileSuffix() {
return fileSuffix;
}
public Region createRegion(MCAWorld world, Path regionFile) {
return this.regionFactory.create(world, regionFile);
}
public String getRegionFileName(int regionX, int regionZ) {
return regionFileNameFunction.getRegionFileName(regionX, regionZ);
}
public Path getRegionFile(Path regionFolder, int regionX, int regionZ) {
return regionFolder.resolve(getRegionFileName(regionX, regionZ));
}
@Nullable
public static RegionType forFileName(String fileName) {
//noinspection ForLoopReplaceableByForEach
for (int i = 0; i < VALUES.length; i++) {
RegionType regionType = VALUES[i];
if (fileName.endsWith(regionType.fileSuffix))
return regionType;
}
return null;
}
@NotNull
public static Region loadRegion(MCAWorld world, Path regionFolder, int regionX, int regionZ) {
//noinspection ForLoopReplaceableByForEach
for (int i = 0; i < VALUES.length; i++) {
RegionType regionType = VALUES[i];
Path regionFile = regionType.getRegionFile(regionFolder, regionX, regionZ);
if (Files.exists(regionFile)) return regionType.createRegion(world, regionFile);
}
return DEFAULT.createRegion(world, DEFAULT.getRegionFile(regionFolder, regionX, regionZ));
}
@FunctionalInterface
interface RegionFactory {
Region create(MCAWorld world, Path regionFile);
}
@FunctionalInterface
interface RegionFileNameFunction {
String getRegionFileName(int regionX, int regionZ);
}
}

4
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/world/Region.java
View File

@ -26,8 +26,8 @@ package de.bluecolored.bluemap.core.world;
import com.flowpowered.math.vector.Vector2i;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
public interface Region {
@ -52,6 +52,6 @@ public interface Region {
Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) throws IOException;
File getRegionFile();
Path getRegionFile();
}