mirror of
https://github.com/PaperMC/Paper.git
synced 2024-09-12 15:08:27 +02:00
417 lines
18 KiB
Diff
417 lines
18 KiB
Diff
|
From ba4afed44f76c2d0b8ba63439ecc28b6c2212d05 Mon Sep 17 00:00:00 2001
|
||
|
From: Aikar <aikar@aikar.co>
|
||
|
Date: Fri, 15 Feb 2019 01:08:19 -0500
|
||
|
Subject: [PATCH] Allow Saving of Oversized Chunks
|
||
|
|
||
|
The Minecraft World Region File format has a hard cap of 1MB per chunk.
|
||
|
This is due to the fact that the header of the file format only allocates
|
||
|
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.
|
||
|
|
||
|
This limit can be reached fairly easily with books, resulting in the chunk being unable
|
||
|
to save to the world. Worse off, is that nothing printed when this occured, and silently
|
||
|
performed a chunk rollback on next load.
|
||
|
|
||
|
This leads to security risk with duplication and is being actively exploited.
|
||
|
|
||
|
This patch catches the too large scenario, falls back and moves any large Entity
|
||
|
or Tile Entity into a new compound, and this compound is saved into a different file.
|
||
|
|
||
|
On Chunk Load, we check for oversized status, and if so, we load the extra file and
|
||
|
merge the Entities and Tile Entities from the oversized chunk back into the level to
|
||
|
then be loaded as normal.
|
||
|
|
||
|
Once a chunk is returned back to normal size, the oversized flag will clear, and no
|
||
|
extra data file will exist.
|
||
|
|
||
|
This fix maintains compatability with all existing Anvil Region Format tools as it
|
||
|
does not alter the save format. They will just not know about the extra entities.
|
||
|
|
||
|
This fix also maintains compatability if someone switches server jars to one without
|
||
|
this fix, as the data will remain in the oversized file. Once the server returns
|
||
|
to a jar with this fix, the data will be restored.
|
||
|
|
||
|
diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
||
|
index 12268f87b9..e1f7e06ab2 100644
|
||
|
--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
||
|
+++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
||
|
@@ -39,6 +39,7 @@ public class NBTCompressedStreamTools {
|
||
|
|
||
|
}
|
||
|
|
||
|
+ public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER
|
||
|
public static NBTTagCompound a(DataInputStream datainputstream) throws IOException {
|
||
|
return a((DataInput) datainputstream, NBTReadLimiter.a);
|
||
|
}
|
||
|
@@ -59,6 +60,7 @@ public class NBTCompressedStreamTools {
|
||
|
}
|
||
|
}
|
||
|
|
||
|
+ public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER
|
||
|
public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException {
|
||
|
a((NBTBase) nbttagcompound, dataoutput);
|
||
|
}
|
||
|
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
|
||
|
index 999adf7dc7..303c13ccd0 100644
|
||
|
--- a/src/main/java/net/minecraft/server/RegionFile.java
|
||
|
+++ b/src/main/java/net/minecraft/server/RegionFile.java
|
||
|
@@ -75,6 +75,7 @@ public class RegionFile {
|
||
|
}
|
||
|
header.clear();
|
||
|
IntBuffer headerAsInts = header.asIntBuffer();
|
||
|
+ initOversizedState();
|
||
|
// Paper End
|
||
|
|
||
|
for(int j1 = 0; j1 < 1024; ++j1) {
|
||
|
@@ -99,7 +100,7 @@ public class RegionFile {
|
||
|
}
|
||
|
|
||
|
@Nullable
|
||
|
- public synchronized DataInputStream a(int i, int j) {
|
||
|
+ public synchronized DataInputStream getReadStream(int i, int j) { return a(i, j); } @Nullable public synchronized DataInputStream a(int i, int j) { // Paper - OBFHELPER
|
||
|
if (this.e(i, j)) {
|
||
|
return null;
|
||
|
} else {
|
||
|
@@ -175,8 +176,8 @@ public class RegionFile {
|
||
|
}
|
||
|
|
||
|
@Nullable
|
||
|
- public DataOutputStream c(int i, int j) {
|
||
|
- return this.e(i, j) ? null : new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new RegionFile.ChunkBuffer(i, j))));
|
||
|
+ public DataOutputStream getWriteStream(int i, int j) { return c(i, j); } @Nullable public DataOutputStream c(int i, int j) { // Paper - OBFHELPER
|
||
|
+ return this.e(i, j) ? null : new DataOutputStream(new RegionFile.ChunkBuffer(i, j)); // Paper - remove middleware, move deflate to .close() for dynamic levels
|
||
|
}
|
||
|
|
||
|
protected synchronized void a(int i, int j, byte[] abyte, int k) {
|
||
|
@@ -187,7 +188,7 @@ public class RegionFile {
|
||
|
int k1 = (k + 5) / 4096 + 1;
|
||
|
|
||
|
if (k1 >= 256) {
|
||
|
- return;
|
||
|
+ throw new ChunkTooLargeException(i, j, k1); // Paper - throw error instead
|
||
|
}
|
||
|
|
||
|
if (i1 != 0 && j1 == k1) {
|
||
|
@@ -338,6 +339,101 @@ public class RegionFile {
|
||
|
logger.error("Error backing up corrupt file" + file.getAbsolutePath(), e);
|
||
|
}
|
||
|
}
|
||
|
+
|
||
|
+ private final byte[] oversized = new byte[1024];
|
||
|
+ private int oversizedCount = 0;
|
||
|
+
|
||
|
+ private synchronized void initOversizedState() throws IOException {
|
||
|
+ File metaFile = getOversizedMetaFile();
|
||
|
+ if (metaFile.exists()) {
|
||
|
+ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
|
||
|
+ System.arraycopy(read, 0, oversized, 0, oversized.length);
|
||
|
+ for (byte temp : oversized) {
|
||
|
+ oversizedCount += temp;
|
||
|
+ }
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
+ private static int getChunkIndex(int x, int z) {
|
||
|
+ return (x & 31) + (z & 31) * 32;
|
||
|
+ }
|
||
|
+ synchronized boolean isOversized(int x, int z) {
|
||
|
+ return this.oversized[getChunkIndex(x, z)] == 1;
|
||
|
+ }
|
||
|
+ synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
|
||
|
+ final int offset = getChunkIndex(x, z);
|
||
|
+ boolean previous = this.oversized[offset] == 1;
|
||
|
+ this.oversized[offset] = (byte) (oversized ? 1 : 0);
|
||
|
+ if (!previous && oversized) {
|
||
|
+ oversizedCount++;
|
||
|
+ } else if (!oversized && previous) {
|
||
|
+ oversizedCount--;
|
||
|
+ }
|
||
|
+ if (previous && !oversized) {
|
||
|
+ File oversizedFile = getOversizedFile(x, z);
|
||
|
+ if (oversizedFile.exists()) {
|
||
|
+ oversizedFile.delete();
|
||
|
+ }
|
||
|
+ }
|
||
|
+ if (oversizedCount > 0) {
|
||
|
+ if (previous != oversized) {
|
||
|
+ writeOversizedMeta();
|
||
|
+ }
|
||
|
+ } else if (previous) {
|
||
|
+ File oversizedMetaFile = getOversizedMetaFile();
|
||
|
+ if (oversizedMetaFile.exists()) {
|
||
|
+ oversizedMetaFile.delete();
|
||
|
+ }
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
+ private void writeOversizedMeta() throws IOException {
|
||
|
+ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
|
||
|
+ }
|
||
|
+
|
||
|
+ private File getOversizedMetaFile() {
|
||
|
+ return new File(getFile().getParentFile(), getFile().getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
|
||
|
+ }
|
||
|
+
|
||
|
+ private File getOversizedFile(int x, int z) {
|
||
|
+ return new File(this.getFile().getParentFile(), this.getFile().getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
|
||
|
+ }
|
||
|
+
|
||
|
+ void writeOversizedData(int x, int z, NBTTagCompound oversizedData) throws IOException {
|
||
|
+ File file = getOversizedFile(x, z);
|
||
|
+ try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new java.io.FileOutputStream(file), new java.util.zip.Deflater(java.util.zip.Deflater.BEST_COMPRESSION), 32 * 1024), 32 * 1024))) {
|
||
|
+ NBTCompressedStreamTools.writeNBT(oversizedData, out);
|
||
|
+ }
|
||
|
+ this.setOversized(x, z, true);
|
||
|
+
|
||
|
+ }
|
||
|
+
|
||
|
+ synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException {
|
||
|
+ File file = getOversizedFile(x, z);
|
||
|
+ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
|
||
|
+ return NBTCompressedStreamTools.readNBT(out);
|
||
|
+ }
|
||
|
+
|
||
|
+ }
|
||
|
+
|
||
|
+ public class ChunkTooLargeException extends RuntimeException {
|
||
|
+ public ChunkTooLargeException(int x, int z, int sectors) {
|
||
|
+ super("Chunk " + x + "," + z + " of " + getFile().toString() + " is too large (" + sectors + "/256)");
|
||
|
+ }
|
||
|
+ }
|
||
|
+ private static class DirectByteArrayOutputStream extends ByteArrayOutputStream {
|
||
|
+ public DirectByteArrayOutputStream() {
|
||
|
+ super();
|
||
|
+ }
|
||
|
+
|
||
|
+ public DirectByteArrayOutputStream(int size) {
|
||
|
+ super(size);
|
||
|
+ }
|
||
|
+
|
||
|
+ public byte[] getBuffer() {
|
||
|
+ return this.buf;
|
||
|
+ }
|
||
|
+ }
|
||
|
// Paper end
|
||
|
|
||
|
class ChunkBuffer extends ByteArrayOutputStream {
|
||
|
@@ -351,8 +447,40 @@ public class RegionFile {
|
||
|
this.c = j;
|
||
|
}
|
||
|
|
||
|
- public void close() {
|
||
|
- RegionFile.this.a(this.b, this.c, this.buf, this.count);
|
||
|
+ public void close() throws IOException {
|
||
|
+ // Paper start - apply dynamic compression
|
||
|
+ int origLength = this.count;
|
||
|
+ byte[] buf = this.buf;
|
||
|
+ DirectByteArrayOutputStream out = compressData(buf, origLength);
|
||
|
+ byte[] bytes = out.getBuffer();
|
||
|
+ int length = out.size();
|
||
|
+
|
||
|
+ RegionFile.this.a(this.b, this.c, bytes, length); // Paper - change to bytes/length
|
||
|
+ // Paper end
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
+ private static DirectByteArrayOutputStream compressData(byte[] buf, int length) throws IOException {
|
||
|
+ final java.util.zip.Deflater deflater;
|
||
|
+ if (length > 1024 * 512) {
|
||
|
+ deflater = new java.util.zip.Deflater(9);
|
||
|
+ } else if (length > 1024 * 128) {
|
||
|
+ deflater = new java.util.zip.Deflater(8);
|
||
|
+ } else {
|
||
|
+ deflater = new java.util.zip.Deflater(6);
|
||
|
+ }
|
||
|
+
|
||
|
+
|
||
|
+ deflater.setInput(buf, 0, length);
|
||
|
+ deflater.finish();
|
||
|
+
|
||
|
+ DirectByteArrayOutputStream out = new DirectByteArrayOutputStream(length);
|
||
|
+ byte[] buffer = new byte[1024 * (length > 1024 * 124 ? 32 : 16)];
|
||
|
+ while (!deflater.finished()) {
|
||
|
+ out.write(buffer, 0, deflater.deflate(buffer));
|
||
|
}
|
||
|
+ out.close();
|
||
|
+ deflater.end();
|
||
|
+ return out;
|
||
|
}
|
||
|
}
|
||
|
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
|
||
|
index 8c8b7cbab5..50a62d6e23 100644
|
||
|
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
|
||
|
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
|
||
|
@@ -16,6 +16,7 @@ public class RegionFileCache {
|
||
|
|
||
|
public static final Map<File, RegionFile> cache = new LinkedHashMap(PaperConfig.regionFileCacheSize, 0.75f, true); // Paper - HashMap -> LinkedHashMap
|
||
|
|
||
|
+ public static synchronized RegionFile getRegionFile(File file, int i, int j) { return a(file, i, j); } // Paper - OBFHELPER
|
||
|
public static synchronized RegionFile a(File file, int i, int j) {
|
||
|
File file1 = new File(file, "region");
|
||
|
File file2 = new File(file1, "r." + (i >> 5) + "." + (j >> 5) + ".mca");
|
||
|
@@ -83,6 +84,129 @@ public class RegionFileCache {
|
||
|
public static synchronized boolean hasRegionFile(File file, int i, int j) {
|
||
|
return RegionFileCache.cache.containsKey(getRegionFileName(file, i, j));
|
||
|
}
|
||
|
+ private static void printOversizedLog(String msg, File file, int x, int z) {
|
||
|
+ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
|
||
|
+ }
|
||
|
+
|
||
|
+ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
|
||
|
+ private static final int OVERZEALOUS_THRESHOLD = 1024 * 2;
|
||
|
+ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
|
||
|
+ private static void resetFilterThresholds() {
|
||
|
+ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
|
||
|
+ }
|
||
|
+ static {
|
||
|
+ resetFilterThresholds();
|
||
|
+ }
|
||
|
+ private static void writeRegion(File file, int x, int z, NBTTagCompound nbttagcompound) throws IOException {
|
||
|
+ RegionFile regionfile = getRegionFile(file, x, z);
|
||
|
+
|
||
|
+ DataOutputStream out = regionfile.getWriteStream(x & 31, z & 31);
|
||
|
+ try {
|
||
|
+ NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
|
||
|
+ out.close();
|
||
|
+ regionfile.setOversized(x, z, false);
|
||
|
+ } catch (RegionFile.ChunkTooLargeException ignored) {
|
||
|
+ printOversizedLog("ChunkTooLarge! Someone is trying to duplicate.", file, x, z);
|
||
|
+ // Clone as we are now modifying it, don't want to corrupt the pending save state
|
||
|
+ nbttagcompound = nbttagcompound.clone();
|
||
|
+ // Filter out TileEntities and Entities
|
||
|
+ NBTTagCompound oversizedData = filterChunkData(nbttagcompound);
|
||
|
+ //noinspection SynchronizationOnLocalVariableOrMethodParameter
|
||
|
+ synchronized (regionfile) {
|
||
|
+ out = regionfile.getWriteStream(x & 31, z & 31);
|
||
|
+ NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
|
||
|
+ try {
|
||
|
+ out.close();
|
||
|
+ // 2048 is below the min allowed, so it means we enter overzealous mode below
|
||
|
+ if (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD) {
|
||
|
+ resetFilterThresholds();
|
||
|
+ }
|
||
|
+ } catch (RegionFile.ChunkTooLargeException e) {
|
||
|
+ printOversizedLog("ChunkTooLarge even after reduction. Trying in overzealous mode.", file, x, z);
|
||
|
+ // Eek, major fail. We have retry logic, so reduce threshholds and fall back
|
||
|
+ SIZE_THRESHOLD = OVERZEALOUS_THRESHOLD;
|
||
|
+ throw e;
|
||
|
+ }
|
||
|
+
|
||
|
+ regionfile.writeOversizedData(x, z, oversizedData);
|
||
|
+ }
|
||
|
+ } catch (Exception e) {
|
||
|
+ e.printStackTrace();
|
||
|
+ throw e;
|
||
|
+ }
|
||
|
+
|
||
|
+ }
|
||
|
+
|
||
|
+ private static NBTTagCompound filterChunkData(NBTTagCompound chunk) {
|
||
|
+ NBTTagCompound oversizedLevel = new NBTTagCompound();
|
||
|
+ NBTTagCompound level = chunk.getCompound("Level");
|
||
|
+ filterChunkList(level, oversizedLevel, "Entities");
|
||
|
+ filterChunkList(level, oversizedLevel, "TileEntities");
|
||
|
+ NBTTagCompound oversized = new NBTTagCompound();
|
||
|
+ oversized.set("Level", oversizedLevel);
|
||
|
+ return oversized;
|
||
|
+ }
|
||
|
+
|
||
|
+ private static void filterChunkList(NBTTagCompound level, NBTTagCompound extra, String key) {
|
||
|
+ NBTTagList list = level.getList(key, 10);
|
||
|
+ NBTTagList newList = extra.getList(key, 10);
|
||
|
+ for (Iterator<NBTBase> iterator = list.list.iterator(); iterator.hasNext(); ) {
|
||
|
+ NBTBase object = iterator.next();
|
||
|
+ if (getNBTSize(object) > SIZE_THRESHOLD) {
|
||
|
+ newList.add(object);
|
||
|
+ iterator.remove();
|
||
|
+ }
|
||
|
+ }
|
||
|
+ level.set(key, list);
|
||
|
+ extra.set(key, newList);
|
||
|
+ }
|
||
|
+
|
||
|
+
|
||
|
+ private static NBTTagCompound readOversizedChunk(RegionFile regionfile, int i, int j) throws IOException {
|
||
|
+ synchronized (regionfile) {
|
||
|
+ try (DataInputStream datainputstream = regionfile.getReadStream(i & 31, j & 31)) {
|
||
|
+ NBTTagCompound oversizedData = regionfile.getOversizedData(i, j);
|
||
|
+ NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
|
||
|
+ if (oversizedData == null) {
|
||
|
+ return chunk;
|
||
|
+ }
|
||
|
+ NBTTagCompound oversizedLevel = oversizedData.getCompound("Level");
|
||
|
+ NBTTagCompound level = chunk.getCompound("Level");
|
||
|
+
|
||
|
+ mergeChunkList(level, oversizedLevel, "Entities");
|
||
|
+ mergeChunkList(level, oversizedLevel, "TileEntities");
|
||
|
+
|
||
|
+ chunk.set("Level", level);
|
||
|
+
|
||
|
+ return chunk;
|
||
|
+ } catch (Throwable throwable) {
|
||
|
+ throwable.printStackTrace();
|
||
|
+ throw throwable;
|
||
|
+ }
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
+ private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) {
|
||
|
+ NBTTagList levelList = level.getList(key, 10);
|
||
|
+ NBTTagList oversizedList = oversizedLevel.getList(key, 10);
|
||
|
+
|
||
|
+ if (!oversizedList.isEmpty()) {
|
||
|
+ levelList.addAll(oversizedList);
|
||
|
+ level.set(key, levelList);
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
+ private static int getNBTSize(NBTBase nbtBase) {
|
||
|
+ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
|
||
|
+ try {
|
||
|
+ nbtBase.write(test);
|
||
|
+ return test.size();
|
||
|
+ } catch (IOException e) {
|
||
|
+ e.printStackTrace();
|
||
|
+ return 0;
|
||
|
+ }
|
||
|
+ }
|
||
|
+
|
||
|
// Paper End
|
||
|
|
||
|
public static synchronized void a() {
|
||
|
@@ -108,6 +232,12 @@ public class RegionFileCache {
|
||
|
// CraftBukkit start - call sites hoisted for synchronization
|
||
|
public static NBTTagCompound read(File file, int i, int j) throws IOException { // Paper - remove synchronization
|
||
|
RegionFile regionfile = a(file, i, j);
|
||
|
+ // Paper start
|
||
|
+ if (regionfile.isOversized(i, j)) {
|
||
|
+ printOversizedLog("Loading Oversized Chunk!", file, i, j);
|
||
|
+ return readOversizedChunk(regionfile, i, j);
|
||
|
+ }
|
||
|
+ // Paper end
|
||
|
|
||
|
DataInputStream datainputstream = regionfile.a(i & 31, j & 31);
|
||
|
|
||
|
@@ -121,11 +251,14 @@ public class RegionFileCache {
|
||
|
@Nullable
|
||
|
public static void write(File file, int i, int j, NBTTagCompound nbttagcompound) throws IOException {
|
||
|
int attempts = 0; Exception laste = null; while (attempts++ < 5) { try { // Paper
|
||
|
- RegionFile regionfile = a(file, i, j);
|
||
|
-
|
||
|
- DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31);
|
||
|
- NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream);
|
||
|
- dataoutputstream.close();
|
||
|
+ writeRegion(file, i, j, nbttagcompound); // Paper - moved to own method
|
||
|
+ // Paper start
|
||
|
+// RegionFile regionfile = a(file, i, j);
|
||
|
+//
|
||
|
+// DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31);
|
||
|
+// NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream);
|
||
|
+// dataoutputstream.close();
|
||
|
+ // Paper end
|
||
|
// Paper start
|
||
|
laste = null; break; // Paper
|
||
|
} catch (Exception exception) {
|
||
|
--
|
||
|
2.20.1
|
||
|
|