diff --git a/Spigot-Server-Patches/0417-Allow-Saving-of-Oversized-Chunks.patch b/Spigot-Server-Patches/0417-Allow-Saving-of-Oversized-Chunks.patch new file mode 100644 index 0000000000..2c0d261c5f --- /dev/null +++ b/Spigot-Server-Patches/0417-Allow-Saving-of-Oversized-Chunks.patch @@ -0,0 +1,416 @@ +From ba4afed44f76c2d0b8ba63439ecc28b6c2212d05 Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Fri, 15 Feb 2019 01:08:19 -0500 +Subject: [PATCH] Allow Saving of Oversized Chunks + +The Minecraft World Region File format has a hard cap of 1MB per chunk. +This is due to the fact that the header of the file format only allocates +a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector. + +This limit can be reached fairly easily with books, resulting in the chunk being unable +to save to the world. Worse off, is that nothing printed when this occured, and silently +performed a chunk rollback on next load. + +This leads to security risk with duplication and is being actively exploited. + +This patch catches the too large scenario, falls back and moves any large Entity +or Tile Entity into a new compound, and this compound is saved into a different file. + +On Chunk Load, we check for oversized status, and if so, we load the extra file and +merge the Entities and Tile Entities from the oversized chunk back into the level to +then be loaded as normal. + +Once a chunk is returned back to normal size, the oversized flag will clear, and no +extra data file will exist. + +This fix maintains compatability with all existing Anvil Region Format tools as it +does not alter the save format. They will just not know about the extra entities. + +This fix also maintains compatability if someone switches server jars to one without +this fix, as the data will remain in the oversized file. Once the server returns +to a jar with this fix, the data will be restored. + +diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java +index 12268f87b9..e1f7e06ab2 100644 +--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java ++++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java +@@ -39,6 +39,7 @@ public class NBTCompressedStreamTools { + + } + ++ public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER + public static NBTTagCompound a(DataInputStream datainputstream) throws IOException { + return a((DataInput) datainputstream, NBTReadLimiter.a); + } +@@ -59,6 +60,7 @@ public class NBTCompressedStreamTools { + } + } + ++ public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER + public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { + a((NBTBase) nbttagcompound, dataoutput); + } +diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java +index 999adf7dc7..303c13ccd0 100644 +--- a/src/main/java/net/minecraft/server/RegionFile.java ++++ b/src/main/java/net/minecraft/server/RegionFile.java +@@ -75,6 +75,7 @@ public class RegionFile { + } + header.clear(); + IntBuffer headerAsInts = header.asIntBuffer(); ++ initOversizedState(); + // Paper End + + for(int j1 = 0; j1 < 1024; ++j1) { +@@ -99,7 +100,7 @@ public class RegionFile { + } + + @Nullable +- public synchronized DataInputStream a(int i, int j) { ++ public synchronized DataInputStream getReadStream(int i, int j) { return a(i, j); } @Nullable public synchronized DataInputStream a(int i, int j) { // Paper - OBFHELPER + if (this.e(i, j)) { + return null; + } else { +@@ -175,8 +176,8 @@ public class RegionFile { + } + + @Nullable +- public DataOutputStream c(int i, int j) { +- return this.e(i, j) ? null : new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new RegionFile.ChunkBuffer(i, j)))); ++ public DataOutputStream getWriteStream(int i, int j) { return c(i, j); } @Nullable public DataOutputStream c(int i, int j) { // Paper - OBFHELPER ++ return this.e(i, j) ? null : new DataOutputStream(new RegionFile.ChunkBuffer(i, j)); // Paper - remove middleware, move deflate to .close() for dynamic levels + } + + protected synchronized void a(int i, int j, byte[] abyte, int k) { +@@ -187,7 +188,7 @@ public class RegionFile { + int k1 = (k + 5) / 4096 + 1; + + if (k1 >= 256) { +- return; ++ throw new ChunkTooLargeException(i, j, k1); // Paper - throw error instead + } + + if (i1 != 0 && j1 == k1) { +@@ -338,6 +339,101 @@ public class RegionFile { + logger.error("Error backing up corrupt file" + file.getAbsolutePath(), e); + } + } ++ ++ private final byte[] oversized = new byte[1024]; ++ private int oversizedCount = 0; ++ ++ private synchronized void initOversizedState() throws IOException { ++ File metaFile = getOversizedMetaFile(); ++ if (metaFile.exists()) { ++ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath()); ++ System.arraycopy(read, 0, oversized, 0, oversized.length); ++ for (byte temp : oversized) { ++ oversizedCount += temp; ++ } ++ } ++ } ++ ++ private static int getChunkIndex(int x, int z) { ++ return (x & 31) + (z & 31) * 32; ++ } ++ synchronized boolean isOversized(int x, int z) { ++ return this.oversized[getChunkIndex(x, z)] == 1; ++ } ++ synchronized void setOversized(int x, int z, boolean oversized) throws IOException { ++ final int offset = getChunkIndex(x, z); ++ boolean previous = this.oversized[offset] == 1; ++ this.oversized[offset] = (byte) (oversized ? 1 : 0); ++ if (!previous && oversized) { ++ oversizedCount++; ++ } else if (!oversized && previous) { ++ oversizedCount--; ++ } ++ if (previous && !oversized) { ++ File oversizedFile = getOversizedFile(x, z); ++ if (oversizedFile.exists()) { ++ oversizedFile.delete(); ++ } ++ } ++ if (oversizedCount > 0) { ++ if (previous != oversized) { ++ writeOversizedMeta(); ++ } ++ } else if (previous) { ++ File oversizedMetaFile = getOversizedMetaFile(); ++ if (oversizedMetaFile.exists()) { ++ oversizedMetaFile.delete(); ++ } ++ } ++ } ++ ++ private void writeOversizedMeta() throws IOException { ++ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized); ++ } ++ ++ private File getOversizedMetaFile() { ++ return new File(getFile().getParentFile(), getFile().getName().replaceAll("\\.mca$", "") + ".oversized.nbt"); ++ } ++ ++ private File getOversizedFile(int x, int z) { ++ return new File(this.getFile().getParentFile(), this.getFile().getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); ++ } ++ ++ void writeOversizedData(int x, int z, NBTTagCompound oversizedData) throws IOException { ++ File file = getOversizedFile(x, z); ++ try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new java.io.FileOutputStream(file), new java.util.zip.Deflater(java.util.zip.Deflater.BEST_COMPRESSION), 32 * 1024), 32 * 1024))) { ++ NBTCompressedStreamTools.writeNBT(oversizedData, out); ++ } ++ this.setOversized(x, z, true); ++ ++ } ++ ++ synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException { ++ File file = getOversizedFile(x, z); ++ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) { ++ return NBTCompressedStreamTools.readNBT(out); ++ } ++ ++ } ++ ++ public class ChunkTooLargeException extends RuntimeException { ++ public ChunkTooLargeException(int x, int z, int sectors) { ++ super("Chunk " + x + "," + z + " of " + getFile().toString() + " is too large (" + sectors + "/256)"); ++ } ++ } ++ private static class DirectByteArrayOutputStream extends ByteArrayOutputStream { ++ public DirectByteArrayOutputStream() { ++ super(); ++ } ++ ++ public DirectByteArrayOutputStream(int size) { ++ super(size); ++ } ++ ++ public byte[] getBuffer() { ++ return this.buf; ++ } ++ } + // Paper end + + class ChunkBuffer extends ByteArrayOutputStream { +@@ -351,8 +447,40 @@ public class RegionFile { + this.c = j; + } + +- public void close() { +- RegionFile.this.a(this.b, this.c, this.buf, this.count); ++ public void close() throws IOException { ++ // Paper start - apply dynamic compression ++ int origLength = this.count; ++ byte[] buf = this.buf; ++ DirectByteArrayOutputStream out = compressData(buf, origLength); ++ byte[] bytes = out.getBuffer(); ++ int length = out.size(); ++ ++ RegionFile.this.a(this.b, this.c, bytes, length); // Paper - change to bytes/length ++ // Paper end ++ } ++ } ++ ++ private static DirectByteArrayOutputStream compressData(byte[] buf, int length) throws IOException { ++ final java.util.zip.Deflater deflater; ++ if (length > 1024 * 512) { ++ deflater = new java.util.zip.Deflater(9); ++ } else if (length > 1024 * 128) { ++ deflater = new java.util.zip.Deflater(8); ++ } else { ++ deflater = new java.util.zip.Deflater(6); ++ } ++ ++ ++ deflater.setInput(buf, 0, length); ++ deflater.finish(); ++ ++ DirectByteArrayOutputStream out = new DirectByteArrayOutputStream(length); ++ byte[] buffer = new byte[1024 * (length > 1024 * 124 ? 32 : 16)]; ++ while (!deflater.finished()) { ++ out.write(buffer, 0, deflater.deflate(buffer)); + } ++ out.close(); ++ deflater.end(); ++ return out; + } + } +diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java +index 8c8b7cbab5..50a62d6e23 100644 +--- a/src/main/java/net/minecraft/server/RegionFileCache.java ++++ b/src/main/java/net/minecraft/server/RegionFileCache.java +@@ -16,6 +16,7 @@ public class RegionFileCache { + + public static final Map cache = new LinkedHashMap(PaperConfig.regionFileCacheSize, 0.75f, true); // Paper - HashMap -> LinkedHashMap + ++ public static synchronized RegionFile getRegionFile(File file, int i, int j) { return a(file, i, j); } // Paper - OBFHELPER + public static synchronized RegionFile a(File file, int i, int j) { + File file1 = new File(file, "region"); + File file2 = new File(file1, "r." + (i >> 5) + "." + (j >> 5) + ".mca"); +@@ -83,6 +84,129 @@ public class RegionFileCache { + public static synchronized boolean hasRegionFile(File file, int i, int j) { + return RegionFileCache.cache.containsKey(getRegionFileName(file, i, j)); + } ++ private static void printOversizedLog(String msg, File file, int x, int z) { ++ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); ++ } ++ ++ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8; ++ private static final int OVERZEALOUS_THRESHOLD = 1024 * 2; ++ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD; ++ private static void resetFilterThresholds() { ++ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD)); ++ } ++ static { ++ resetFilterThresholds(); ++ } ++ private static void writeRegion(File file, int x, int z, NBTTagCompound nbttagcompound) throws IOException { ++ RegionFile regionfile = getRegionFile(file, x, z); ++ ++ DataOutputStream out = regionfile.getWriteStream(x & 31, z & 31); ++ try { ++ NBTCompressedStreamTools.writeNBT(nbttagcompound, out); ++ out.close(); ++ regionfile.setOversized(x, z, false); ++ } catch (RegionFile.ChunkTooLargeException ignored) { ++ printOversizedLog("ChunkTooLarge! Someone is trying to duplicate.", file, x, z); ++ // Clone as we are now modifying it, don't want to corrupt the pending save state ++ nbttagcompound = nbttagcompound.clone(); ++ // Filter out TileEntities and Entities ++ NBTTagCompound oversizedData = filterChunkData(nbttagcompound); ++ //noinspection SynchronizationOnLocalVariableOrMethodParameter ++ synchronized (regionfile) { ++ out = regionfile.getWriteStream(x & 31, z & 31); ++ NBTCompressedStreamTools.writeNBT(nbttagcompound, out); ++ try { ++ out.close(); ++ // 2048 is below the min allowed, so it means we enter overzealous mode below ++ if (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD) { ++ resetFilterThresholds(); ++ } ++ } catch (RegionFile.ChunkTooLargeException e) { ++ printOversizedLog("ChunkTooLarge even after reduction. Trying in overzealous mode.", file, x, z); ++ // Eek, major fail. We have retry logic, so reduce threshholds and fall back ++ SIZE_THRESHOLD = OVERZEALOUS_THRESHOLD; ++ throw e; ++ } ++ ++ regionfile.writeOversizedData(x, z, oversizedData); ++ } ++ } catch (Exception e) { ++ e.printStackTrace(); ++ throw e; ++ } ++ ++ } ++ ++ private static NBTTagCompound filterChunkData(NBTTagCompound chunk) { ++ NBTTagCompound oversizedLevel = new NBTTagCompound(); ++ NBTTagCompound level = chunk.getCompound("Level"); ++ filterChunkList(level, oversizedLevel, "Entities"); ++ filterChunkList(level, oversizedLevel, "TileEntities"); ++ NBTTagCompound oversized = new NBTTagCompound(); ++ oversized.set("Level", oversizedLevel); ++ return oversized; ++ } ++ ++ private static void filterChunkList(NBTTagCompound level, NBTTagCompound extra, String key) { ++ NBTTagList list = level.getList(key, 10); ++ NBTTagList newList = extra.getList(key, 10); ++ for (Iterator iterator = list.list.iterator(); iterator.hasNext(); ) { ++ NBTBase object = iterator.next(); ++ if (getNBTSize(object) > SIZE_THRESHOLD) { ++ newList.add(object); ++ iterator.remove(); ++ } ++ } ++ level.set(key, list); ++ extra.set(key, newList); ++ } ++ ++ ++ private static NBTTagCompound readOversizedChunk(RegionFile regionfile, int i, int j) throws IOException { ++ synchronized (regionfile) { ++ try (DataInputStream datainputstream = regionfile.getReadStream(i & 31, j & 31)) { ++ NBTTagCompound oversizedData = regionfile.getOversizedData(i, j); ++ NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream); ++ if (oversizedData == null) { ++ return chunk; ++ } ++ NBTTagCompound oversizedLevel = oversizedData.getCompound("Level"); ++ NBTTagCompound level = chunk.getCompound("Level"); ++ ++ mergeChunkList(level, oversizedLevel, "Entities"); ++ mergeChunkList(level, oversizedLevel, "TileEntities"); ++ ++ chunk.set("Level", level); ++ ++ return chunk; ++ } catch (Throwable throwable) { ++ throwable.printStackTrace(); ++ throw throwable; ++ } ++ } ++ } ++ ++ private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) { ++ NBTTagList levelList = level.getList(key, 10); ++ NBTTagList oversizedList = oversizedLevel.getList(key, 10); ++ ++ if (!oversizedList.isEmpty()) { ++ levelList.addAll(oversizedList); ++ level.set(key, levelList); ++ } ++ } ++ ++ private static int getNBTSize(NBTBase nbtBase) { ++ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream()); ++ try { ++ nbtBase.write(test); ++ return test.size(); ++ } catch (IOException e) { ++ e.printStackTrace(); ++ return 0; ++ } ++ } ++ + // Paper End + + public static synchronized void a() { +@@ -108,6 +232,12 @@ public class RegionFileCache { + // CraftBukkit start - call sites hoisted for synchronization + public static NBTTagCompound read(File file, int i, int j) throws IOException { // Paper - remove synchronization + RegionFile regionfile = a(file, i, j); ++ // Paper start ++ if (regionfile.isOversized(i, j)) { ++ printOversizedLog("Loading Oversized Chunk!", file, i, j); ++ return readOversizedChunk(regionfile, i, j); ++ } ++ // Paper end + + DataInputStream datainputstream = regionfile.a(i & 31, j & 31); + +@@ -121,11 +251,14 @@ public class RegionFileCache { + @Nullable + public static void write(File file, int i, int j, NBTTagCompound nbttagcompound) throws IOException { + int attempts = 0; Exception laste = null; while (attempts++ < 5) { try { // Paper +- RegionFile regionfile = a(file, i, j); +- +- DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31); +- NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream); +- dataoutputstream.close(); ++ writeRegion(file, i, j, nbttagcompound); // Paper - moved to own method ++ // Paper start ++// RegionFile regionfile = a(file, i, j); ++// ++// DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31); ++// NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream); ++// dataoutputstream.close(); ++ // Paper end + // Paper start + laste = null; break; // Paper + } catch (Exception exception) { +-- +2.20.1 +