Paper/patches/server/0986-Allow-Saving-of-Oversized-Chunks.patch
Jake Potrebic bd38e0318a
Updated Upstream (Bukkit/CraftBukkit) (#10379)
Updated Upstream (Bukkit/CraftBukkit)

Upstream has released updates that appear to apply and compile correctly.
This update has not been tested by PaperMC and as with ANY update, please do your own testing

Bukkit Changes:
f02baa38 PR-988: Add World#getIntersectingChunks(BoundingBox)
9321d665 Move getItemInUse up to LivingEntity
819eef73 PR-959: Add access to current item's remaining ticks
c4fdadb0 SPIGOT-7601: Add AbstractArrow#getItem
be8261ca Add support for Java 22
26119676 PR-979: Add more translation keys
66753362 PR-985: Correct book maximum pages and characters per page documentation
c8be92fa PR-980: Improve getArmorContents() documentation
f1120ee2 PR-983: Expose riptide velocity to PlayerRiptideEvent

CraftBukkit Changes:
dfaa89bbe PR-1369: Add World#getIntersectingChunks(BoundingBox)
51bbab2b9 Move getItemInUse up to LivingEntity
668e09602 PR-1331: Add access to current item's remaining ticks
a639406d1 SPIGOT-7601: Add AbstractArrow#getItem
0398930fc SPIGOT-7602: Allow opening in-world horse and related inventories
ffd15611c SPIGOT-7608: Allow empty lists to morph to any PDT list
2188dcfa9 Add support for Java 22
45d6a609f SPIGOT-7604: Revert "SPIGOT-7365: DamageCause blocked by shield should trigger invulnerableTime"
06d915943 SPIGOT-7365: DamageCause blocked by shield should trigger invulnerableTime
ca3bc3707 PR-1361: Add more translation keys
366c3ca80 SPIGOT-7600: EntityChangeBlockEvent is not fired for frog eggs
06d0f9ba8 SPIGOT-7593: Fix sapling growth physics / client-side updates
45c2608e4 PR-1366: Expose riptide velocity to PlayerRiptideEvent
29b6bb79b SPIGOT-7587: Remove fixes for now-resolved MC-142590 and MC-109346
2024-04-06 12:53:39 -07:00

212 lines
10 KiB
Diff

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 15 Feb 2019 01:08:19 -0500
Subject: [PATCH] Allow Saving of Oversized Chunks
Note 1.17 update: With 1.17, Entities are no longer stored in chunk slices, so this needs updating!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
The Minecraft World Region File format has a hard cap of 1MB per chunk.
This is due to the fact that the header of the file format only allocates
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.
This limit can be reached fairly easily with books, resulting in the chunk being unable
to save to the world. Worse off, is that nothing printed when this occured, and silently
performed a chunk rollback on next load.
This leads to security risk with duplication and is being actively exploited.
This patch catches the too large scenario, falls back and moves any large Entity
or Tile Entity into a new compound, and this compound is saved into a different file.
On Chunk Load, we check for oversized status, and if so, we load the extra file and
merge the Entities and Tile Entities from the oversized chunk back into the level to
then be loaded as normal.
Once a chunk is returned back to normal size, the oversized flag will clear, and no
extra data file will exist.
This fix maintains compatability with all existing Anvil Region Format tools as it
does not alter the save format. They will just not know about the extra entities.
This fix also maintains compatability if someone switches server jars to one without
this fix, as the data will remain in the oversized file. Once the server returns
to a jar with this fix, the data will be restored.
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
index bc8038da65f834249c61a262fc1a5abb7cc91a63..6c89b92cac521808873e9e1eccc363695275cd7a 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
@@ -18,8 +18,11 @@ import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
+import java.util.zip.InflaterInputStream; // Paper
import javax.annotation.Nullable;
import net.minecraft.Util;
+import net.minecraft.nbt.CompoundTag; // Paper
+import net.minecraft.nbt.NbtIo; // Paper
import net.minecraft.world.level.ChunkPos;
import org.slf4j.Logger;
@@ -45,6 +48,7 @@ public class RegionFile implements AutoCloseable {
@VisibleForTesting
protected final RegionBitmap usedSectors;
public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(); // Paper
+ public final Path regionFile; // Paper
public RegionFile(Path file, Path directory, boolean dsync) throws IOException {
this(file, directory, RegionFileVersion.getCompressionFormat(), dsync); // Paper - Configurable region compression format
@@ -52,6 +56,8 @@ public class RegionFile implements AutoCloseable {
public RegionFile(Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync) throws IOException {
this.header = ByteBuffer.allocateDirect(8192);
+ this.regionFile = file; // Paper
+ initOversizedState(); // Paper
this.usedSectors = new RegionBitmap();
this.version = outputChunkStreamVersion;
if (!Files.isDirectory(directory, new LinkOption[0])) {
@@ -431,6 +437,74 @@ public class RegionFile implements AutoCloseable {
}
public static final int MAX_CHUNK_SIZE = 500 * 1024 * 1024; // Paper - don't write garbage data to disk if writing serialization fails
+ // Paper start
+ private final byte[] oversized = new byte[1024];
+ private int oversizedCount;
+
+ private synchronized void initOversizedState() throws IOException {
+ Path metaFile = getOversizedMetaFile();
+ if (Files.exists(metaFile)) {
+ final byte[] read = java.nio.file.Files.readAllBytes(metaFile);
+ System.arraycopy(read, 0, oversized, 0, oversized.length);
+ for (byte temp : oversized) {
+ oversizedCount += temp;
+ }
+ }
+ }
+
+ private static int getChunkIndex(int x, int z) {
+ return (x & 31) + (z & 31) * 32;
+ }
+ synchronized boolean isOversized(int x, int z) {
+ return this.oversized[getChunkIndex(x, z)] == 1;
+ }
+ synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
+ final int offset = getChunkIndex(x, z);
+ boolean previous = this.oversized[offset] == 1;
+ this.oversized[offset] = (byte) (oversized ? 1 : 0);
+ if (!previous && oversized) {
+ oversizedCount++;
+ } else if (!oversized && previous) {
+ oversizedCount--;
+ }
+ if (previous && !oversized) {
+ Path oversizedFile = getOversizedFile(x, z);
+ if (Files.exists(oversizedFile)) {
+ Files.delete(oversizedFile);
+ }
+ }
+ if (oversizedCount > 0) {
+ if (previous != oversized) {
+ writeOversizedMeta();
+ }
+ } else if (previous) {
+ Path oversizedMetaFile = getOversizedMetaFile();
+ if (Files.exists(oversizedMetaFile)) {
+ Files.delete(oversizedMetaFile);
+ }
+ }
+ }
+
+ private void writeOversizedMeta() throws IOException {
+ java.nio.file.Files.write(getOversizedMetaFile(), oversized);
+ }
+
+ private Path getOversizedMetaFile() {
+ return this.regionFile.getParent().resolve(this.regionFile.getFileName().toString().replaceAll("\\.mca$", "") + ".oversized.nbt");
+ }
+
+ private Path getOversizedFile(int x, int z) {
+ return this.regionFile.getParent().resolve(this.regionFile.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
+ }
+
+ synchronized CompoundTag getOversizedData(int x, int z) throws IOException {
+ Path file = getOversizedFile(x, z);
+ try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) {
+ return NbtIo.read((java.io.DataInput) out);
+ }
+
+ }
+ // Paper end
private class ChunkBuffer extends ByteArrayOutputStream {
private final ChunkPos pos;
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
index f1ecc3832da094400ed9d45bfc60af10da682b4a..f27cf743bbc379520263909541d653dd38d1be58 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
@@ -121,6 +121,43 @@ public class RegionFileStorage implements AutoCloseable {
}
}
+ // Paper start
+ private static void printOversizedLog(String msg, Path file, int x, int z) {
+ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
+ }
+
+ private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException {
+ synchronized (regionfile) {
+ try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) {
+ CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
+ CompoundTag chunk = NbtIo.read((DataInput) datainputstream);
+ if (oversizedData == null) {
+ return chunk;
+ }
+ CompoundTag oversizedLevel = oversizedData.getCompound("Level");
+
+ mergeChunkList(chunk.getCompound("Level"), oversizedLevel, "Entities", "Entities");
+ mergeChunkList(chunk.getCompound("Level"), oversizedLevel, "TileEntities", "TileEntities");
+
+ return chunk;
+ } catch (Throwable throwable) {
+ throwable.printStackTrace();
+ throw throwable;
+ }
+ }
+ }
+
+ private static void mergeChunkList(CompoundTag level, CompoundTag oversizedLevel, String key, String oversizedKey) {
+ net.minecraft.nbt.ListTag levelList = level.getList(key, net.minecraft.nbt.Tag.TAG_COMPOUND);
+ net.minecraft.nbt.ListTag oversizedList = oversizedLevel.getList(oversizedKey, net.minecraft.nbt.Tag.TAG_COMPOUND);
+
+ if (!oversizedList.isEmpty()) {
+ levelList.addAll(oversizedList);
+ level.put(key, levelList);
+ }
+ }
+ // Paper end
+
@Nullable
public CompoundTag read(ChunkPos pos) throws IOException {
// CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing
@@ -132,6 +169,12 @@ public class RegionFileStorage implements AutoCloseable {
try { // Paper
DataInputStream datainputstream = regionfile.getChunkDataInputStream(pos);
+ // Paper start
+ if (regionfile.isOversized(pos.x, pos.z)) {
+ printOversizedLog("Loading Oversized Chunk!", regionfile.regionFile, pos.x, pos.z);
+ return readOversizedChunk(regionfile, pos);
+ }
+ // Paper end
CompoundTag nbttagcompound;
label43:
{
@@ -223,6 +266,7 @@ public class RegionFileStorage implements AutoCloseable {
try {
NbtIo.write(nbt, (DataOutput) dataoutputstream);
+ regionfile.setOversized(pos.x, pos.z, false); // Paper - We don't do this anymore, mojang stores differently, but clear old meta flag if it exists to get rid of our own meta file once last oversized is gone
// Paper start - don't write garbage data to disk if writing serialization fails
dataoutputstream.close(); // Only write if successful
} catch (final RegionFileSizeException e) {