2021-05-21 02:31:08 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Thu, 20 May 2021 07:02:22 -0700
Subject: [PATCH] Fix and optimise world force upgrading
The WorldUpgrader class was incorrectly modified by
CB. It will store an IChunkLoader instance for all
dimension types in the world, but obviously with how
CB shifts around worlds only one dimension type exists
per world. But this would be OK if CB did this
change correctly. All IChunkLoader instances
will point to the same regionfiles. And all
IChunkLoader instances are going to be read from.
This problem hasn't really been reported because
it relies on the persistent legacy data to be converted
as well to cause corruption. Why? Because the legacy
data is also shared, it will result in different
outputs from conversion (as once conversion for legacy
persistent data takes place, it is REMOVED - so the next
convert will _not_ have the data). Which means different
sizes on disk. Which means different regionfile sector
allocations. Which means there are 3 different possible
regionfile sector allocations in memory, and none of them
are going to be correct.
I've fixed this by writing a world upgrader suited to
CB's changes to world folder format. It was brain dead
easy to add threading, so I did.
diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
new file mode 100644
index 0000000000000000000000000000000000000000..db9f1241094a7529662ed643ba651d86ec86e09c
--- /dev/null
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
@@ -0,0 +1,200 @@
+package io.papermc.paper.world;
+
+import com.mojang.datafixers.DataFixer;
+import net.minecraft.SharedConstants;
+import net.minecraft.nbt.NBTTagCompound;
+import net.minecraft.resources.ResourceKey;
+import net.minecraft.util.worldupdate.WorldUpgrader;
+import net.minecraft.world.level.ChunkCoordIntPair;
+import net.minecraft.world.level.chunk.storage.IChunkLoader;
+import net.minecraft.world.level.chunk.storage.RegionFileCache;
+import net.minecraft.world.level.dimension.DimensionManager;
+import net.minecraft.world.level.dimension.WorldDimension;
+import net.minecraft.world.level.storage.Convertable;
+import net.minecraft.world.level.storage.WorldPersistentData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+public class ThreadedWorldUpgrader {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final ResourceKey<WorldDimension> dimensionType;
+ private final ResourceKey<DimensionManager> worldKey;
+ private final String worldName;
+ private final ExecutorService threadPool;
+ private final DataFixer dataFixer;
+ private final boolean removeCaches;
+
+ public ThreadedWorldUpgrader(final ResourceKey<WorldDimension> dimensionType, final ResourceKey<DimensionManager> worldKey, final String worldName, final int threads,
+ final DataFixer dataFixer, final boolean removeCaches) {
+ this.dimensionType = dimensionType;
+ this.worldKey = worldKey;
+ this.worldName = worldName;
+ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
+ private final AtomicInteger threadCounter = new AtomicInteger();
+
+ @Override
+ public Thread newThread(final Runnable run) {
+ final Thread ret = new Thread(run);
+
+ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
+ ret.setUncaughtExceptionHandler((thread, throwable) -> {
+ LOGGER.fatal("Error upgrading world", throwable);
+ });
+
+ return ret;
+ }
+ });
+ this.dataFixer = dataFixer;
+ this.removeCaches = removeCaches;
+ }
+
+ public void convert() {
+ final File worldFolder = Convertable.getFolder(new File(this.worldName), this.dimensionType);
+ final WorldPersistentData worldPersistentData = new WorldPersistentData(new File(worldFolder, "data"), this.dataFixer);
+
+ final File regionFolder = new File(worldFolder, "region");
+
+ LOGGER.info("Force upgrading " + this.worldName);
+ LOGGER.info("Counting regionfiles for " + this.worldName);
+ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
+ return WorldUpgrader.getRegionfileRegex().matcher(name).matches();
+ });
+ if (regionFiles == null) {
+ LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
+ return;
+ }
+ LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
+ LOGGER.info("Starting conversion now for world " + this.worldName);
+
+ final WorldInfo info = new WorldInfo(() -> worldPersistentData,
+ new IChunkLoader(regionFolder, this.dataFixer, false), this.removeCaches, this.worldKey);
+
+ long expectedChunks = (long)regionFiles.length * (32L * 32L);
+
+ for (final File regionFile : regionFiles) {
+ final ChunkCoordIntPair regionPos = RegionFileCache.getRegionFileCoordinates(regionFile);
+ if (regionPos == null) {
+ expectedChunks -= (32L * 32L);
+ continue;
+ }
+
+ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
+ }
+ this.threadPool.shutdown();
+
+ final DecimalFormat format = new DecimalFormat("#0.00");
+
+ final long start = System.nanoTime();
+
+ while (!this.threadPool.isTerminated()) {
+ final long current = info.convertedChunks.get();
+
+ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
+
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ignore) {}
+ }
+
+ final long end = System.nanoTime();
+
+ try {
+ info.loader.close();
+ } catch (final IOException ex) {
+ LOGGER.fatal("Failed to close chunk loader", ex);
+ }
+ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
+ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
+ }
+
+ private static final class WorldInfo {
+
+ public final Supplier<WorldPersistentData> persistentDataSupplier;
+ public final IChunkLoader loader;
+ public final boolean removeCaches;
+ public final ResourceKey<DimensionManager> worldKey;
+ public final AtomicLong convertedChunks = new AtomicLong();
+ public final AtomicLong modifiedChunks = new AtomicLong();
+
+ private WorldInfo(final Supplier<WorldPersistentData> persistentDataSupplier, final IChunkLoader loader, final boolean removeCaches,
+ final ResourceKey<DimensionManager> worldKey) {
+ this.persistentDataSupplier = persistentDataSupplier;
+ this.loader = loader;
+ this.removeCaches = removeCaches;
+ this.worldKey = worldKey;
+ }
+ }
+
+ private static final class ConvertTask implements Runnable {
+
+ private final WorldInfo worldInfo;
+ private final int regionX;
+ private final int regionZ;
+
+ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
+ this.worldInfo = worldInfo;
+ this.regionX = regionX;
+ this.regionZ = regionZ;
+ }
+
+ @Override
+ public void run() {
+ final int regionCX = this.regionX << 5;
+ final int regionCZ = this.regionZ << 5;
+
+ final Supplier<WorldPersistentData> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
+ final IChunkLoader loader = this.worldInfo.loader;
+ final boolean removeCaches = this.worldInfo.removeCaches;
+ final ResourceKey<DimensionManager> worldKey = this.worldInfo.worldKey;
+
+ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
+ for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
+ final ChunkCoordIntPair chunkPos = new ChunkCoordIntPair(cx, cz);
+ try {
+ // no need to check the coordinate of the chunk, the regionfilecache does that for us
+
+ NBTTagCompound chunkNBT = loader.read(chunkPos);
+
+ if (chunkNBT == null) {
+ continue;
+ }
+
+ final int versionBefore = IChunkLoader.getVersion(chunkNBT);
+
+ chunkNBT = loader.getChunkData(worldKey, persistentDataSupplier, chunkNBT, chunkPos, null);
+
+ boolean modified = versionBefore < SharedConstants.getGameVersion().getWorldVersion();
+
+ if (removeCaches) {
+ final NBTTagCompound level = chunkNBT.getCompound("Level");
+ modified |= level.hasKey("Heightmaps");
+ level.remove("Heightmaps");
+ modified |= level.hasKey("isLightOn");
+ level.remove("isLightOn");
+ }
+
+ if (modified) {
+ this.worldInfo.modifiedChunks.getAndIncrement();
+ loader.write(chunkPos, chunkNBT);
+ }
+ } catch (final Exception ex) {
+ LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
+ } finally {
+ this.worldInfo.convertedChunks.getAndIncrement();
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
index bf4051349917cc1d727fc5544237e0291cb6f1e6..15b972b4a93b8fe3655acec47bc84b0f2c4620a6 100644
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
@@ -15,6 +15,7 @@ import java.nio.file.Paths;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.function.BooleanSupplier;
+import io.papermc.paper.world.ThreadedWorldUpgrader;
import joptsimple.NonOptionArgumentSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
@@ -277,6 +278,15 @@ public class Main {
}
// Paper end
+ // Paper start - fix and optimise world upgrading
+ public static void convertWorldButItWorks(ResourceKey<WorldDimension> dimensionType, ResourceKey<DimensionManager> worldKey, String worldName,
+ DataFixer dataFixer, boolean removeCaches) {
+ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
+ final ThreadedWorldUpgrader worldUpgrader = new ThreadedWorldUpgrader(dimensionType, worldKey, worldName, threads, dataFixer, removeCaches);
+ worldUpgrader.convert();
+ }
+ // Paper end - fix and optimise world upgrading
+
public static void convertWorld(Convertable.ConversionSession convertable_conversionsession, DataFixer datafixer, boolean flag, BooleanSupplier booleansupplier, ImmutableSet<ResourceKey<DimensionManager>> immutableset) { // CraftBukkit
Main.LOGGER.info("Forcing world upgrade! {}", convertable_conversionsession.getLevelName()); // CraftBukkit
WorldUpgrader worldupgrader = new WorldUpgrader(convertable_conversionsession, datafixer, immutableset, flag);
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
2021-05-28 09:29:41 +02:00
index d9ea784758fe52782042ce4b61faa915f895b3f8..ca8ca3ada9d8074a2b67c590a98c459563513ba6 100644
2021-05-21 02:31:08 +02:00
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
2021-05-28 09:29:41 +02:00
@@ -515,13 +515,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
2021-05-21 02:31:08 +02:00
worlddata = new WorldDataServer(worldsettings, generatorsettings, Lifecycle.stable());
}
worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
- if (options.has("forceUpgrade")) {
- net.minecraft.server.Main.convertWorld(worldSession, DataConverterRegistry.a(), options.has("eraseCache"), () -> {
- return true;
- }, worlddata.getGeneratorSettings().d().d().stream().map((entry1) -> {
- return ResourceKey.a(IRegistry.K, ((ResourceKey) entry1.getKey()).a());
- }).collect(ImmutableSet.toImmutableSet()));
- }
+ // Paper - move down
IWorldDataServer iworlddataserver = worlddata;
GeneratorSettings generatorsettings = worlddata.getGeneratorSettings();
2021-05-28 09:29:41 +02:00
@@ -541,6 +535,14 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
2021-05-21 02:31:08 +02:00
chunkgenerator = worlddimension.c();
}
+ // Paper start - fix and optimise world upgrading
+ if (options.has("forceUpgrade")) {
+ net.minecraft.server.Main.convertWorldButItWorks(
+ dimensionKey, World.getDimensionKey(dimensionmanager), worldSession.getLevelName(), DataConverterRegistry.getDataFixer(), options.has("eraseCache")
+ );
+ }
+ // Paper end - fix and optimise world upgrading
+
ResourceKey<World> worldKey = ResourceKey.a(IRegistry.L, dimensionKey.a());
if (dimensionKey == WorldDimension.OVERWORLD) {
diff --git a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
index a4970654496c52fcd02c5c055ff5ac551bd19da3..dca2e9e45116df22d8c95d1be8f0a7e3c2d2b6b1 100644
--- a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
+++ b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
@@ -55,7 +55,7 @@ public class WorldUpgrader {
private volatile int m;
private final Object2FloatMap<ResourceKey<DimensionManager>> n = Object2FloatMaps.synchronize(new Object2FloatOpenCustomHashMap(SystemUtils.k())); // CraftBukkit
private volatile IChatBaseComponent o = new ChatMessage("optimizeWorld.stage.counting");
- private static final Pattern p = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$");
+ private static final Pattern p = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); public static final Pattern getRegionfileRegex() { return p; } // Paper - OBFHELPER
private final WorldPersistentData q;
public WorldUpgrader(Convertable.ConversionSession convertable_conversionsession, DataFixer datafixer, ImmutableSet<ResourceKey<DimensionManager>> immutableset, boolean flag) { // CraftBukkit
diff --git a/src/main/java/net/minecraft/world/level/World.java b/src/main/java/net/minecraft/world/level/World.java
index 6581fe0d93a5c2e7b444a44c01726e02d4a28e63..f7f593a9e58b537109fa6ca1c783f6614f4bfad5 100644
--- a/src/main/java/net/minecraft/world/level/World.java
+++ b/src/main/java/net/minecraft/world/level/World.java
@@ -181,6 +181,15 @@ public abstract class World implements GeneratorAccess, AutoCloseable {
return typeKey;
}
+ // Paper start - fix and optimise world upgrading
+ // copied from below
+ public static ResourceKey<DimensionManager> getDimensionKey(DimensionManager manager) {
+ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().customRegistry.a().c(manager).orElseThrow(() -> {
+ return new IllegalStateException("Unregistered dimension type: " + manager);
+ });
+ }
+ // Paper end - fix and optimise world upgrading
+
protected World(WorldDataMutable worlddatamutable, ResourceKey<World> resourcekey, final DimensionManager dimensionmanager, Supplier<GameProfilerFiller> supplier, boolean flag, boolean flag1, long i, org.bukkit.generator.ChunkGenerator gen, org.bukkit.World.Environment env, java.util.concurrent.Executor executor) { // Paper
this.spigotConfig = new org.spigotmc.SpigotWorldConfig(((net.minecraft.world.level.storage.WorldDataServer) worlddatamutable).getName()); // Spigot
this.paperConfig = new com.destroystokyo.paper.PaperWorldConfig(((net.minecraft.world.level.storage.WorldDataServer) worlddatamutable).getName(), this.spigotConfig); // Paper
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/IChunkLoader.java b/src/main/java/net/minecraft/world/level/chunk/storage/IChunkLoader.java
index 890362d28ab9cb760c73fe5014e144fb08ada6b8..e20b9e6c46093d48d5fa5eb3006087d4e998c205 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/IChunkLoader.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/IChunkLoader.java
@@ -113,6 +113,7 @@ public class IChunkLoader implements AutoCloseable {
return nbttagcompound;
}
+ public static int getVersion(NBTTagCompound nbttagcompound) { return a(nbttagcompound); } // Paper - OBFHELPER
public static int a(NBTTagCompound nbttagcompound) {
return nbttagcompound.hasKeyOfType("DataVersion", 99) ? nbttagcompound.getInt("DataVersion") : -1;
}
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileCache.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileCache.java
index ebb0d6988f87013ea5d523ab4a1b31cb669ccc43..74d826853389b8e01ffe2b076cf2b179d29da216 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileCache.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileCache.java
@@ -30,6 +30,28 @@ public class RegionFileCache implements AutoCloseable { // Paper - no final
// Paper start
+ public static ChunkCoordIntPair getRegionFileCoordinates(File file) {
+ String fileName = file.getName();
+ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
+ return null;
+ }
+
+ String[] split = fileName.split("\\.");
+
+ if (split.length != 4) {
+ return null;
+ }
+
+ try {
+ int x = Integer.parseInt(split[1]);
+ int z = Integer.parseInt(split[2]);
+
+ return new ChunkCoordIntPair(x << 5, z << 5);
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+ }
+
public synchronized RegionFile getRegionFileIfLoaded(ChunkCoordIntPair chunkcoordintpair) { // Paper - synchronize for async io
return this.cache.getAndMoveToFirst(ChunkCoordIntPair.pair(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
}
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2021-05-30 00:56:24 +02:00
index 8209f188dbc932114268486a2ebd77df989a86ec..cebecee640ed5a7fc2b978e00ff7eb012228267d 100644
2021-05-21 02:31:08 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2021-05-30 00:56:24 +02:00
@@ -1146,14 +1146,7 @@ public final class CraftServer implements Server {
2021-05-21 02:31:08 +02:00
}
worlddata.checkName(name);
worlddata.a(console.getServerModName(), console.getModded().isPresent());
-
- if (console.options.has("forceUpgrade")) {
- net.minecraft.server.Main.convertWorld(worldSession, DataConverterRegistry.a(), console.options.has("eraseCache"), () -> {
- return true;
- }, worlddata.getGeneratorSettings().d().d().stream().map((entry) -> {
- return ResourceKey.a(IRegistry.K, ((ResourceKey) entry.getKey()).a());
- }).collect(ImmutableSet.toImmutableSet()));
- }
+ // Paper - move down
long j = BiomeManager.a(creator.seed());
List<MobSpawner> list = ImmutableList.of(new MobSpawnerPhantom(), new MobSpawnerPatrol(), new MobSpawnerCat(), new VillageSiege(), new MobSpawnerTrader(worlddata));
2021-05-30 00:56:24 +02:00
@@ -1170,6 +1163,14 @@ public final class CraftServer implements Server {
2021-05-21 10:29:49 +02:00
chunkgenerator = worlddimension.c();
}
+ // Paper start - fix and optimise world upgrading
+ if (console.options.has("forceUpgrade")) {
+ net.minecraft.server.Main.convertWorldButItWorks(
+ actualDimension, net.minecraft.world.level.World.getDimensionKey(dimensionmanager), worldSession.getLevelName(), DataConverterRegistry.getDataFixer(), console.options.has("eraseCache")
+ );
+ }
+ // Paper end - fix and optimise world upgrading
+
ResourceKey<net.minecraft.world.level.World> worldKey;
String levelName = this.getServer().getDedicatedServerProperties().levelName;
if (name.equals(levelName + "_nether")) {
