Paper/patches/server/0621-Fix-and-optimise-world-force-upgrading.patch

393 lines
20 KiB
Diff
Raw Normal View History

2021-06-11 14:02:28 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Thu, 20 May 2021 07:02:22 -0700
Subject: [PATCH] Fix and optimise world force upgrading
The WorldUpgrader class was incorrectly modified by
CB. It will store an IChunkLoader instance for all
dimension types in the world, but obviously with how
CB shifts around worlds only one dimension type exists
per world. But this would be OK if CB did this
change correctly. All IChunkLoader instances
will point to the same regionfiles. And all
IChunkLoader instances are going to be read from.
This problem hasn't really been reported because
it relies on the persistent legacy data to be converted
as well to cause corruption. Why? Because the legacy
data is also shared, it will result in different
outputs from conversion (as once conversion for legacy
persistent data takes place, it is REMOVED - so the next
convert will _not_ have the data). Which means different
sizes on disk. Which means different regionfile sector
allocations. Which means there are 3 different possible
regionfile sector allocations in memory, and none of them
are going to be correct.
I've fixed this by writing a world upgrader suited to
CB's changes to world folder format. It was brain dead
easy to add threading, so I did.
== AT ==
public net.minecraft.util.worldupdate.WorldUpgrader REGEX
2021-06-11 14:02:28 +02:00
diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
new file mode 100644
2022-06-08 16:24:55 +02:00
index 0000000000000000000000000000000000000000..95cac7edae8ac64811fc6a2f6b97dd4a0fceb0b0
2021-06-11 14:02:28 +02:00
--- /dev/null
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
2022-02-28 22:43:31 +01:00
@@ -0,0 +1,209 @@
2021-06-11 14:02:28 +02:00
+package io.papermc.paper.world;
+
+import com.mojang.datafixers.DataFixer;
2021-11-24 22:30:53 +01:00
+import com.mojang.serialization.Codec;
2021-06-11 14:02:28 +02:00
+import net.minecraft.SharedConstants;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.resources.ResourceKey;
+import net.minecraft.util.worldupdate.WorldUpgrader;
+import net.minecraft.world.level.ChunkPos;
2021-11-24 22:30:53 +01:00
+import net.minecraft.world.level.Level;
+import net.minecraft.world.level.chunk.ChunkGenerator;
2021-06-11 14:02:28 +02:00
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
+import net.minecraft.world.level.dimension.DimensionType;
+import net.minecraft.world.level.dimension.LevelStem;
2021-11-24 22:30:53 +01:00
+import net.minecraft.world.level.levelgen.WorldGenSettings;
2021-06-11 14:02:28 +02:00
+import net.minecraft.world.level.storage.DimensionDataStorage;
+import net.minecraft.world.level.storage.LevelStorageSource;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.text.DecimalFormat;
2021-11-24 22:30:53 +01:00
+import java.util.Optional;
2021-06-11 14:02:28 +02:00
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+public class ThreadedWorldUpgrader {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final ResourceKey<LevelStem> dimensionType;
+ private final String worldName;
+ private final File worldDir;
2021-06-11 14:02:28 +02:00
+ private final ExecutorService threadPool;
+ private final DataFixer dataFixer;
2021-11-24 22:30:53 +01:00
+ private final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
2021-06-11 14:02:28 +02:00
+ private final boolean removeCaches;
+
2022-02-28 22:43:31 +01:00
+ public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final String worldName, final File worldDir, final int threads,
2021-11-24 22:30:53 +01:00
+ final DataFixer dataFixer, final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey, final boolean removeCaches) {
2021-06-11 14:02:28 +02:00
+ this.dimensionType = dimensionType;
+ this.worldName = worldName;
+ this.worldDir = worldDir;
2021-06-11 14:02:28 +02:00
+ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
+ private final AtomicInteger threadCounter = new AtomicInteger();
+
+ @Override
+ public Thread newThread(final Runnable run) {
+ final Thread ret = new Thread(run);
+
+ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
+ ret.setUncaughtExceptionHandler((thread, throwable) -> {
+ LOGGER.fatal("Error upgrading world", throwable);
+ });
+
+ return ret;
+ }
+ });
+ this.dataFixer = dataFixer;
2021-11-24 22:30:53 +01:00
+ this.generatorKey = generatorKey;
2021-06-11 14:02:28 +02:00
+ this.removeCaches = removeCaches;
+ }
+
+ public void convert() {
2021-11-24 22:30:53 +01:00
+ final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile();
2021-06-11 14:02:28 +02:00
+ final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer);
+
+ final File regionFolder = new File(worldFolder, "region");
+
+ LOGGER.info("Force upgrading " + this.worldName);
+ LOGGER.info("Counting regionfiles for " + this.worldName);
+ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
2021-06-15 04:59:31 +02:00
+ return WorldUpgrader.REGEX.matcher(name).matches();
2021-06-11 14:02:28 +02:00
+ });
+ if (regionFiles == null) {
+ LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
+ return;
+ }
+ LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
+ LOGGER.info("Starting conversion now for world " + this.worldName);
+
+ final WorldInfo info = new WorldInfo(() -> worldPersistentData,
2021-11-24 22:30:53 +01:00
+ new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey);
2021-06-11 14:02:28 +02:00
+
+ long expectedChunks = (long)regionFiles.length * (32L * 32L);
+
+ for (final File regionFile : regionFiles) {
2021-11-24 22:30:53 +01:00
+ final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath());
2021-06-11 14:02:28 +02:00
+ if (regionPos == null) {
+ expectedChunks -= (32L * 32L);
+ continue;
+ }
+
+ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
+ }
+ this.threadPool.shutdown();
+
+ final DecimalFormat format = new DecimalFormat("#0.00");
+
+ final long start = System.nanoTime();
+
+ while (!this.threadPool.isTerminated()) {
+ final long current = info.convertedChunks.get();
+
+ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
+
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ignore) {}
+ }
+
+ final long end = System.nanoTime();
+
+ try {
+ info.loader.close();
+ } catch (final IOException ex) {
+ LOGGER.fatal("Failed to close chunk loader", ex);
+ }
+ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
+ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
+ }
+
+ private static final class WorldInfo {
+
+ public final Supplier<DimensionDataStorage> persistentDataSupplier;
+ public final ChunkStorage loader;
+ public final boolean removeCaches;
2021-11-24 22:30:53 +01:00
+ public final ResourceKey<LevelStem> worldKey;
+ public final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
2021-06-11 14:02:28 +02:00
+ public final AtomicLong convertedChunks = new AtomicLong();
+ public final AtomicLong modifiedChunks = new AtomicLong();
+
+ private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches,
2021-11-24 22:30:53 +01:00
+ final ResourceKey<LevelStem> worldKey, Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey) {
2021-06-11 14:02:28 +02:00
+ this.persistentDataSupplier = persistentDataSupplier;
+ this.loader = loader;
+ this.removeCaches = removeCaches;
+ this.worldKey = worldKey;
2021-11-24 22:30:53 +01:00
+ this.generatorKey = generatorKey;
2021-06-11 14:02:28 +02:00
+ }
+ }
+
+ private static final class ConvertTask implements Runnable {
+
+ private final WorldInfo worldInfo;
+ private final int regionX;
+ private final int regionZ;
+
+ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
+ this.worldInfo = worldInfo;
+ this.regionX = regionX;
+ this.regionZ = regionZ;
+ }
+
+ @Override
+ public void run() {
+ final int regionCX = this.regionX << 5;
+ final int regionCZ = this.regionZ << 5;
+
+ final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
+ final ChunkStorage loader = this.worldInfo.loader;
+ final boolean removeCaches = this.worldInfo.removeCaches;
2021-11-24 22:30:53 +01:00
+ final ResourceKey<LevelStem> worldKey = this.worldInfo.worldKey;
2021-06-11 14:02:28 +02:00
+
+ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
+ for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
+ final ChunkPos chunkPos = new ChunkPos(cx, cz);
+ try {
+ // no need to check the coordinate of the chunk, the regionfilecache does that for us
+
2022-06-08 16:24:55 +02:00
+ CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null);
2021-06-11 14:02:28 +02:00
+
+ if (chunkNBT == null) {
+ continue;
+ }
+
+ final int versionBefore = ChunkStorage.getVersion(chunkNBT);
+
2021-11-24 22:30:53 +01:00
+ chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null);
2021-06-11 14:02:28 +02:00
+
+ boolean modified = versionBefore < SharedConstants.getCurrentVersion().getWorldVersion();
+
+ if (removeCaches) {
+ final CompoundTag level = chunkNBT.getCompound("Level");
+ modified |= level.contains("Heightmaps");
+ level.remove("Heightmaps");
+ modified |= level.contains("isLightOn");
+ level.remove("isLightOn");
+ }
+
+ if (modified) {
+ this.worldInfo.modifiedChunks.getAndIncrement();
+ loader.write(chunkPos, chunkNBT);
+ }
+ } catch (final Exception ex) {
+ LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
+ } finally {
+ this.worldInfo.convertedChunks.getAndIncrement();
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
2023-03-14 21:25:13 +01:00
index 13e5362e12eb857b0cc27277c95efe085e77348d..1619da8a904785e0cc00ddbcfba1c3fa2b8d9b93 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
2023-03-14 21:25:13 +01:00
@@ -18,6 +18,7 @@ import java.nio.file.Paths;
2021-06-11 14:02:28 +02:00
import java.util.Optional;
2022-06-08 11:31:06 +02:00
import java.util.UUID;
2021-06-11 14:02:28 +02:00
import java.util.function.BooleanSupplier;
+import io.papermc.paper.world.ThreadedWorldUpgrader;
import joptsimple.NonOptionArgumentSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
2023-03-14 21:25:13 +01:00
@@ -351,6 +352,15 @@ public class Main {
2022-12-07 21:16:54 +01:00
return new WorldLoader.InitConfig(worldloader_d, Commands.CommandSelection.DEDICATED, serverPropertiesHandler.functionPermissionLevel);
2021-06-11 14:02:28 +02:00
}
+ // Paper start - fix and optimise world upgrading
2022-02-28 22:43:31 +01:00
+ public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey<net.minecraft.world.level.dimension.LevelStem> dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession,
2021-11-24 22:30:53 +01:00
+ DataFixer dataFixer, Optional<net.minecraft.resources.ResourceKey<com.mojang.serialization.Codec<? extends net.minecraft.world.level.chunk.ChunkGenerator>>> generatorKey, boolean removeCaches) {
2021-06-11 14:02:28 +02:00
+ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
2022-06-08 14:19:54 +02:00
+ final ThreadedWorldUpgrader worldUpgrader = new ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, generatorKey, removeCaches);
2021-06-11 14:02:28 +02:00
+ worldUpgrader.convert();
+ }
+ // Paper end - fix and optimise world upgrading
+
2022-12-07 21:16:54 +01:00
public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, Registry<LevelStem> dimensionOptionsRegistry) {
2021-06-11 14:02:28 +02:00
Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit
2022-12-07 21:16:54 +01:00
WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, dimensionOptionsRegistry, eraseCache);
2021-06-11 14:02:28 +02:00
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
2023-03-14 21:25:13 +01:00
index ff97f603d9962dba69f1ae06d2a25986d2354927..7e4e484539ad705d61cd0d1b4db49e31fe3ca8f7 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
2023-03-14 21:25:13 +01:00
@@ -553,11 +553,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2022-12-07 21:16:54 +01:00
worlddata = new PrimaryLevelData(worldsettings, worldoptions, worlddimensions_b.specialWorldProperty(), lifecycle);
2021-06-11 14:02:28 +02:00
}
worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
2021-06-15 04:59:31 +02:00
- if (this.options.has("forceUpgrade")) {
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.options.has("eraseCache"), () -> {
2021-06-11 14:02:28 +02:00
- return true;
2022-12-07 21:16:54 +01:00
- }, iregistry);
2021-06-11 14:02:28 +02:00
- }
+ // Paper - move down
2022-06-08 11:31:06 +02:00
PrimaryLevelData iworlddataserver = worlddata;
2022-12-07 21:16:54 +01:00
boolean flag = worlddata.isDebugWorld();
2023-03-14 21:25:13 +01:00
@@ -572,6 +568,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2022-06-08 11:31:06 +02:00
biomeProvider = gen.getDefaultBiomeProvider(worldInfo);
2021-06-11 14:02:28 +02:00
}
+ // Paper start - fix and optimise world upgrading
+ if (options.has("forceUpgrade")) {
+ net.minecraft.server.Main.convertWorldButItWorks(
2022-06-08 11:31:06 +02:00
+ dimensionKey, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), options.has("eraseCache")
2021-06-11 14:02:28 +02:00
+ );
+ }
+ // Paper end - fix and optimise world upgrading
2022-12-07 21:16:54 +01:00
ResourceKey<Level> worldKey = ResourceKey.create(Registries.DIMENSION, dimensionKey.location());
2021-06-11 14:02:28 +02:00
if (dimensionKey == LevelStem.OVERWORLD) {
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
2023-03-14 21:25:13 +01:00
index 1770c97a3190a89a9b1dbde126b94a42662efea0..8a6f01af6d9a210fbd5cf14002c426c2e36fc51b 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
2023-03-14 21:25:13 +01:00
@@ -185,6 +185,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
2021-06-15 15:20:52 +02:00
public final Map<Explosion.CacheKey, Float> explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions
public java.util.ArrayDeque<net.minecraft.world.level.block.RedstoneTorchBlock.Toggle> redstoneUpdateInfos; // Paper - Move from Map in BlockRedstoneTorch to here
2021-06-11 14:02:28 +02:00
+ // Paper start - fix and optimise world upgrading
+ // copied from below
+ public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) {
2022-12-08 04:24:00 +01:00
+ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryAccess().registryOrThrow(net.minecraft.core.registries.Registries.DIMENSION_TYPE).getResourceKey(manager).orElseThrow(() -> {
2021-06-11 14:02:28 +02:00
+ return new IllegalStateException("Unregistered dimension type: " + manager);
+ });
+ }
+ // Paper end - fix and optimise world upgrading
2021-06-15 15:20:52 +02:00
+
public CraftWorld getWorld() {
return this.world;
}
2021-06-11 14:02:28 +02:00
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
2022-12-07 21:16:54 +01:00
index b294ef87fb93e7f4651dc04128124f297575860d..65fd57609e45ccd49ebfc1ba80d25243da13ab6e 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
2021-11-24 22:30:53 +01:00
@@ -32,6 +32,28 @@ public class RegionFileStorage implements AutoCloseable {
2021-06-15 04:59:31 +02:00
}
2021-06-11 14:02:28 +02:00
// Paper start
2021-11-24 22:30:53 +01:00
+ public static @Nullable ChunkPos getRegionFileCoordinates(Path file) {
+ String fileName = file.getFileName().toString();
2021-06-11 14:02:28 +02:00
+ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
+ return null;
+ }
+
+ String[] split = fileName.split("\\.");
+
+ if (split.length != 4) {
+ return null;
+ }
+
+ try {
+ int x = Integer.parseInt(split[1]);
+ int z = Integer.parseInt(split[2]);
+
+ return new ChunkPos(x << 5, z << 5);
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+ }
+
2021-06-15 04:59:31 +02:00
public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) {
2021-06-11 14:02:28 +02:00
return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
}
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2023-03-14 21:25:13 +01:00
index a8116749fb7a2b30e2623165eedb9ee167ee6120..29d8b20d1e90b0097a98f454f2f1209b27fdfa61 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2023-03-14 21:25:13 +01:00
@@ -1210,12 +1210,7 @@ public final class CraftServer implements Server {
worlddata.customDimensions = iregistry;
2021-06-11 14:02:28 +02:00
worlddata.checkName(name);
2021-11-24 22:30:53 +01:00
worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified());
2021-06-11 14:02:28 +02:00
-
- if (console.options.has("forceUpgrade")) {
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), console.options.has("eraseCache"), () -> {
- return true;
2022-12-07 21:16:54 +01:00
- }, iregistry);
2021-06-11 14:02:28 +02:00
- }
+ // Paper - move down
long j = BiomeManager.obfuscateSeed(creator.seed());
List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata));
2023-03-14 21:25:13 +01:00
@@ -1226,6 +1221,13 @@ public final class CraftServer implements Server {
2022-06-08 11:31:06 +02:00
biomeProvider = generator.getDefaultBiomeProvider(worldInfo);
2021-06-11 14:02:28 +02:00
}
+ // Paper start - fix and optimise world upgrading
+ if (console.options.has("forceUpgrade")) {
+ net.minecraft.server.Main.convertWorldButItWorks(
2022-06-08 11:31:06 +02:00
+ actualDimension, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), console.options.has("eraseCache")
2021-06-11 14:02:28 +02:00
+ );
+ }
+ // Paper end - fix and optimise world upgrading
ResourceKey<net.minecraft.world.level.Level> worldKey;
String levelName = this.getServer().getProperties().levelName;
if (name.equals(levelName + "_nether")) {