mirror of
https://github.com/PaperMC/Paper.git
synced 2024-11-23 11:06:29 +01:00
01a13871de
Patch documentation to come Issues with the old system that are fixed now: - World generation does not scale with cpu cores effectively. - Relies on the main thread for scheduling and maintaining chunk state, dropping chunk load/generate rates at lower tps. - Unreliable prioritisation of chunk gen/load calls that block the main thread. - Shutdown logic is utterly unreliable, as it has to wait for all chunks to unload - is it guaranteed that the chunk system is in a state on shutdown that it can reliably do this? Watchdog shutdown also typically failed due to thread checks, which is now resolved. - Saving of data is not unified (i.e can save chunk data without saving entity data, poses problems for desync if shutdown is really abnormal. - Entities are not loaded with chunks. This caused quite a bit of headache for Chunk#getEntities API, but now the new chunk system loads entities with chunks so that they are ready whenever the chunk loads in. Effectively brings the behavior back to 1.16 era, but still storing entities in their own separate regionfiles. The above list is not complete. The patch documentation will complete it. New chunk system hard relies on starlight and dataconverter, and most importantly the new concurrent utilities in ConcurrentUtil. Some of the old async chunk i/o interface (i.e the old file io thread reroutes _some_ calls to the new file io thread) is kept for plugin compat reasons. It will be removed in the next major version of minecraft. The old legacy chunk system patches have been moved to the removed folder in case we need them again.
390 lines
19 KiB
Diff
390 lines
19 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
|
|
Date: Thu, 20 May 2021 07:02:22 -0700
|
|
Subject: [PATCH] Fix and optimise world force upgrading
|
|
|
|
The WorldUpgrader class was incorrectly modified by
|
|
CB. It will store an IChunkLoader instance for all
|
|
dimension types in the world, but obviously with how
|
|
CB shifts around worlds only one dimension type exists
|
|
per world. But this would be OK if CB did this
|
|
change correctly. All IChunkLoader instances
|
|
will point to the same regionfiles. And all
|
|
IChunkLoader instances are going to be read from.
|
|
|
|
This problem hasn't really been reported because
|
|
it relies on the persistent legacy data to be converted
|
|
as well to cause corruption. Why? Because the legacy
|
|
data is also shared, it will result in different
|
|
outputs from conversion (as once conversion for legacy
|
|
persistent data takes place, it is REMOVED - so the next
|
|
convert will _not_ have the data). Which means different
|
|
sizes on disk. Which means different regionfile sector
|
|
allocations. Which means there are 3 different possible
|
|
regionfile sector allocations in memory, and none of them
|
|
are going to be correct.
|
|
|
|
I've fixed this by writing a world upgrader suited to
|
|
CB's changes to world folder format. It was brain dead
|
|
easy to add threading, so I did.
|
|
|
|
diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..95cac7edae8ac64811fc6a2f6b97dd4a0fceb0b0
|
|
--- /dev/null
|
|
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
|
|
@@ -0,0 +1,209 @@
|
|
+package io.papermc.paper.world;
|
|
+
|
|
+import com.mojang.datafixers.DataFixer;
|
|
+import com.mojang.serialization.Codec;
|
|
+import net.minecraft.SharedConstants;
|
|
+import net.minecraft.nbt.CompoundTag;
|
|
+import net.minecraft.resources.ResourceKey;
|
|
+import net.minecraft.util.worldupdate.WorldUpgrader;
|
|
+import net.minecraft.world.level.ChunkPos;
|
|
+import net.minecraft.world.level.Level;
|
|
+import net.minecraft.world.level.chunk.ChunkGenerator;
|
|
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
|
|
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
|
|
+import net.minecraft.world.level.dimension.DimensionType;
|
|
+import net.minecraft.world.level.dimension.LevelStem;
|
|
+import net.minecraft.world.level.levelgen.WorldGenSettings;
|
|
+import net.minecraft.world.level.storage.DimensionDataStorage;
|
|
+import net.minecraft.world.level.storage.LevelStorageSource;
|
|
+import org.apache.logging.log4j.LogManager;
|
|
+import org.apache.logging.log4j.Logger;
|
|
+import java.io.File;
|
|
+import java.io.IOException;
|
|
+import java.text.DecimalFormat;
|
|
+import java.util.Optional;
|
|
+import java.util.concurrent.ExecutorService;
|
|
+import java.util.concurrent.Executors;
|
|
+import java.util.concurrent.ThreadFactory;
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|
+import java.util.concurrent.atomic.AtomicLong;
|
|
+import java.util.function.Supplier;
|
|
+
|
|
+public class ThreadedWorldUpgrader {
|
|
+
|
|
+ private static final Logger LOGGER = LogManager.getLogger();
|
|
+
|
|
+ private final ResourceKey<LevelStem> dimensionType;
|
|
+ private final String worldName;
|
|
+ private final File worldDir;
|
|
+ private final ExecutorService threadPool;
|
|
+ private final DataFixer dataFixer;
|
|
+ private final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
|
|
+ private final boolean removeCaches;
|
|
+
|
|
+ public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final String worldName, final File worldDir, final int threads,
|
|
+ final DataFixer dataFixer, final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey, final boolean removeCaches) {
|
|
+ this.dimensionType = dimensionType;
|
|
+ this.worldName = worldName;
|
|
+ this.worldDir = worldDir;
|
|
+ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
|
|
+ private final AtomicInteger threadCounter = new AtomicInteger();
|
|
+
|
|
+ @Override
|
|
+ public Thread newThread(final Runnable run) {
|
|
+ final Thread ret = new Thread(run);
|
|
+
|
|
+ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
|
|
+ ret.setUncaughtExceptionHandler((thread, throwable) -> {
|
|
+ LOGGER.fatal("Error upgrading world", throwable);
|
|
+ });
|
|
+
|
|
+ return ret;
|
|
+ }
|
|
+ });
|
|
+ this.dataFixer = dataFixer;
|
|
+ this.generatorKey = generatorKey;
|
|
+ this.removeCaches = removeCaches;
|
|
+ }
|
|
+
|
|
+ public void convert() {
|
|
+ final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile();
|
|
+ final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer);
|
|
+
|
|
+ final File regionFolder = new File(worldFolder, "region");
|
|
+
|
|
+ LOGGER.info("Force upgrading " + this.worldName);
|
|
+ LOGGER.info("Counting regionfiles for " + this.worldName);
|
|
+ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
|
|
+ return WorldUpgrader.REGEX.matcher(name).matches();
|
|
+ });
|
|
+ if (regionFiles == null) {
|
|
+ LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
|
|
+ return;
|
|
+ }
|
|
+ LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
|
|
+ LOGGER.info("Starting conversion now for world " + this.worldName);
|
|
+
|
|
+ final WorldInfo info = new WorldInfo(() -> worldPersistentData,
|
|
+ new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey);
|
|
+
|
|
+ long expectedChunks = (long)regionFiles.length * (32L * 32L);
|
|
+
|
|
+ for (final File regionFile : regionFiles) {
|
|
+ final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath());
|
|
+ if (regionPos == null) {
|
|
+ expectedChunks -= (32L * 32L);
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
|
|
+ }
|
|
+ this.threadPool.shutdown();
|
|
+
|
|
+ final DecimalFormat format = new DecimalFormat("#0.00");
|
|
+
|
|
+ final long start = System.nanoTime();
|
|
+
|
|
+ while (!this.threadPool.isTerminated()) {
|
|
+ final long current = info.convertedChunks.get();
|
|
+
|
|
+ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
|
|
+
|
|
+ try {
|
|
+ Thread.sleep(1000L);
|
|
+ } catch (final InterruptedException ignore) {}
|
|
+ }
|
|
+
|
|
+ final long end = System.nanoTime();
|
|
+
|
|
+ try {
|
|
+ info.loader.close();
|
|
+ } catch (final IOException ex) {
|
|
+ LOGGER.fatal("Failed to close chunk loader", ex);
|
|
+ }
|
|
+ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
|
|
+ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
|
|
+ }
|
|
+
|
|
+ private static final class WorldInfo {
|
|
+
|
|
+ public final Supplier<DimensionDataStorage> persistentDataSupplier;
|
|
+ public final ChunkStorage loader;
|
|
+ public final boolean removeCaches;
|
|
+ public final ResourceKey<LevelStem> worldKey;
|
|
+ public final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
|
|
+ public final AtomicLong convertedChunks = new AtomicLong();
|
|
+ public final AtomicLong modifiedChunks = new AtomicLong();
|
|
+
|
|
+ private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches,
|
|
+ final ResourceKey<LevelStem> worldKey, Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey) {
|
|
+ this.persistentDataSupplier = persistentDataSupplier;
|
|
+ this.loader = loader;
|
|
+ this.removeCaches = removeCaches;
|
|
+ this.worldKey = worldKey;
|
|
+ this.generatorKey = generatorKey;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static final class ConvertTask implements Runnable {
|
|
+
|
|
+ private final WorldInfo worldInfo;
|
|
+ private final int regionX;
|
|
+ private final int regionZ;
|
|
+
|
|
+ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
|
|
+ this.worldInfo = worldInfo;
|
|
+ this.regionX = regionX;
|
|
+ this.regionZ = regionZ;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public void run() {
|
|
+ final int regionCX = this.regionX << 5;
|
|
+ final int regionCZ = this.regionZ << 5;
|
|
+
|
|
+ final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
|
|
+ final ChunkStorage loader = this.worldInfo.loader;
|
|
+ final boolean removeCaches = this.worldInfo.removeCaches;
|
|
+ final ResourceKey<LevelStem> worldKey = this.worldInfo.worldKey;
|
|
+
|
|
+ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
|
|
+ for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
|
|
+ final ChunkPos chunkPos = new ChunkPos(cx, cz);
|
|
+ try {
|
|
+ // no need to check the coordinate of the chunk, the regionfilecache does that for us
|
|
+
|
|
+ CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null);
|
|
+
|
|
+ if (chunkNBT == null) {
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ final int versionBefore = ChunkStorage.getVersion(chunkNBT);
|
|
+
|
|
+ chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null);
|
|
+
|
|
+ boolean modified = versionBefore < SharedConstants.getCurrentVersion().getWorldVersion();
|
|
+
|
|
+ if (removeCaches) {
|
|
+ final CompoundTag level = chunkNBT.getCompound("Level");
|
|
+ modified |= level.contains("Heightmaps");
|
|
+ level.remove("Heightmaps");
|
|
+ modified |= level.contains("isLightOn");
|
|
+ level.remove("isLightOn");
|
|
+ }
|
|
+
|
|
+ if (modified) {
|
|
+ this.worldInfo.modifiedChunks.getAndIncrement();
|
|
+ loader.write(chunkPos, chunkNBT);
|
|
+ }
|
|
+ } catch (final Exception ex) {
|
|
+ LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
|
|
+ } finally {
|
|
+ this.worldInfo.convertedChunks.getAndIncrement();
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
|
|
index 2f82002c52af7304ff6b2d6fe8f094314daf0bba..5962f7a2b185d7d54a0f9e341a4fdf6e6f1c1ec5 100644
|
|
--- a/src/main/java/net/minecraft/server/Main.java
|
|
+++ b/src/main/java/net/minecraft/server/Main.java
|
|
@@ -16,6 +16,7 @@ import java.util.Objects;
|
|
import java.util.Optional;
|
|
import java.util.UUID;
|
|
import java.util.function.BooleanSupplier;
|
|
+import io.papermc.paper.world.ThreadedWorldUpgrader;
|
|
import joptsimple.NonOptionArgumentSpec;
|
|
import joptsimple.OptionParser;
|
|
import joptsimple.OptionSet;
|
|
@@ -314,6 +315,15 @@ public class Main {
|
|
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey<net.minecraft.world.level.dimension.LevelStem> dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession,
|
|
+ DataFixer dataFixer, Optional<net.minecraft.resources.ResourceKey<com.mojang.serialization.Codec<? extends net.minecraft.world.level.chunk.ChunkGenerator>>> generatorKey, boolean removeCaches) {
|
|
+ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
|
|
+ final ThreadedWorldUpgrader worldUpgrader = new ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, generatorKey, removeCaches);
|
|
+ worldUpgrader.convert();
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
+
|
|
public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, WorldGenSettings generatorOptions) {
|
|
Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit
|
|
WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, generatorOptions, eraseCache);
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 6758b78d1d780de3fecd6818f0ef83855fb82eef..a61a0a61ee5c96de68521ae3d01fd9f75526502c 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -545,11 +545,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
worlddata = new PrimaryLevelData(worldsettings, generatorsettings, Lifecycle.stable());
|
|
}
|
|
worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
|
|
- if (this.options.has("forceUpgrade")) {
|
|
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.options.has("eraseCache"), () -> {
|
|
- return true;
|
|
- }, worlddata.worldGenSettings());
|
|
- }
|
|
+ // Paper - move down
|
|
|
|
PrimaryLevelData iworlddataserver = worlddata;
|
|
WorldGenSettings generatorsettings = worlddata.worldGenSettings();
|
|
@@ -564,6 +560,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
biomeProvider = gen.getDefaultBiomeProvider(worldInfo);
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ if (options.has("forceUpgrade")) {
|
|
+ net.minecraft.server.Main.convertWorldButItWorks(
|
|
+ dimensionKey, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), options.has("eraseCache")
|
|
+ );
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
ResourceKey<Level> worldKey = ResourceKey.create(Registry.DIMENSION_REGISTRY, dimensionKey.location());
|
|
|
|
if (dimensionKey == LevelStem.OVERWORLD) {
|
|
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
|
|
index 1fcbf1fe2ed5d46766ae71e556f124e7822829e3..15050e9bfbb5c5639a8b225a9beb3834aab4eafb 100644
|
|
--- a/src/main/java/net/minecraft/world/level/Level.java
|
|
+++ b/src/main/java/net/minecraft/world/level/Level.java
|
|
@@ -182,6 +182,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
|
|
public final Map<Explosion.CacheKey, Float> explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions
|
|
public java.util.ArrayDeque<net.minecraft.world.level.block.RedstoneTorchBlock.Toggle> redstoneUpdateInfos; // Paper - Move from Map in BlockRedstoneTorch to here
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ // copied from below
|
|
+ public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) {
|
|
+ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryHolder.ownedRegistryOrThrow(net.minecraft.core.Registry.DIMENSION_TYPE_REGISTRY).getResourceKey(manager).orElseThrow(() -> {
|
|
+ return new IllegalStateException("Unregistered dimension type: " + manager);
|
|
+ });
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
+
|
|
public CraftWorld getWorld() {
|
|
return this.world;
|
|
}
|
|
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
index 7bfb0716964af5ee300150d500c97e8f90c849d4..c881d15efa94fc379ed2817a534fef3a4dd3d90d 100644
|
|
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
@@ -32,6 +32,28 @@ public class RegionFileStorage implements AutoCloseable {
|
|
}
|
|
|
|
// Paper start
|
|
+ public static @Nullable ChunkPos getRegionFileCoordinates(Path file) {
|
|
+ String fileName = file.getFileName().toString();
|
|
+ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
|
|
+ return null;
|
|
+ }
|
|
+
|
|
+ String[] split = fileName.split("\\.");
|
|
+
|
|
+ if (split.length != 4) {
|
|
+ return null;
|
|
+ }
|
|
+
|
|
+ try {
|
|
+ int x = Integer.parseInt(split[1]);
|
|
+ int z = Integer.parseInt(split[2]);
|
|
+
|
|
+ return new ChunkPos(x << 5, z << 5);
|
|
+ } catch (NumberFormatException ex) {
|
|
+ return null;
|
|
+ }
|
|
+ }
|
|
+
|
|
public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) {
|
|
return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
|
|
}
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
index ab8ab0180508948c7b21bafcbf914483673047d3..60224fce02148ac7e8a28524b8769372eff98817 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
@@ -1204,12 +1204,7 @@ public final class CraftServer implements Server {
|
|
}
|
|
worlddata.checkName(name);
|
|
worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified());
|
|
-
|
|
- if (console.options.has("forceUpgrade")) {
|
|
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), console.options.has("eraseCache"), () -> {
|
|
- return true;
|
|
- }, worlddata.worldGenSettings());
|
|
- }
|
|
+ // Paper - move down
|
|
|
|
long j = BiomeManager.obfuscateSeed(creator.seed());
|
|
List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata));
|
|
@@ -1221,6 +1216,13 @@ public final class CraftServer implements Server {
|
|
biomeProvider = generator.getDefaultBiomeProvider(worldInfo);
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ if (console.options.has("forceUpgrade")) {
|
|
+ net.minecraft.server.Main.convertWorldButItWorks(
|
|
+ actualDimension, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), console.options.has("eraseCache")
|
|
+ );
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
ResourceKey<net.minecraft.world.level.Level> worldKey;
|
|
String levelName = this.getServer().getProperties().levelName;
|
|
if (name.equals(levelName + "_nether")) {
|