From 97b1cc361b8ce14e6ad45d3c84e997613a90722b Mon Sep 17 00:00:00 2001 From: Aikar Date: Sun, 12 Apr 2020 15:56:03 -0400 Subject: [PATCH] Allow shutting down server during a watchdog hang gracefully If the request to shut down the server is received while we are in a watchdog hang, immediately treat it as a crash and begin the shutdown process. Shutdown process is now improved to also shutdown cleanly when not using restart scripts either. If a server is deadlocked, a server owner can send SIGUP (or any other signal the JVM understands to shut down as it currently does) and the watchdog will no longer need to wait until the full timeout, allowing you to trigger a close process and try to shut the server down gracefully, saving player and world data. Previously there was no way to trigger this outside of waiting for a full watchdog timeout, which may be set to a really long time... --- ...91-Asynchronous-chunk-IO-and-loading.patch | 64 +++++++------- ...own-server-during-a-watchdog-hang-gr.patch | 87 +++++++++++++++++++ 2 files changed, 119 insertions(+), 32 deletions(-) create mode 100644 Spigot-Server-Patches/0469-Allow-shutting-down-server-during-a-watchdog-hang-gr.patch diff --git a/Spigot-Server-Patches/0391-Asynchronous-chunk-IO-and-loading.patch b/Spigot-Server-Patches/0391-Asynchronous-chunk-IO-and-loading.patch index fbfef29282..309cd900da 100644 --- a/Spigot-Server-Patches/0391-Asynchronous-chunk-IO-and-loading.patch +++ b/Spigot-Server-Patches/0391-Asynchronous-chunk-IO-and-loading.patch @@ -1,4 +1,4 @@ -From 6e4b6c3681b093e95db973304a8cd0b04880cb76 Mon Sep 17 00:00:00 2001 +From d8323c167f2f692f00b4396f04b8ea4af7b2f3ac Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Sat, 13 Jul 2019 09:23:10 -0700 Subject: [PATCH] Asynchronous chunk IO and loading @@ -121,7 +121,7 @@ tasks required to be executed by the chunk load task (i.e lighting and some poi tasks). diff --git a/src/main/java/co/aikar/timings/WorldTimingsHandler.java b/src/main/java/co/aikar/timings/WorldTimingsHandler.java -index 27ce4a828e..30bafb214b 100644 +index 27ce4a828..30bafb214 100644 --- a/src/main/java/co/aikar/timings/WorldTimingsHandler.java +++ b/src/main/java/co/aikar/timings/WorldTimingsHandler.java @@ -61,6 +61,17 @@ public class WorldTimingsHandler { @@ -161,7 +161,7 @@ index 27ce4a828e..30bafb214b 100644 public static Timing getTickList(WorldServer worldserver, String timingsType) { diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java -index dbd1439970..6916ed30c4 100644 +index dbd143997..6916ed30c 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -1,5 +1,6 @@ @@ -237,7 +237,7 @@ index dbd1439970..6916ed30c4 100644 + } } diff --git a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java -index 23626bef3a..1edcecd2ee 100644 +index 23626bef3..1edcecd2e 100644 --- a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java +++ b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java @@ -9,6 +9,7 @@ import java.util.concurrent.Executors; @@ -318,7 +318,7 @@ index 23626bef3a..1edcecd2ee 100644 diff --git a/src/main/java/com/destroystokyo/paper/io/IOUtil.java b/src/main/java/com/destroystokyo/paper/io/IOUtil.java new file mode 100644 -index 0000000000..5af0ac3d9e +index 000000000..5af0ac3d9 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/IOUtil.java @@ -0,0 +1,62 @@ @@ -386,7 +386,7 @@ index 0000000000..5af0ac3d9e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java new file mode 100644 -index 0000000000..4f10a8311e +index 000000000..4f10a8311 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java @@ -0,0 +1,661 @@ @@ -1053,7 +1053,7 @@ index 0000000000..4f10a8311e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java new file mode 100644 -index 0000000000..97f2e433c4 +index 000000000..97f2e433c --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java @@ -0,0 +1,277 @@ @@ -1336,7 +1336,7 @@ index 0000000000..97f2e433c4 +} diff --git a/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java new file mode 100644 -index 0000000000..ee906b594b +index 000000000..ee906b594 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java @@ -0,0 +1,241 @@ @@ -1583,7 +1583,7 @@ index 0000000000..ee906b594b +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java new file mode 100644 -index 0000000000..305da47868 +index 000000000..305da4786 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java @@ -0,0 +1,149 @@ @@ -1738,7 +1738,7 @@ index 0000000000..305da47868 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java new file mode 100644 -index 0000000000..60312b85f9 +index 000000000..60312b85f --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java @@ -0,0 +1,112 @@ @@ -1856,7 +1856,7 @@ index 0000000000..60312b85f9 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java new file mode 100644 -index 0000000000..1dfa8abfd8 +index 000000000..1dfa8abfd --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java @@ -0,0 +1,40 @@ @@ -1902,7 +1902,7 @@ index 0000000000..1dfa8abfd8 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java new file mode 100644 -index 0000000000..0745a2015a +index 000000000..2b20c159f --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java @@ -0,0 +1,492 @@ @@ -2283,7 +2283,7 @@ index 0000000000..0745a2015a + drainChunkWaitQueue(); + + if (this.workers == null) { -+ if (Bukkit.isPrimaryThread()) { ++ if (Bukkit.isPrimaryThread() || MinecraftServer.getServer().hasStopped()) { + ((IAsyncTaskHandler)this.world.getChunkProvider().serverThreadQueue).executeAll(); + } else { + CompletableFuture wait = new CompletableFuture<>(); @@ -2399,7 +2399,7 @@ index 0000000000..0745a2015a + +} diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java -index b582171c51..03d7ce8294 100644 +index b582171c5..03d7ce829 100644 --- a/src/main/java/net/minecraft/server/ChunkProviderServer.java +++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java @@ -218,11 +218,137 @@ public class ChunkProviderServer extends IChunkProvider { @@ -2569,7 +2569,7 @@ index b582171c51..03d7ce8294 100644 } finally { playerChunkMap.callbackExecutor.run(); diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java -index a950ad801d..26f1a4b095 100644 +index a950ad801..26f1a4b09 100644 --- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java +++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java @@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet; @@ -2838,7 +2838,7 @@ index a950ad801d..26f1a4b095 100644 nbttagcompound1.set("PostProcessing", a(ichunkaccess.l())); diff --git a/src/main/java/net/minecraft/server/ChunkStatus.java b/src/main/java/net/minecraft/server/ChunkStatus.java -index 134a4f0b7d..40ce30cdc2 100644 +index 134a4f0b7..40ce30cdc 100644 --- a/src/main/java/net/minecraft/server/ChunkStatus.java +++ b/src/main/java/net/minecraft/server/ChunkStatus.java @@ -153,6 +153,7 @@ public class ChunkStatus { @@ -2874,7 +2874,7 @@ index 134a4f0b7d..40ce30cdc2 100644 return this.c() >= chunkstatus.c(); } diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java -index 7e5ece9d50..cfe43e882e 100644 +index 7e5ece9d5..cfe43e882 100644 --- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java +++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java @@ -91,7 +91,7 @@ public abstract class IAsyncTaskHandler implements Mailbox { @@ -3898,7 +3898,7 @@ index 1d1b267f32..4b87ca2ecb 100644 public static TicketType a(String s, Comparator comparator) { return new TicketType<>(s, comparator, 0L); diff --git a/src/main/java/net/minecraft/server/VillagePlace.java b/src/main/java/net/minecraft/server/VillagePlace.java -index c999f8c9bf..b59ef1a633 100644 +index c999f8c9b..b59ef1a63 100644 --- a/src/main/java/net/minecraft/server/VillagePlace.java +++ b/src/main/java/net/minecraft/server/VillagePlace.java @@ -24,8 +24,16 @@ public class VillagePlace extends RegionFileSection { @@ -3987,7 +3987,7 @@ index c999f8c9bf..b59ef1a633 100644 HAS_SPACE(VillagePlaceRecord::d), IS_OCCUPIED(VillagePlaceRecord::e), ANY((villageplacerecord) -> { diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java -index df7503a5ec..d4ef2403d5 100644 +index df7503a5e..d4ef2403d 100644 --- a/src/main/java/net/minecraft/server/WorldServer.java +++ b/src/main/java/net/minecraft/server/WorldServer.java @@ -82,6 +82,79 @@ public class WorldServer extends World { @@ -4080,7 +4080,7 @@ index df7503a5ec..d4ef2403d5 100644 // CraftBukkit start diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java -index a71bb86508..1d275520fb 100644 +index a71bb8650..1d275520f 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -554,22 +554,23 @@ public class CraftWorld implements World { @@ -4141,7 +4141,7 @@ index a71bb86508..1d275520fb 100644 // Spigot start @Override diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java -index 07936eeba2..5bdcdcf9e8 100644 +index 07936eeba..5bdcdcf9e 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -6,6 +6,7 @@ import java.lang.management.ThreadInfo; diff --git a/Spigot-Server-Patches/0469-Allow-shutting-down-server-during-a-watchdog-hang-gr.patch b/Spigot-Server-Patches/0469-Allow-shutting-down-server-during-a-watchdog-hang-gr.patch new file mode 100644 index 0000000000..9855d192e9 --- /dev/null +++ b/Spigot-Server-Patches/0469-Allow-shutting-down-server-during-a-watchdog-hang-gr.patch @@ -0,0 +1,87 @@ +From e231dc58a43fc554fa22970df42b43f2e59f07ba Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Sun, 12 Apr 2020 15:50:48 -0400 +Subject: [PATCH] Allow shutting down server during a watchdog hang gracefully + +If the request to shut down the server is received while we are in +a watchdog hang, immediately treat it as a crash and begin the shutdown +process. Shutdown process is now improved to also shutdown cleanly when +not using restart scripts either. + +If a server is deadlocked, a server owner can send SIGUP (or any other signal +the JVM understands to shut down as it currently does) and the watchdog +will no longer need to wait until the full timeout, allowing you to trigger +a close process and try to shut the server down gracefully, saving player and +world data. + +Previously there was no way to trigger this outside of waiting for a full watchdog +timeout, which may be set to a really long time... + +diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java +index 2686874f2..a9b533751 100644 +--- a/src/main/java/net/minecraft/server/MinecraftServer.java ++++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -176,7 +176,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant 0 && new File( split[0] ).isFile() ) +diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java +index 5bdcdcf9e..704e8426a 100644 +--- a/src/main/java/org/spigotmc/WatchdogThread.java ++++ b/src/main/java/org/spigotmc/WatchdogThread.java +@@ -69,7 +69,7 @@ public class WatchdogThread extends Thread + long currentTime = monotonicMillis(); + if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") ) + { +- boolean isLongTimeout = currentTime > lastTick + timeoutTime; ++ boolean isLongTimeout = currentTime > lastTick + timeoutTime || !MinecraftServer.getServer().isRunning(); + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... +@@ -135,9 +135,15 @@ public class WatchdogThread extends Thread + + if ( isLongTimeout ) + { +- if ( restart && !MinecraftServer.getServer().hasStopped() ) ++ if ( !MinecraftServer.getServer().hasStopped() ) + { +- RestartCommand.restart(); ++ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us ++ AsyncCatcher.shuttingDown = true; ++ MinecraftServer.getServer().forceTicks = true; ++ if (restart) { ++ RestartCommand.addShutdownHook( SpigotConfig.restartScript ); ++ } ++ MinecraftServer.getServer().close(); + } + break; + } // Paper end +-- +2.25.1 +