Paper/patches/server/0229-Add-Early-Warning-Feature-to-WatchDog.patch

160 lines
9.8 KiB
Diff
Raw Normal View History

2021-06-11 14:02:28 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: miclebrick <miclebrick@outlook.com>
Date: Wed, 8 Aug 2018 15:30:52 -0400
Subject: [PATCH] Add Early Warning Feature to WatchDog
Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.
This will help diagnose what was going on in that time before the crash.
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
Updated Upstream (Bukkit/CraftBukkit) (#10242) * Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: a6a9d2a4 Remove some old ApiStatus.Experimental annotations be72314c SPIGOT-7300, PR-829: Add new DamageSource API providing enhanced information about entity damage b252cf05 SPIGOT-7576, PR-970: Add methods in MushroomCow to change stew effects b1c689bd PR-902: Add Server#isLoggingIPs to get log-ips configuration 08f86d1c PR-971: Add Player methods for client-side potion effects 2e3024a9 PR-963: Add API for in-world structures a23292a7 SPIGOT-7530, PR-948: Improve Resource Pack API with new 1.20.3 functionality 1851857b SPIGOT-3071, PR-969: Add entity spawn method with spawn reason cde4c52a SPIGOT-5553, PR-964: Add EntityKnockbackEvent CraftBukkit Changes: 38fd4bd50 Fix accidentally renamed internal damage method 80f0ce4be SPIGOT-7300, PR-1180: Add new DamageSource API providing enhanced information about entity damage 7e43f3b16 SPIGOT-7581: Fix typo in BlockMushroom ea14b7d90 SPIGOT-7576, PR-1347: Add methods in MushroomCow to change stew effects 4c687f243 PR-1259: Add Server#isLoggingIPs to get log-ips configuration 22a541a29 Improve support for per-world game rules cb7dccce2 PR-1348: Add Player methods for client-side potion effects b8d6109f0 PR-1335: Add API for in-world structures 4398a1b5b SPIGOT-7577: Make CraftWindCharge#explode discard the entity e74107678 Fix Crafter maximum stack size 0bb0f4f6a SPIGOT-7530, PR-1314: Improve Resource Pack API with new 1.20.3 functionality 4949f556d SPIGOT-3071, PR-1345: Add entity spawn method with spawn reason 20ac73ca2 PR-1353: Fix Structure#place not working as documented with 0 palette 3c1b77871 SPIGOT-6911, PR-1349: Change max book length in CraftMetaBook 333701839 SPIGOT-7572: Bee nests generated without bees f48f4174c SPIGOT-5553, PR-1336: Add EntityKnockbackEvent
2024-02-11 22:28:00 +01:00
index 0f3589ae1c5577c612b93289fc42cdb977486b6b..0b704d5d39263b66f7846a9c4a116ac4d96383cf 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -1092,6 +1092,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2023-03-14 19:59:51 +01:00
this.status = this.buildServerStatus();
2021-06-11 14:02:28 +02:00
2022-06-07 23:45:11 +02:00
// Spigot start
2022-06-08 06:22:42 +02:00
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
2023-10-27 01:34:58 +02:00
Arrays.fill( this.recentTps, 20 );
// Paper start - further improve server tick loop
long tickSection = Util.getNanos();
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
Updated Upstream (Bukkit/CraftBukkit) (#10242) * Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: a6a9d2a4 Remove some old ApiStatus.Experimental annotations be72314c SPIGOT-7300, PR-829: Add new DamageSource API providing enhanced information about entity damage b252cf05 SPIGOT-7576, PR-970: Add methods in MushroomCow to change stew effects b1c689bd PR-902: Add Server#isLoggingIPs to get log-ips configuration 08f86d1c PR-971: Add Player methods for client-side potion effects 2e3024a9 PR-963: Add API for in-world structures a23292a7 SPIGOT-7530, PR-948: Improve Resource Pack API with new 1.20.3 functionality 1851857b SPIGOT-3071, PR-969: Add entity spawn method with spawn reason cde4c52a SPIGOT-5553, PR-964: Add EntityKnockbackEvent CraftBukkit Changes: 38fd4bd50 Fix accidentally renamed internal damage method 80f0ce4be SPIGOT-7300, PR-1180: Add new DamageSource API providing enhanced information about entity damage 7e43f3b16 SPIGOT-7581: Fix typo in BlockMushroom ea14b7d90 SPIGOT-7576, PR-1347: Add methods in MushroomCow to change stew effects 4c687f243 PR-1259: Add Server#isLoggingIPs to get log-ips configuration 22a541a29 Improve support for per-world game rules cb7dccce2 PR-1348: Add Player methods for client-side potion effects b8d6109f0 PR-1335: Add API for in-world structures 4398a1b5b SPIGOT-7577: Make CraftWindCharge#explode discard the entity e74107678 Fix Crafter maximum stack size 0bb0f4f6a SPIGOT-7530, PR-1314: Improve Resource Pack API with new 1.20.3 functionality 4949f556d SPIGOT-3071, PR-1345: Add entity spawn method with spawn reason 20ac73ca2 PR-1353: Fix Structure#place not working as documented with 0 palette 3c1b77871 SPIGOT-6911, PR-1349: Change max book length in CraftMetaBook 333701839 SPIGOT-7572: Bee nests generated without bees f48f4174c SPIGOT-5553, PR-1336: Add EntityKnockbackEvent
2024-02-11 22:28:00 +01:00
index 8edf2f049aa077792bfb507384f4cf5bb716cb64..6db918db1984d298c57eb93f7cf38aa26f01a04c 100644
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
@@ -198,6 +198,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
this.paperConfigurations.initializeGlobalConfiguration(this.registryAccess());
this.paperConfigurations.initializeWorldDefaultsConfiguration(this.registryAccess());
// Paper end - initialize global and world-defaults configuration
+ org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash); // Paper - start watchdog thread
io.papermc.paper.command.PaperCommands.registerCommands(this); // Paper - setup /paper command
com.destroystokyo.paper.Metrics.PaperMetrics.startMetrics(); // Paper - start metrics
com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // Paper - load version history now
2021-06-11 14:02:28 +02:00
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
Updated Upstream (Bukkit/CraftBukkit) (#10242) * Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: a6a9d2a4 Remove some old ApiStatus.Experimental annotations be72314c SPIGOT-7300, PR-829: Add new DamageSource API providing enhanced information about entity damage b252cf05 SPIGOT-7576, PR-970: Add methods in MushroomCow to change stew effects b1c689bd PR-902: Add Server#isLoggingIPs to get log-ips configuration 08f86d1c PR-971: Add Player methods for client-side potion effects 2e3024a9 PR-963: Add API for in-world structures a23292a7 SPIGOT-7530, PR-948: Improve Resource Pack API with new 1.20.3 functionality 1851857b SPIGOT-3071, PR-969: Add entity spawn method with spawn reason cde4c52a SPIGOT-5553, PR-964: Add EntityKnockbackEvent CraftBukkit Changes: 38fd4bd50 Fix accidentally renamed internal damage method 80f0ce4be SPIGOT-7300, PR-1180: Add new DamageSource API providing enhanced information about entity damage 7e43f3b16 SPIGOT-7581: Fix typo in BlockMushroom ea14b7d90 SPIGOT-7576, PR-1347: Add methods in MushroomCow to change stew effects 4c687f243 PR-1259: Add Server#isLoggingIPs to get log-ips configuration 22a541a29 Improve support for per-world game rules cb7dccce2 PR-1348: Add Player methods for client-side potion effects b8d6109f0 PR-1335: Add API for in-world structures 4398a1b5b SPIGOT-7577: Make CraftWindCharge#explode discard the entity e74107678 Fix Crafter maximum stack size 0bb0f4f6a SPIGOT-7530, PR-1314: Improve Resource Pack API with new 1.20.3 functionality 4949f556d SPIGOT-3071, PR-1345: Add entity spawn method with spawn reason 20ac73ca2 PR-1353: Fix Structure#place not working as documented with 0 palette 3c1b77871 SPIGOT-6911, PR-1349: Change max book length in CraftMetaBook 333701839 SPIGOT-7572: Bee nests generated without bees f48f4174c SPIGOT-5553, PR-1336: Add EntityKnockbackEvent
2024-02-11 22:28:00 +01:00
index d25ee3ca530289c0deda45b01196ece98788e628..abdff07588cb06f8ec4f3f2172f7dce92fa8a979 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
Updated Upstream (Bukkit/CraftBukkit) (#10242) * Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: a6a9d2a4 Remove some old ApiStatus.Experimental annotations be72314c SPIGOT-7300, PR-829: Add new DamageSource API providing enhanced information about entity damage b252cf05 SPIGOT-7576, PR-970: Add methods in MushroomCow to change stew effects b1c689bd PR-902: Add Server#isLoggingIPs to get log-ips configuration 08f86d1c PR-971: Add Player methods for client-side potion effects 2e3024a9 PR-963: Add API for in-world structures a23292a7 SPIGOT-7530, PR-948: Improve Resource Pack API with new 1.20.3 functionality 1851857b SPIGOT-3071, PR-969: Add entity spawn method with spawn reason cde4c52a SPIGOT-5553, PR-964: Add EntityKnockbackEvent CraftBukkit Changes: 38fd4bd50 Fix accidentally renamed internal damage method 80f0ce4be SPIGOT-7300, PR-1180: Add new DamageSource API providing enhanced information about entity damage 7e43f3b16 SPIGOT-7581: Fix typo in BlockMushroom ea14b7d90 SPIGOT-7576, PR-1347: Add methods in MushroomCow to change stew effects 4c687f243 PR-1259: Add Server#isLoggingIPs to get log-ips configuration 22a541a29 Improve support for per-world game rules cb7dccce2 PR-1348: Add Player methods for client-side potion effects b8d6109f0 PR-1335: Add API for in-world structures 4398a1b5b SPIGOT-7577: Make CraftWindCharge#explode discard the entity e74107678 Fix Crafter maximum stack size 0bb0f4f6a SPIGOT-7530, PR-1314: Improve Resource Pack API with new 1.20.3 functionality 4949f556d SPIGOT-3071, PR-1345: Add entity spawn method with spawn reason 20ac73ca2 PR-1353: Fix Structure#place not working as documented with 0 palette 3c1b77871 SPIGOT-6911, PR-1349: Change max book length in CraftMetaBook 333701839 SPIGOT-7572: Bee nests generated without bees f48f4174c SPIGOT-5553, PR-1336: Add EntityKnockbackEvent
2024-02-11 22:28:00 +01:00
@@ -927,6 +927,7 @@ public final class CraftServer implements Server {
2021-06-11 14:02:28 +02:00
@Override
public void reload() {
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
2021-06-13 01:45:00 +02:00
this.reloadCount++;
this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
Updated Upstream (Bukkit/CraftBukkit) (#10242) * Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: a6a9d2a4 Remove some old ApiStatus.Experimental annotations be72314c SPIGOT-7300, PR-829: Add new DamageSource API providing enhanced information about entity damage b252cf05 SPIGOT-7576, PR-970: Add methods in MushroomCow to change stew effects b1c689bd PR-902: Add Server#isLoggingIPs to get log-ips configuration 08f86d1c PR-971: Add Player methods for client-side potion effects 2e3024a9 PR-963: Add API for in-world structures a23292a7 SPIGOT-7530, PR-948: Improve Resource Pack API with new 1.20.3 functionality 1851857b SPIGOT-3071, PR-969: Add entity spawn method with spawn reason cde4c52a SPIGOT-5553, PR-964: Add EntityKnockbackEvent CraftBukkit Changes: 38fd4bd50 Fix accidentally renamed internal damage method 80f0ce4be SPIGOT-7300, PR-1180: Add new DamageSource API providing enhanced information about entity damage 7e43f3b16 SPIGOT-7581: Fix typo in BlockMushroom ea14b7d90 SPIGOT-7576, PR-1347: Add methods in MushroomCow to change stew effects 4c687f243 PR-1259: Add Server#isLoggingIPs to get log-ips configuration 22a541a29 Improve support for per-world game rules cb7dccce2 PR-1348: Add Player methods for client-side potion effects b8d6109f0 PR-1335: Add API for in-world structures 4398a1b5b SPIGOT-7577: Make CraftWindCharge#explode discard the entity e74107678 Fix Crafter maximum stack size 0bb0f4f6a SPIGOT-7530, PR-1314: Improve Resource Pack API with new 1.20.3 functionality 4949f556d SPIGOT-3071, PR-1345: Add entity spawn method with spawn reason 20ac73ca2 PR-1353: Fix Structure#place not working as documented with 0 palette 3c1b77871 SPIGOT-6911, PR-1349: Change max book length in CraftMetaBook 333701839 SPIGOT-7572: Bee nests generated without bees f48f4174c SPIGOT-5553, PR-1336: Add EntityKnockbackEvent
2024-02-11 22:28:00 +01:00
@@ -1016,6 +1017,7 @@ public final class CraftServer implements Server {
2021-06-13 01:45:00 +02:00
this.enablePlugins(PluginLoadOrder.STARTUP);
this.enablePlugins(PluginLoadOrder.POSTWORLD);
this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
2021-06-11 14:02:28 +02:00
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
}
@Override
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
index 9c5c19ab9dc14d844631f47a93f3349409efdf43..56dbc23f10d946a7ec992b08fd868f3ee5089068 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/SpigotConfig.java
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
@@ -229,7 +229,7 @@ public class SpigotConfig
2021-06-13 01:45:00 +02:00
SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
2023-10-27 01:34:58 +02:00
- WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
+ // WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); // Paper - moved to after paper config initialization
2021-06-11 14:02:28 +02:00
}
public static boolean bungee;
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
2024-01-24 14:05:59 +01:00
index ce17005f8c255eb3096736ef9f0d1de64b612d79..017c2eadbc5dfec155c21b5d8a80f0b1e380398e 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -14,6 +14,10 @@ public class WatchdogThread extends Thread
2021-06-11 14:02:28 +02:00
private static WatchdogThread instance;
private long timeoutTime;
private boolean restart;
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
+ private final long earlyWarningDelay; // Paper
+ public static volatile boolean hasStarted; // Paper
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
private volatile long lastTick;
private volatile boolean stopping;
@@ -22,6 +26,8 @@ public class WatchdogThread extends Thread
2021-06-11 14:02:28 +02:00
super( "Paper Watchdog Thread" );
this.timeoutTime = timeoutTime;
this.restart = restart;
+ earlyWarningEvery = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningEvery, timeoutTime); // Paper
+ earlyWarningDelay = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningDelay, timeoutTime); // Paper
2021-06-11 14:02:28 +02:00
}
private static long monotonicMillis()
@@ -61,9 +67,18 @@ public class WatchdogThread extends Thread
2021-06-13 01:45:00 +02:00
while ( !this.stopping )
2021-06-11 14:02:28 +02:00
{
2021-06-13 01:45:00 +02:00
//
- if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
2021-06-11 14:02:28 +02:00
+ // Paper start
+ Logger log = Bukkit.getServer().getLogger();
2021-06-13 01:45:00 +02:00
+ long currentTime = WatchdogThread.monotonicMillis();
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
2021-06-11 14:02:28 +02:00
{
- Logger log = Bukkit.getServer().getLogger();
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+ // Don't spam early warning dumps
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+ if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+ lastEarlyWarning = currentTime;
+ if (isLongTimeout) {
+ // Paper end
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
2024-01-24 14:05:59 +01:00
@@ -92,29 +107,45 @@ public class WatchdogThread extends Thread
}
2021-06-11 14:02:28 +02:00
}
2024-01-24 14:05:59 +01:00
// Paper end
2021-06-11 14:02:28 +02:00
+ } else
+ {
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
+ }
+ // Paper end - Different message for short timeout
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
2021-06-13 01:45:00 +02:00
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
2021-06-11 14:02:28 +02:00
log.log( Level.SEVERE, "------------------------------" );
//
+ // Paper start - Only print full dump on long timeouts
+ if ( isLongTimeout )
+ {
log.log( Level.SEVERE, "Entire Thread Dump:" );
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
for ( ThreadInfo thread : threads )
{
2021-06-13 01:45:00 +02:00
WatchdogThread.dumpThread( thread, log );
2021-06-11 14:02:28 +02:00
}
+ } else {
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
+ }
+
log.log( Level.SEVERE, "------------------------------" );
+ if ( isLongTimeout )
+ {
2021-06-13 01:45:00 +02:00
if ( this.restart && !MinecraftServer.getServer().hasStopped() )
2021-06-11 14:02:28 +02:00
{
RestartCommand.restart();
}
break;
+ } // Paper end
}
try
{
- sleep( 10000 );
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
} catch ( InterruptedException ex )
{
2023-10-27 01:34:58 +02:00
this.interrupt();