Paper/patches/server/0245-Add-Early-Warning-Feature-to-WatchDog.patch

183 lines
10 KiB
Diff
Raw Normal View History

2021-06-11 14:02:28 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: miclebrick <miclebrick@outlook.com>
Date: Wed, 8 Aug 2018 15:30:52 -0400
Subject: [PATCH] Add Early Warning Feature to WatchDog
Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.
This will help diagnose what was going on in that time before the crash.
diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
index bd508025b771424c942fd856c31d520b6f548082..62621562137cba4804f0465c58d25ca2786328e5 100644
--- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
+++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
@@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration;
import co.aikar.timings.Timings;
import co.aikar.timings.TimingsManager;
import org.spigotmc.SpigotConfig;
+import org.spigotmc.WatchdogThread;
public class PaperConfig {
@@ -297,6 +298,14 @@ public class PaperConfig {
}
}
+ public static int watchdogPrintEarlyWarningEvery = 5000;
+ public static int watchdogPrintEarlyWarningDelay = 10000;
+ private static void watchdogEarlyWarning() {
+ watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000);
+ watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000);
+ WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
+ }
+
public static int tabSpamIncrement = 1;
public static int tabSpamLimit = 500;
private static void tabSpamLimiters() {
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index d82015cd355a3450faf4cf3eb0f1a1cb10119c77..a7244e66b7bbf2b474304ab41ad31a606ab6ba9c 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
Updated Upstream (Bukkit/CraftBukkit/Spigot) Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: 8503c3c9 #621: Add HumanEntity#getItemInUse and Material#getSlipperiness 248deb09 #622: Add methods to check if item is the breed item for an entity 2ce691d8 Clarify Player#breakBlock only works for blocks in the same world 5dcdd48e SPIGOT-6514: Small Dripleaf block data is missing half property cc9610b7 #619: Add Player#breakBlock() 862bc475 Fix bad merge of SPIGOT-6502 fix 989bb0c1 Downgrade SnakeYAML due to issues with comments parsing 1dff62ae Fix inverted visual fire docs CraftBukkit Changes: 40caacc8 SPIGOT-6526: World entities are not populated when plugin onEnable is called c9a92ad0 SPIGOT-6536: Marker position not set on spawn 20d3e57c #855: Add HumanEntity#getItemInUse and Material#getSlipperiness d9c69b44 SPIGOT-6529: Fix BundleMeta#setItems 8bd43be5 SPIGOT-6535: PlayerGameModeChangeEvent event incorrectly reports old gamemode 4ece3ff3 #856: Add methods to check if item is the breed item for an entity dd4bec5f Add additional validation to Player#breakBlock bc835ae6 SPIGOT-6532: Fix Entity#setGlowing 384e116e Restore 1.16.5 behaviour of InventoryDragEvent being called even when a single item is 'dragged' to its own slot b42e708c Fix new map colors rendering as transparent cfe7fecf SPIGOT-6524: Inventory desync when InventoryClickEvent is cancelled eeae1b19 SPIGOT-6522: ItemStack on cursor is always AIR 7490724d Fix missing PlayerEditBookEvent 06875f76 SPIGOT-6513: Placing ItemStack in Inventory causes InventoryAction.NOTHING 27835bde SPIGOT-6519: Fix end gateway teleports 4ac634ad SPIGOT-6515: "Un-waterlogging" throws UnsupportedOperationException in some cases da425fa2 SPIGOT-6518: Anvils falling onto dripstone can sometimes crash server 50530da9 SPIGOT-6514: Small Dripleaf block data is missing half property 6fdecf20 #853: Implement Player#breakBlock() 4db9c49f SPIGOT-6510: Bukkit#createMap throws NullPointerException 89e2b127 SPIGOT-6517: Spider jockey crash on dripstone cbf2f678 SPIGOT-6508: Rename conflicted getServer 74575d48 SPIGOT-6506: Fix crash with custom inventories a3df386f Fix NPE with Entity.getNearbyEntities d747f8ed Fix NPE with World.getNearbyEntities 4d2c7800 Fix second usage of worldGenSettings just in case 5182f923 SPIGOT-6504: Fix generating fresh worlds Spigot Changes: 66f9d3c1 Rebuild patches 191e4971 Rebuild patches a09c0bb6 Restore Spigot experience merging
2021-06-13 07:13:07 +02:00
@@ -1076,6 +1076,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
this.updateStatusIcon(this.status);
// Spigot start
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
Arrays.fill( recentTps, 20 );
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
lastTick = start - TICK_TIME; // Paper
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index 38c7a8befad8633243b55d0f692f49f2e1b61256..02b85866b6f4b8975f66fd8c706cda8df49ac6a8 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
@@ -806,6 +806,7 @@ public final class CraftServer implements Server {
2021-06-11 14:02:28 +02:00
@Override
public void reload() {
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
2021-06-13 01:45:00 +02:00
this.reloadCount++;
this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
@@ -921,6 +922,7 @@ public final class CraftServer implements Server {
2021-06-13 01:45:00 +02:00
this.enablePlugins(PluginLoadOrder.STARTUP);
this.enablePlugins(PluginLoadOrder.POSTWORLD);
this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
2021-06-11 14:02:28 +02:00
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
}
@Override
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
2021-06-13 01:45:00 +02:00
index 314fa148fe783a0558ba00b068e0bf69a91577e1..66a3395a7a0cc8eb1b3cd24761bed9d8c3349f4c 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/SpigotConfig.java
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
@@ -229,7 +229,7 @@ public class SpigotConfig
2021-06-13 01:45:00 +02:00
SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
2021-06-11 14:02:28 +02:00
- WatchdogThread.doStart( timeoutTime, restartOnCrash );
+ //WatchdogThread.doStart( timeoutTime, restartOnCrash ); // Paper - moved to PaperConfig
}
public static boolean bungee;
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
2021-06-13 01:45:00 +02:00
index 6e1fa4f0616ccfd258acd1b4f5b08fc0ad4c9529..a01a266a25e3267c94c20f8597b4b596efe20faa 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -5,6 +5,7 @@ import java.lang.management.MonitorInfo;
import java.lang.management.ThreadInfo;
import java.util.logging.Level;
import java.util.logging.Logger;
+import com.destroystokyo.paper.PaperConfig;
import net.minecraft.server.MinecraftServer;
import org.bukkit.Bukkit;
@@ -14,6 +15,10 @@ public class WatchdogThread extends Thread
private static WatchdogThread instance;
private long timeoutTime;
private boolean restart;
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
+ private final long earlyWarningDelay; // Paper
+ public static volatile boolean hasStarted; // Paper
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
private volatile long lastTick;
private volatile boolean stopping;
@@ -22,6 +27,8 @@ public class WatchdogThread extends Thread
super( "Paper Watchdog Thread" );
this.timeoutTime = timeoutTime;
this.restart = restart;
+ earlyWarningEvery = Math.min(PaperConfig.watchdogPrintEarlyWarningEvery, timeoutTime); // Paper
+ earlyWarningDelay = Math.min(PaperConfig.watchdogPrintEarlyWarningDelay, timeoutTime); // Paper
}
private static long monotonicMillis()
2021-06-13 01:45:00 +02:00
@@ -61,9 +68,18 @@ public class WatchdogThread extends Thread
while ( !this.stopping )
2021-06-11 14:02:28 +02:00
{
2021-06-13 01:45:00 +02:00
//
- if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
2021-06-11 14:02:28 +02:00
+ // Paper start
+ Logger log = Bukkit.getServer().getLogger();
2021-06-13 01:45:00 +02:00
+ long currentTime = WatchdogThread.monotonicMillis();
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
2021-06-11 14:02:28 +02:00
{
- Logger log = Bukkit.getServer().getLogger();
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+ // Don't spam early warning dumps
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+ if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+ lastEarlyWarning = currentTime;
+ if (isLongTimeout) {
+ // Paper end
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
2021-06-13 01:45:00 +02:00
@@ -93,29 +109,45 @@ public class WatchdogThread extends Thread
2021-06-11 14:02:28 +02:00
}
}
// Paper end
+ } else
+ {
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
+ }
+ // Paper end - Different message for short timeout
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
2021-06-13 01:45:00 +02:00
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
2021-06-11 14:02:28 +02:00
log.log( Level.SEVERE, "------------------------------" );
//
+ // Paper start - Only print full dump on long timeouts
+ if ( isLongTimeout )
+ {
log.log( Level.SEVERE, "Entire Thread Dump:" );
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
for ( ThreadInfo thread : threads )
{
2021-06-13 01:45:00 +02:00
WatchdogThread.dumpThread( thread, log );
2021-06-11 14:02:28 +02:00
}
+ } else {
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
+ }
+
log.log( Level.SEVERE, "------------------------------" );
+ if ( isLongTimeout )
+ {
2021-06-13 01:45:00 +02:00
if ( this.restart && !MinecraftServer.getServer().hasStopped() )
2021-06-11 14:02:28 +02:00
{
RestartCommand.restart();
}
break;
+ } // Paper end
}
try
{
- sleep( 10000 );
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
} catch ( InterruptedException ex )
{
interrupt();