From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: miclebrick Date: Wed, 8 Aug 2018 15:30:52 -0400 Subject: [PATCH] Add Early Warning Feature to WatchDog Detect when the server has been hung for a long duration, and start printing thread dumps at an interval until the point of crash. This will help diagnose what was going on in that time before the crash. diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java index a6d27dcdf954bc2682aba1d9efab42d2d626f8da..dd81701230133442186524e9cd65d70232b824f3 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration; import co.aikar.timings.Timings; import co.aikar.timings.TimingsManager; import org.spigotmc.SpigotConfig; +import org.spigotmc.WatchdogThread; public class PaperConfig { @@ -315,6 +316,14 @@ public class PaperConfig { } } + public static int watchdogPrintEarlyWarningEvery = 5000; + public static int watchdogPrintEarlyWarningDelay = 10000; + private static void watchdogEarlyWarning() { + watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000); + watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000); + WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); + } + public static int tabSpamIncrement = 1; public static int tabSpamLimit = 500; private static void tabSpamLimiters() { diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 74e780212773e0d3cce19cb9c33cdcaa4a708f14..cc4a94669c5f0dee99b6d8cb89650cf0c701e9ab 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -1100,6 +1100,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable + // Paper start + Logger log = Bukkit.getServer().getLogger(); + long currentTime = WatchdogThread.monotonicMillis(); + if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable { - Logger log = Bukkit.getServer().getLogger(); + boolean isLongTimeout = currentTime > lastTick + timeoutTime; + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + lastEarlyWarning = currentTime; + if (isLongTimeout) { + // Paper end log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" ); @@ -93,29 +109,45 @@ public class WatchdogThread extends Thread } } // Paper end + } else + { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---"); + log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump"); + } + // Paper end - Different message for short timeout log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); log.log( Level.SEVERE, "------------------------------" ); // + // Paper start - Only print full dump on long timeouts + if ( isLongTimeout ) + { log.log( Level.SEVERE, "Entire Thread Dump:" ); ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true ); for ( ThreadInfo thread : threads ) { WatchdogThread.dumpThread( thread, log ); } + } else { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---"); + } + log.log( Level.SEVERE, "------------------------------" ); + if ( isLongTimeout ) + { if ( this.restart && !MinecraftServer.getServer().hasStopped() ) { RestartCommand.restart(); } break; + } // Paper end } try { - sleep( 10000 ); + sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout } catch ( InterruptedException ex ) { interrupt();