From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: miclebrick Date: Wed, 8 Aug 2018 15:30:52 -0400 Subject: [PATCH] Add Early Warning Feature to WatchDog Detect when the server has been hung for a long duration, and start printing thread dumps at an interval until the point of crash. This will help diagnose what was going on in that time before the crash. diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 52f5c91d71a8fc86d14c3c4c9e258b9cf70ad57b..41bad67b3a6b8ccdcf9467858cb7d9c0e0a03f0e 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -1029,6 +1029,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable + // Paper start + Logger log = Bukkit.getServer().getLogger(); + long currentTime = WatchdogThread.monotonicMillis(); + if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable { - Logger log = Bukkit.getServer().getLogger(); + boolean isLongTimeout = currentTime > lastTick + timeoutTime; + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + lastEarlyWarning = currentTime; + if (isLongTimeout) { + // Paper end log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" ); @@ -93,30 +108,46 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa } } // Paper end + } else + { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---"); + log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump"); + } + // Paper end - Different message for short timeout log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper io.papermc.paper.chunk.system.scheduling.ChunkTaskScheduler.dumpAllChunkLoadInfo(isLongTimeout); // Paper // Paper - rewrite chunk system WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); log.log( Level.SEVERE, "------------------------------" ); // + // Paper start - Only print full dump on long timeouts + if ( isLongTimeout ) + { log.log( Level.SEVERE, "Entire Thread Dump:" ); ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true ); for ( ThreadInfo thread : threads ) { WatchdogThread.dumpThread( thread, log ); } + } else { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---"); + } + log.log( Level.SEVERE, "------------------------------" ); + if ( isLongTimeout ) + { if ( this.restart && !MinecraftServer.getServer().hasStopped() ) { RestartCommand.restart(); } break; + } // Paper end } try { - sleep( 10000 ); + sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout } catch ( InterruptedException ex ) { interrupt();