Paper/Spigot-Server-Patches/0281-Add-Early-Warning-Feature-to-WatchDog.patch

187 lines
9.2 KiB
Diff
Raw Normal View History

From 50e490ebdc5b66293d76047c4e693b038cb1a308 Mon Sep 17 00:00:00 2001
From: miclebrick <miclebrick@outlook.com>
Date: Wed, 8 Aug 2018 15:30:52 -0400
Subject: [PATCH] Add Early Warning Feature to WatchDog
Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.
This will help diagnose what was going on in that time before the crash.
diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
index d773ef5b0d..833659bbb2 100644
--- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
+++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
@@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration;
import co.aikar.timings.Timings;
import co.aikar.timings.TimingsManager;
import org.spigotmc.SpigotConfig;
+import org.spigotmc.WatchdogThread;
public class PaperConfig {
@@ -312,6 +313,14 @@ public class PaperConfig {
}
}
+ public static int watchdogPrintEarlyWarningEvery = 5000;
+ public static int watchdogPrintEarlyWarningDelay = 10000;
+ private static void watchdogEarlyWarning() {
+ watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000);
+ watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000);
+ WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
+ }
+
2018-08-29 18:26:24 +02:00
public static int tabSpamIncrement = 1;
public static int tabSpamLimit = 500;
private static void tabSpamLimiters() {
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 7e27f10790..3f2a3dd178 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
2019-05-07 02:33:26 +02:00
@@ -847,6 +847,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
2019-05-05 04:23:25 +02:00
this.a(this.serverPing);
// Spigot start
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
Arrays.fill( recentTps, 20 );
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
lastTick = start - TICK_TIME; // Paper
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index f6c5a9c364..d316652ffb 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2019-05-06 05:53:47 +02:00
@@ -754,6 +754,7 @@ public final class CraftServer implements Server {
@Override
public void reload() {
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
reloadCount++;
configuration = YamlConfiguration.loadConfiguration(getConfigFile());
commandsConfiguration = YamlConfiguration.loadConfiguration(getCommandsConfigFile());
2019-05-06 05:53:47 +02:00
@@ -852,6 +853,7 @@ public final class CraftServer implements Server {
enablePlugins(PluginLoadOrder.STARTUP);
enablePlugins(PluginLoadOrder.POSTWORLD);
getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
}
@Override
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
index 6c5430ff94..4b8581f0ed 100644
--- a/src/main/java/org/spigotmc/SpigotConfig.java
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
Updated Upstream (Bukkit/CraftBukkit/Spigot) Upstream has released updates that appears to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Warning: this commit contains more mapping changes from upstream, As always, ensure that you have working backups and test this build before deployment; Developers working on paper will, yet again, need to delete their work/Minecraft/1.13.2 folder Bukkit Changes: 7fca5fd4 SPIGOT-4558: Preserve user order in the face of copied defaults in configurations 15c9b1eb Ignore spurious slot IDs sent by client, e.g. in enchanting tables 5d2a10c5 SPIGOT-3747: Add API for force loaded chunks d6dd2bb3 SPIGOT-3538: Add getHitBlockFace for ProjectileHitEvent 771db4aa SPIGOT-794: Call EntityPlaceEvent for Minecart placement 55462509 Add InventoryView#getSlotType 2f3ce5b6 Remove EntityTransformEvent and CustomItemTagContainer from draft API f04ad7b6 Make ProjectileLaunchEvent extend EntitySpawnEvent ccb85808 Define EntitySpawnEvent b8cc3ebe Add PlayerItemDamageEvent 184a495d Ease ClassLoader Deadlocks Where Possible 11ac4728 Expand Boolean Prompt Values in Conversation API aae62d51 Added getAllSessionData() to the Conversation API. 9290ff91 Add InventoryView#getInventory API 995e530f Add API to get / set base arrow damage CraftBukkit Changes: c4a67eed SPIGOT-4556: Fix plugins closing inventory during drop events 5be2ddcb Replace version constants with methods to prevent compiler inlining a5b9c7b3 Use API method to create offset command completions 2bc7d1df SPIGOT-3747: Add API for force loaded chunks a408f375 SPIGOT-3538: Add getHitBlockFace for ProjectileHitEvent b54b9409 SPIGOT-2864: Make Arrow / Item setTicksLived behave like FallingBlock 79ded7a8 SPIGOT-1811: Death message not shown on respawn screen b4a4f15d SPIGOT-943: InventoryCloseEvent called on death regardless of open inventory 0afed592 SPIGOT-794: Call EntityPlaceEvent for Minecart placement 2b2d084a Add InventoryView#getSlotType 01a9959a Do not use deprecated ItemSpawnEvent constructor 9642498d SPIGOT-4547: Call EntitySpawnEvent as general spawn fallback event 963f4a5f Add PlayerItemDamageEvent 63db0445 Add API to get / set base arrow damage 531c25d7 Add CraftMagicNumbers.MAPPINGS_VERSION for use by NMS plugins d05c8b14 Mappings Update bd36e200 SPIGOT-4551: Ignore invalid attribute modifier slots Spigot Changes: 518206a1 Remove redundant trove depend 1959ad21 MC-11211,SPIGOT-4552: Fix placing double slabs at y = 255 29ab5e43 SPIGOT-3661: Allow arguments in restart-script 7cc46316 SPIGOT-852: Growth modifiers for beetroots, potatoes, carrots 82e117e1 Squelch "fatal: Resolve operation not in progress" message 0a1a68e7 Mappings Update & Patch Rebuild
2019-01-01 04:15:55 +01:00
@@ -225,7 +225,7 @@ public class SpigotConfig
restartScript = getString( "settings.restart-script", restartScript );
restartMessage = transform( getString( "messages.restart", "Server is restarting" ) );
commands.put( "restart", new RestartCommand( "restart" ) );
- WatchdogThread.doStart( timeoutTime, restartOnCrash );
+ //WatchdogThread.doStart( timeoutTime, restartOnCrash ); // Paper - moved to PaperConfig
}
public static boolean bungee;
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index 9532aada82..a1d93200e6 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -5,6 +5,7 @@ import java.lang.management.MonitorInfo;
import java.lang.management.ThreadInfo;
import java.util.logging.Level;
import java.util.logging.Logger;
+import com.destroystokyo.paper.PaperConfig;
import net.minecraft.server.MinecraftServer;
import org.bukkit.Bukkit;
@@ -13,6 +14,10 @@ public class WatchdogThread extends Thread
private static WatchdogThread instance;
private final long timeoutTime;
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
+ private final long earlyWarningDelay; // Paper
+ public static volatile boolean hasStarted; // Paper
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
private final boolean restart;
private volatile long lastTick;
private volatile boolean stopping;
@@ -22,6 +27,8 @@ public class WatchdogThread extends Thread
super( "Paper Watchdog Thread" );
this.timeoutTime = timeoutTime;
this.restart = restart;
+ earlyWarningEvery = Math.min(PaperConfig.watchdogPrintEarlyWarningEvery, timeoutTime); // Paper
+ earlyWarningDelay = Math.min(PaperConfig.watchdogPrintEarlyWarningDelay, timeoutTime); // Paper
}
private static long monotonicMillis()
@@ -56,10 +63,17 @@ public class WatchdogThread extends Thread
{
while ( !stopping )
{
- //
- if ( lastTick != 0 && monotonicMillis() > lastTick + timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
+ // Paper start
+ Logger log = Bukkit.getServer().getLogger();
+ long currentTime = monotonicMillis();
+ if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") )
{
- Logger log = Bukkit.getServer().getLogger();
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+ // Don't spam early warning dumps
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+ lastEarlyWarning = currentTime;
+ if (isLongTimeout) {
+ // Paper end
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
@@ -89,29 +103,46 @@ public class WatchdogThread extends Thread
}
}
// Paper end
+ } else
+ {
2018-10-24 21:05:39 +02:00
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
+ }
+ // Paper end - Different message for short timeout
log.log( Level.SEVERE, "------------------------------" );
2019-05-05 04:23:25 +02:00
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
log.log( Level.SEVERE, "------------------------------" );
//
+ // Paper start - Only print full dump on long timeouts
+ if ( isLongTimeout )
+ {
log.log( Level.SEVERE, "Entire Thread Dump:" );
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
for ( ThreadInfo thread : threads )
{
dumpThread( thread, log );
}
+ } else {
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
+ }
+
+
log.log( Level.SEVERE, "------------------------------" );
+ if ( isLongTimeout )
+ {
if ( restart )
{
RestartCommand.restart();
}
break;
+ } // Paper end
}
try
{
- sleep( 10000 );
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
} catch ( InterruptedException ex )
{
interrupt();
--
2.21.0