Paper/patches/server/0388-Improved-Watchdog-Support.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Sun, 12 Apr 2020 15:50:48 -0400
Subject: [PATCH] Improved Watchdog Support

Forced Watchdog Crash support and Improve Async Shutdown

If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.

If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.

Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...

Additionally, fix everything to do with shutting the server down asynchronously.

Previously, nearly everything about the process was fragile and unsafe. Main might
not have actually been frozen, and might still be manipulating state.

Or, some reuest might ask main to do something in the shutdown but main is dead.

Or worse, other things might start closing down items such as the Console or Thread Pool
before we are fully shutdown.

This change tries to resolve all of these issues by moving everything into the stop
method and guaranteeing only one thread is stopping the server.

We then issue Thread Death to the main thread of another thread initiates the stop process.
We have to ensure Thread Death propagates correctly though to stop main completely.

This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.

This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
are properly accounted for and wont trip watchdog on init.

diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
index 5a19e30a9b7e65a70f68a429b8ca741f788a303b..7b1843e16745ca8db2244e17490d291401f22679 100644
--- a/src/main/java/com/destroystokyo/paper/Metrics.java
+++ b/src/main/java/com/destroystokyo/paper/Metrics.java
@@ -92,7 +92,12 @@ public class Metrics {
      * Starts the Scheduler which submits our data every 30 minutes.
      */
     private void startSubmitting() {
-        final Runnable submitTask = this::submitData;
+        final Runnable submitTask = () -> {
+            if (MinecraftServer.getServer().hasStopped()) {
+                return;
+            }
+            submitData();
+        };
 
         // Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the
         // bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay.
diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
index 336795dff742b7c6957fbd3476aff31d25a5e659..30a58229aa6dac5039511d0c0df5f2912ea7de9f 100644
--- a/src/main/java/net/minecraft/CrashReport.java
+++ b/src/main/java/net/minecraft/CrashReport.java
@@ -230,6 +230,7 @@ public class CrashReport {
     }
 
     public static CrashReport forThrowable(Throwable cause, String title) {
+        if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
         while (cause instanceof CompletionException && cause.getCause() != null) {
             cause = cause.getCause();
         }
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 12ca13f5b0556c53fd36d638ee4fa854b89ee8ec..3a4222f78a02e10ecccc03df3c580895fbb8059d 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -284,7 +284,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
     public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
     public int autosavePeriod;
     public Commands vanillaCommandDispatcher;
-    private boolean forceTicks;
+    public boolean forceTicks; // Paper
     // CraftBukkit end
     // Spigot start
     public static final int TPS = 20;
@@ -295,6 +295,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
     public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations;
     public static long currentTickLong = 0L; // Paper
 
+    public volatile Thread shutdownThread; // Paper
+    public volatile boolean abnormalExit = false; // Paper
+
     public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
         AtomicReference<S> atomicreference = new AtomicReference();
         Thread thread = new Thread(() -> {
@@ -867,6 +870,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
     // CraftBukkit start
     private boolean hasStopped = false;
+    public volatile boolean hasFullyShutdown = false; // Paper
     private final Object stopLock = new Object();
     public final boolean hasStopped() {
         synchronized (this.stopLock) {
@@ -881,6 +885,19 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             if (this.hasStopped) return;
             this.hasStopped = true;
         }
+        // Paper start - kill main thread, and kill it hard
+        shutdownThread = Thread.currentThread();
+        org.spigotmc.WatchdogThread.doStop(); // Paper
+        if (!isSameThread()) {
+            MinecraftServer.LOGGER.info("Stopping main thread (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)");
+            while (this.getRunningThread().isAlive()) {
+                this.getRunningThread().stop();
+                try {
+                    Thread.sleep(1);
+                } catch (InterruptedException e) {}
+            }
+        }
+        // Paper end
         // CraftBukkit end
         if (this.metricsRecorder.isRecording()) {
             this.cancelRecordingMetrics();
@@ -966,7 +983,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             this.getProfileCache().save(false); // Paper
         }
         // Spigot end
+        // Paper start - move final shutdown items here
+        LOGGER.info("Flushing Chunk IO");
         com.destroystokyo.paper.io.PaperFileIOThread.Holder.INSTANCE.close(true, true); // Paper
+        LOGGER.info("Closing Thread Pool");
+        Util.shutdownExecutors(); // Paper
+        LOGGER.info("Closing Server");
+        try {
+            net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+        } catch (Exception e) {
+        }
+        this.onServerExit();
+        // Paper end
     }
 
     public String getLocalIp() {
@@ -1059,6 +1087,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
     protected void runServer() {
         try {
+            long serverStartTime = Util.getNanos(); // Paper
             if (!this.initServer()) {
                 throw new IllegalStateException("Failed to initialize server");
             }
@@ -1071,6 +1100,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             this.updateStatusIcon(this.status);
 
             // Spigot start
+            // Paper start - move done tracking
+            LOGGER.info("Running delayed init tasks");
+            this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init,
+            // this is going to be the first thing the tick process does anyways, so move done and run it after
+            // everything is init before watchdog tick.
+            // anything at 3+ won't be caught here but also will trip watchdog....
+            // tasks are default scheduled at -1 + delay, and first tick will tick at 1
+            String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (Util.getNanos() - serverStartTime) / 1.0E9D);
+            LOGGER.info("Done ({})! For help, type \"help\"", doneTime);
+            // Paper end
+
+            org.spigotmc.WatchdogThread.tick(); // Paper
             org.spigotmc.WatchdogThread.hasStarted = true; // Paper
             Arrays.fill( recentTps, 20 );
             long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
@@ -1125,6 +1166,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 JvmProfiler.INSTANCE.onServerTick(this.averageTickTime);
             }
         } catch (Throwable throwable) {
+            // Paper start
+            if (throwable instanceof ThreadDeath) {
+                MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
+                return;
+            }
+            // Paper end
             MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
             // Spigot Start
             if ( throwable.getCause() != null )
@@ -1155,14 +1202,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                     this.services.profileCache().clearExecutor();
                 }
 
-                org.spigotmc.WatchdogThread.doStop(); // Spigot
+                //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
                 // CraftBukkit start - Restore terminal to original settings
                 try {
-                    net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                    //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
                 } catch (Exception ignored) {
                 }
                 // CraftBukkit end
-                this.onServerExit();
+                //this.onServerExit(); // Paper - moved into stop
             }
 
         }
@@ -1226,6 +1273,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
     @Override
     public TickTask wrapRunnable(Runnable runnable) {
+        // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
+        if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
+            runnable.run();
+            runnable = () -> {};
+        }
+        // Paper end
         return new TickTask(this.tickCount, runnable);
     }
 
@@ -1452,6 +1505,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 try {
                     crashreport = CrashReport.forThrowable(throwable, "Exception ticking world");
                 } catch (Throwable t) {
+                    if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper
                     throw new RuntimeException("Error generating crash report", t);
                 }
                 // Spigot End
@@ -1943,7 +1997,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             this.packRepository.setSelected(dataPacks);
             this.worldData.setDataPackConfig(MinecraftServer.getSelectedPacks(this.packRepository));
             this.resources.managers.updateRegistryTags(this.registryAccess());
-            this.getPlayerList().saveAll();
+            // Paper start
+            if (Thread.currentThread() != this.serverThread) {
+                return;
+            }
+            // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements
+            for (ServerPlayer player : this.getPlayerList().getPlayers()) {
+                player.getAdvancements().save();
+            }
+            // Paper end
             this.getPlayerList().reloadResources();
             this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary());
             this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager);
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
index 637846c2dda1cf27c194ca4f16da454a62dc3f4b..648bc209938364a387c3f81dcd073db398e9f864 100644
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
@@ -274,7 +274,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
             long j = Util.getNanos() - i;
             String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
 
-            DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
+            //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init
             if (dedicatedserverproperties.announcePlayerAchievements != null) {
                 ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this);
             }
@@ -404,7 +404,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
             //this.remoteStatusListener.b(); // Paper - don't wait for remote connections
         }
 
-        System.exit(0); // CraftBukkit
+        hasFullyShutdown = true; // Paper
+        System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
     }
 
     @Override
@@ -745,7 +746,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
     @Override
     public void stopServer() {
         super.stopServer();
-        Util.shutdownExecutors();
+        //Util.shutdownExecutors(); // Paper - moved into super
         SkullBlockEntity.clear();
     }
 
diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java
index 72af82374f47a1effc3d2f45beee1f611b4bb4b6..fcc9dd6e1c54e4ca16102150aa4c12ecc7de06df 100644
--- a/src/main/java/net/minecraft/server/level/ChunkMap.java
+++ b/src/main/java/net/minecraft/server/level/ChunkMap.java
@@ -667,6 +667,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
             MutableBoolean mutableboolean = new MutableBoolean();
 
             do {
+                boolean isShuttingDown = level.getServer().hasStopped(); // Paper
                 mutableboolean.setFalse();
                 list.stream().map((playerchunk) -> {
                     CompletableFuture completablefuture;
diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
index d9175bcd1060dbaee90e50466da3c6d816fb614e..602a30f7dc51ffad9e5f5cc0b339108a2225aafd 100644
--- a/src/main/java/net/minecraft/server/players/PlayerList.java
+++ b/src/main/java/net/minecraft/server/players/PlayerList.java
@@ -518,7 +518,7 @@ public abstract class PlayerList {
         this.cserver.getPluginManager().callEvent(playerQuitEvent);
         entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
 
-        entityplayer.doTick(); // SPIGOT-924
+        if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
         // CraftBukkit end
 
         // Paper start - Remove from collideRule team if needed
diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
index 6fefa619299d3202158490630d62c16aef71e831..7a4ade1a4190bf4fbb048919ae2be230f7b80fff 100644
--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
@@ -152,6 +152,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
         try {
             task.run();
         } catch (Exception var3) {
+            if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper
             LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3);
         }
 
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
index ef4e48127168acdd614bbdcac97a98da5240928e..4c93c2ee69c2728d798a750981275f4fc6840525 100644
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
@@ -794,6 +794,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
         try {
             tickConsumer.accept(entity);
         } catch (Throwable throwable) {
+            if (throwable instanceof ThreadDeath) throw throwable; // Paper
             // Paper start - Prevent tile entity and entity crashes
             final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level.getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
             MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
index ebe17598d3ef17cc5f0e99b876ed67936d46060d..c8444f4bfd127b7d8194aaa984505eff249ae094 100644
--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
@@ -1085,6 +1085,7 @@ public class LevelChunk extends ChunkAccess {
 
                         gameprofilerfiller.pop();
                     } catch (Throwable throwable) {
+                        if (throwable instanceof ThreadDeath) throw throwable; // Paper
                         // Paper start - Prevent tile entity and entity crashes
                         final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
                         net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index a35501ea43bf3589b346b1e684c318b44ca57977..4016b31bd020e00c0e79328646f9b5411b312e88 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
@@ -2075,7 +2075,7 @@ public final class CraftServer implements Server {
 
     @Override
     public boolean isPrimaryThread() {
-        return Thread.currentThread().equals(console.serverThread); // Paper - Fix issues with detecting main thread properly
+        return Thread.currentThread().equals(console.serverThread) || Thread.currentThread().equals(net.minecraft.server.MinecraftServer.getServer().shutdownThread); // Paper - Fix issues with detecting main thread properly, the only time Watchdog will be used is during a crash shutdown which is a "try our best" scenario
     }
 
     // Paper start
diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java
index 08e74f41516a545a2371a7418d995ab288831834..cce6886bb3973eed8f0c7ca7b1189547324fd4e2 100644
--- a/src/main/java/org/bukkit/craftbukkit/Main.java
+++ b/src/main/java/org/bukkit/craftbukkit/Main.java
@@ -12,6 +12,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
+import net.minecraft.util.ExceptionCollector;
+import net.minecraft.world.level.lighting.LayerLightEventListener;
 import net.minecrell.terminalconsole.TerminalConsoleAppender; // Paper
 
 public class Main {
@@ -169,6 +171,36 @@ public class Main {
 
         OptionSet options = null;
 
+        // Paper start - preload logger classes to avoid plugins mixing versions
+        tryPreloadClass("org.apache.logging.log4j.core.Core");
+        tryPreloadClass("org.apache.logging.log4j.core.appender.AsyncAppender");
+        tryPreloadClass("org.apache.logging.log4j.core.Appender");
+        tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector");
+        tryPreloadClass("org.apache.logging.log4j.core.Filter");
+        tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler");
+        tryPreloadClass("org.apache.logging.log4j.core.LogEvent");
+        tryPreloadClass("org.apache.logging.log4j.core.Logger");
+        tryPreloadClass("org.apache.logging.log4j.core.LoggerContext");
+        tryPreloadClass("org.apache.logging.log4j.core.LogEventListener");
+        tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent");
+        tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable");
+        tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.Message");
+        tryPreloadClass("org.apache.logging.log4j.message.MessageFactory");
+        tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor");
+        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent");
+        tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil");
+        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement");
+        // Paper end
         try {
             options = parser.parse(args);
         } catch (joptsimple.OptionException ex) {
@@ -264,8 +296,64 @@ public class Main {
             } catch (Throwable t) {
                 t.printStackTrace();
             }
+            // Paper start
+            // load some required classes to avoid errors during shutdown if jar is replaced
+            // also to guarantee our version loads over plugins
+            tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow");
+            tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry");
+            tryPreloadClass("com.google.common.collect.Iterables");
+            for (int i = 1; i <= 15; i++) {
+                tryPreloadClass("com.google.common.collect.Iterables$" + i, false);
+            }
+            tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean");
+            tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt");
+            tryPreloadClass("org.jline.terminal.impl.MouseSupport");
+            tryPreloadClass("org.jline.terminal.impl.MouseSupport$1");
+            tryPreloadClass("org.jline.terminal.Terminal$MouseTracking");
+            tryPreloadClass("co.aikar.timings.TimingHistory");
+            tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport");
+            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext");
+            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11");
+            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$12");
+            tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8");
+            tryPreloadClass("io.netty.util.concurrent.DefaultPromise");
+            tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1");
+            tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil");
+            tryPreloadClass("io.netty.util.internal.SystemPropertyUtil");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4");
+            tryPreloadClass("org.slf4j.helpers.MessageFormatter");
+            tryPreloadClass("org.slf4j.helpers.FormattingTuple");
+            tryPreloadClass("org.slf4j.helpers.BasicMarker");
+            tryPreloadClass("org.slf4j.helpers.Util");
+            tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent");
+            tryPreloadClass("com.destroystokyo.paper.event.entity.EntityRemoveFromWorldEvent");
+            // Minecraft, seen during saving
+            tryPreloadClass(LayerLightEventListener.DummyLightLayerEventListener.class.getName());
+            tryPreloadClass(LayerLightEventListener.class.getName());
+            tryPreloadClass(ExceptionCollector.class.getName());
+            // Paper end
+        }
+    }
+
+    // Paper start
+    private static void tryPreloadClass(String className) {
+        tryPreloadClass(className, true);
+    }
+    private static void tryPreloadClass(String className, boolean printError) {
+        try {
+            Class.forName(className);
+        } catch (ClassNotFoundException e) {
+            if (printError) System.err.println("An expected class  " + className + " was not found for preloading: " + e.getMessage());
         }
     }
+    // Paper end
 
     private static List<String> asList(String... params) {
         return Arrays.asList(params);
diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
index b4a19d80bbf71591f25729fd0e98590350cb31d0..e948ec5a573b22645664eb53bc3e9932246544e4 100644
--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
@@ -12,12 +12,28 @@ public class ServerShutdownThread extends Thread {
     @Override
     public void run() {
         try {
+            // Paper start - try to shutdown on main
+            server.safeShutdown(false, false);
+            for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
+                Thread.sleep(100);
+            }
+            if (server.hasStopped()) {
+                while (!server.hasFullyShutdown) Thread.sleep(1000);
+                return;
+            }
+            // Looks stalled, close async
             org.spigotmc.AsyncCatcher.enabled = false; // Spigot
             org.spigotmc.AsyncCatcher.shuttingDown = true; // Paper
+            server.forceTicks = true;
             this.server.close();
+            while (!server.hasFullyShutdown) Thread.sleep(1000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            // Paper end
         } finally {
+            org.apache.logging.log4j.LogManager.shutdown(); // Paper
             try {
-                net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
             } catch (Exception e) {
             }
         }
diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
index a142a56a920e153ed84c08cece993f10d76f7793..92d97a5810a379b427a99b4c63fb9844d823a84f 100644
--- a/src/main/java/org/spigotmc/RestartCommand.java
+++ b/src/main/java/org/spigotmc/RestartCommand.java
@@ -139,7 +139,7 @@ public class RestartCommand extends Command
     // Paper end
 
     // Paper start - copied from above and modified to return if the hook registered
-    private static boolean addShutdownHook(String restartScript)
+    public static boolean addShutdownHook(String restartScript)
     {
         String[] split = restartScript.split( " " );
         if ( split.length > 0 && new File( split[0] ).isFile() )
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index e62f085de1678568d422ef0eda5b9bfbd8b4d556..f3d60841f7b2e29e667291092b97ec909ba03ab6 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -11,6 +11,7 @@ import org.bukkit.Bukkit;
 public class WatchdogThread extends Thread
 {
 
+    public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
     private static WatchdogThread instance;
     private long timeoutTime;
     private boolean restart;
@@ -39,6 +40,7 @@ public class WatchdogThread extends Thread
     {
         if ( WatchdogThread.instance == null )
         {
+            if (timeoutTime <= 0) timeoutTime = 300; // Paper
             WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
             WatchdogThread.instance.start();
         } else
@@ -70,12 +72,13 @@ public class WatchdogThread extends Thread
             // Paper start
             Logger log = Bukkit.getServer().getLogger();
             long currentTime = WatchdogThread.monotonicMillis();
-            if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
+            MinecraftServer server = MinecraftServer.getServer();
+            if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
             {
-                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+                boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
                 // Don't spam early warning dumps
                 if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
-                if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+                if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
                 lastEarlyWarning = currentTime;
                 if (isLongTimeout) {
                 // Paper end
@@ -117,7 +120,7 @@ public class WatchdogThread extends Thread
                 log.log( Level.SEVERE, "------------------------------" );
                 log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
                 com.destroystokyo.paper.io.chunk.ChunkTaskManager.dumpAllChunkLoadInfo(); // Paper
-                WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
+                WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( server.serverThread.getId(), Integer.MAX_VALUE ), log );
                 log.log( Level.SEVERE, "------------------------------" );
                 //
                 // Paper start - Only print full dump on long timeouts
@@ -137,9 +140,25 @@ public class WatchdogThread extends Thread
 
                 if ( isLongTimeout )
                 {
-                if ( this.restart && !MinecraftServer.getServer().hasStopped() )
+                if ( !server.hasStopped() )
                 {
-                    RestartCommand.restart();
+                    AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+                    AsyncCatcher.shuttingDown = true;
+                    server.forceTicks = true;
+                    if (restart) {
+                        RestartCommand.addShutdownHook( SpigotConfig.restartScript );
+                    }
+                    // try one last chance to safe shutdown on main incase it 'comes back'
+                    server.abnormalExit = true;
+                    server.safeShutdown(false, restart);
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                    if (!server.hasStopped()) {
+                        server.close();
+                    }
                 }
                 break;
                 } // Paper end
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index 3dc317e466e1b93dff030794dd7f29ca1b266778..d285dbec16272db6b8a71865e05924ad66087407 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN" packages="com.mojang.util">
+<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable">
     <Appenders>
         <Queue name="ServerGuiConsole">
             <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />