Watchdog Thread.

By: md_5 <git@md-5.net>

paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch

@@ -107,11 +107,10 @@
      private static final int OVERLOADED_TICKS_THRESHOLD = 20;
      private static final long OVERLOADED_WARNING_INTERVAL_NANOS = 10L * TimeUtil.NANOSECONDS_PER_SECOND;
      private static final int OVERLOADED_TICKS_WARNING_INTERVAL = 100;
-@@ -276,6 +301,19 @@
-     private static final AtomicReference<RuntimeException> fatalException = new AtomicReference();
+@@ -277,6 +302,19 @@
      private final SuppressedExceptionCollector suppressedExceptions;
      private final DiscontinuousFrame tickFrame;
-+
+ 
 +    // CraftBukkit start
 +    public final WorldLoader.DataLoadContext worldLoader;
 +    public org.bukkit.craftbukkit.CraftServer server;
@@ -124,9 +123,10 @@
 +    public Commands vanillaCommandDispatcher;
 +    private boolean forceTicks;
 +    // CraftBukkit end
- 
++
      public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
          AtomicReference<S> atomicreference = new AtomicReference();
+         Thread thread = new Thread(() -> {
 @@ -290,14 +328,14 @@
              thread.setPriority(8);
          }
@@ -620,11 +620,10 @@
          if (flush) {
              Iterator iterator1 = this.getAllLevels().iterator();
  
-@@ -626,20 +890,42 @@
-     @Override
+@@ -627,19 +891,41 @@
      public void close() {
          this.stopServer();
-+    }
+     }
 +
 +    // CraftBukkit start
 +    private boolean hasStopped = false;
@@ -633,7 +632,7 @@
 +        synchronized (this.stopLock) {
 +            return this.hasStopped;
 +        }
-     }
++    }
 +    // CraftBukkit end
  
      public void stopServer() {
@@ -688,10 +687,11 @@
                  this.nextTickTimeNanos += i;
  
                  try {
-@@ -830,6 +1118,12 @@
+@@ -830,6 +1118,13 @@
                      this.services.profileCache().clearExecutor();
                  }
  
++                org.spigotmc.WatchdogThread.doStop(); // Spigot
 +                // CraftBukkit start - Restore terminal to original settings
 +                try {
 +                    this.reader.getTerminal().restore();
@@ -701,7 +701,7 @@
                  this.onServerExit();
              }
  
-@@ -889,9 +1183,16 @@
+@@ -889,9 +1184,16 @@
      }
  
      private boolean haveTime() {
@@ -719,7 +719,7 @@
      public static boolean throwIfFatalException() {
          RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
  
-@@ -903,7 +1204,7 @@
+@@ -903,7 +1205,7 @@
      }
  
      public static void setFatalException(RuntimeException exception) {
@@ -728,7 +728,7 @@
      }
  
      @Override
-@@ -977,7 +1278,7 @@
+@@ -977,7 +1279,7 @@
          }
      }
  
@@ -737,7 +737,15 @@
          Profiler.get().incrementCounter("runTask");
          super.doRunTask(ticktask);
      }
-@@ -1041,11 +1342,13 @@
+@@ -1025,6 +1327,7 @@
+     }
+ 
+     public void tickServer(BooleanSupplier shouldKeepTicking) {
++        org.spigotmc.WatchdogThread.tick(); // Spigot
+         long i = Util.getNanos();
+         int j = this.pauseWhileEmptySeconds() * 20;
+ 
+@@ -1041,11 +1344,13 @@
                      this.autoSave();
                  }
  
@@ -751,7 +759,7 @@
          ++this.tickCount;
          this.tickRateManager.tick();
          this.tickChildren(shouldKeepTicking);
-@@ -1055,7 +1358,7 @@
+@@ -1055,7 +1360,7 @@
          }
  
          --this.ticksUntilAutosave;
@@ -760,7 +768,7 @@
              this.autoSave();
          }
  
-@@ -1071,10 +1374,13 @@
+@@ -1071,10 +1376,13 @@
          this.smoothedTickTimeMillis = this.smoothedTickTimeMillis * 0.8F + (float) k / (float) TimeUtil.NANOSECONDS_PER_MILLISECOND * 0.19999999F;
          this.logTickMethodTime(i);
          gameprofilerfiller.pop();
@@ -775,7 +783,7 @@
          MinecraftServer.LOGGER.debug("Autosave started");
          ProfilerFiller gameprofilerfiller = Profiler.get();
  
-@@ -1082,6 +1388,7 @@
+@@ -1082,6 +1390,7 @@
          this.saveEverything(true, false, false);
          gameprofilerfiller.pop();
          MinecraftServer.LOGGER.debug("Autosave finished");
@@ -783,7 +791,7 @@
      }
  
      private void logTickMethodTime(long tickStartTime) {
-@@ -1154,11 +1461,34 @@
+@@ -1154,11 +1463,34 @@
          this.getPlayerList().getPlayers().forEach((entityplayer) -> {
              entityplayer.connection.suspendFlushing();
          });
@@ -796,7 +804,7 @@
 +        SpigotTimings.commandFunctionsTimer.stopTiming(); // Spigot
          gameprofilerfiller.popPush("levels");
          Iterator iterator = this.getAllLevels().iterator();
- 
++
 +        // CraftBukkit start
 +        // Run tasks that are waiting on processing
 +        SpigotTimings.processQueueTimer.startTiming(); // Spigot
@@ -804,7 +812,7 @@
 +            this.processQueue.remove().run();
 +        }
 +        SpigotTimings.processQueueTimer.stopTiming(); // Spigot
-+
+ 
 +        SpigotTimings.timeUpdateTimer.startTiming(); // Spigot
 +        // Send time updates to everyone, it will get the right time from the world the player is in.
 +        if (this.tickCount % 20 == 0) {
@@ -818,7 +826,7 @@
          while (iterator.hasNext()) {
              ServerLevel worldserver = (ServerLevel) iterator.next();
  
-@@ -1167,16 +1497,20 @@
+@@ -1167,16 +1499,20 @@
  
                  return s + " " + String.valueOf(worldserver.dimension().location());
              });
@@ -839,7 +847,7 @@
              } catch (Throwable throwable) {
                  CrashReport crashreport = CrashReport.forThrowable(throwable, "Exception ticking world");
  
-@@ -1189,18 +1523,24 @@
+@@ -1189,18 +1525,24 @@
          }
  
          gameprofilerfiller.popPush("connection");
@@ -864,7 +872,7 @@
  
          gameprofilerfiller.popPush("send chunks");
          iterator = this.playerList.getPlayers().iterator();
-@@ -1265,7 +1605,23 @@
+@@ -1265,7 +1607,23 @@
      @Nullable
      public ServerLevel getLevel(ResourceKey<Level> key) {
          return (ServerLevel) this.levels.get(key);
@@ -888,7 +896,7 @@
  
      public Set<ResourceKey<Level>> levelKeys() {
          return this.levels.keySet();
-@@ -1296,7 +1652,7 @@
+@@ -1296,7 +1654,7 @@
  
      @DontObfuscate
      public String getServerModName() {
@@ -897,7 +905,7 @@
      }
  
      public SystemReport fillSystemReport(SystemReport details) {
-@@ -1634,11 +1990,11 @@
+@@ -1634,11 +1992,11 @@
  
      public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
          CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@@ -911,7 +919,7 @@
          }, this).thenCompose((immutablelist) -> {
              MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
              List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
-@@ -1654,6 +2010,7 @@
+@@ -1654,6 +2012,7 @@
          }).thenAcceptAsync((minecraftserver_reloadableresources) -> {
              this.resources.close();
              this.resources = minecraftserver_reloadableresources;
@@ -919,7 +927,7 @@
              this.packRepository.setSelected(dataPacks);
              WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
  
-@@ -1952,7 +2309,7 @@
+@@ -1952,7 +2311,7 @@
              final List<String> list = Lists.newArrayList();
              final GameRules gamerules = this.getGameRules();
  
@@ -928,7 +936,7 @@
                  @Override
                  public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
                      list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
-@@ -2058,7 +2415,7 @@
+@@ -2058,7 +2417,7 @@
              try {
                  label51:
                  {
@@ -937,7 +945,7 @@
  
                      try {
                          arraylist = Lists.newArrayList(NativeModuleLister.listModules());
-@@ -2108,6 +2465,22 @@
+@@ -2108,6 +2467,22 @@
  
      }
  
@@ -960,7 +968,7 @@
      private ProfilerFiller createProfiler() {
          if (this.willStartRecordingMetrics) {
              this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
-@@ -2234,6 +2607,11 @@
+@@ -2234,6 +2609,11 @@
          }
  
      }

paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch

@@ -190,6 +190,15 @@
              }
  
              if (dedicatedserverproperties.enableQuery) {
+@@ -197,7 +276,7 @@
+                 this.rconThread = RconThread.create(this);
+             }
+ 
+-            if (this.getMaxTickLength() > 0L) {
++            if (false && this.getMaxTickLength() > 0L) {  // Spigot - disable
+                 Thread thread1 = new Thread(new ServerWatchdog(this));
+ 
+                 thread1.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandlerWithName(DedicatedServer.LOGGER));
 @@ -293,6 +372,7 @@
              this.queryThreadGs4.stop();
          }

paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java

@@ -2139,7 +2139,7 @@ public final class CraftServer implements Server {
 
     @Override
     public boolean isPrimaryThread() {
-        return Thread.currentThread().equals(this.console.serverThread) || this.console.hasStopped(); // All bets are off if we have shut down (e.g. due to watchdog)
+        return Thread.currentThread().equals(this.console.serverThread) || this.console.hasStopped() || !org.spigotmc.AsyncCatcher.enabled; // All bets are off if we have shut down (e.g. due to watchdog)
     }
 
     @Override
@@ -2580,6 +2580,11 @@ public final class CraftServer implements Server {
         {
             return org.spigotmc.SpigotConfig.config;
         }
+
+        @Override
+        public void restart() {
+            org.spigotmc.RestartCommand.restart();
+        }
     };
 
     public org.bukkit.Server.Spigot spigot()

paper-server/src/main/java/org/spigotmc/RestartCommand.java

@@ -0,0 +1,131 @@
+package org.spigotmc;
+
+import java.io.File;
+import java.util.List;
+import net.minecraft.server.MinecraftServer;
+import net.minecraft.server.level.ServerPlayer;
+import org.bukkit.command.Command;
+import org.bukkit.command.CommandSender;
+import org.bukkit.craftbukkit.util.CraftChatMessage;
+
+public class RestartCommand extends Command
+{
+
+    public RestartCommand(String name)
+    {
+        super( name );
+        this.description = "Restarts the server";
+        this.usageMessage = "/restart";
+        this.setPermission( "bukkit.command.restart" );
+    }
+
+    @Override
+    public boolean execute(CommandSender sender, String currentAlias, String[] args)
+    {
+        if ( this.testPermission( sender ) )
+        {
+            MinecraftServer.getServer().processQueue.add( new Runnable()
+            {
+                @Override
+                public void run()
+                {
+                    RestartCommand.restart();
+                }
+            } );
+        }
+        return true;
+    }
+
+    public static void restart()
+    {
+        RestartCommand.restart( SpigotConfig.restartScript );
+    }
+
+    private static void restart(final String restartScript)
+    {
+        AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+        try
+        {
+            String[] split = restartScript.split( " " );
+            if ( split.length > 0 && new File( split[0] ).isFile() )
+            {
+                System.out.println( "Attempting to restart with " + restartScript );
+
+                // Disable Watchdog
+                WatchdogThread.doStop();
+
+                // Kick all players
+                for ( ServerPlayer p : (List<ServerPlayer>) MinecraftServer.getServer().getPlayerList().players )
+                {
+                    p.connection.disconnect( CraftChatMessage.fromStringOrEmpty( SpigotConfig.restartMessage, true ) );
+                }
+                // Give the socket a chance to send the packets
+                try
+                {
+                    Thread.sleep( 100 );
+                } catch ( InterruptedException ex )
+                {
+                }
+                // Close the socket so we can rebind with the new process
+                MinecraftServer.getServer().getConnection().stop();
+
+                // Give time for it to kick in
+                try
+                {
+                    Thread.sleep( 100 );
+                } catch ( InterruptedException ex )
+                {
+                }
+
+                // Actually shutdown
+                try
+                {
+                    MinecraftServer.getServer().close();
+                } catch ( Throwable t )
+                {
+                }
+
+                // This will be done AFTER the server has completely halted
+                Thread shutdownHook = new Thread()
+                {
+                    @Override
+                    public void run()
+                    {
+                        try
+                        {
+                            String os = System.getProperty( "os.name" ).toLowerCase(java.util.Locale.ENGLISH);
+                            if ( os.contains( "win" ) )
+                            {
+                                Runtime.getRuntime().exec( "cmd /c start " + restartScript );
+                            } else
+                            {
+                                Runtime.getRuntime().exec( "sh " + restartScript );
+                            }
+                        } catch ( Exception e )
+                        {
+                            e.printStackTrace();
+                        }
+                    }
+                };
+
+                shutdownHook.setDaemon( true );
+                Runtime.getRuntime().addShutdownHook( shutdownHook );
+            } else
+            {
+                System.out.println( "Startup script '" + SpigotConfig.restartScript + "' does not exist! Stopping server." );
+
+                // Actually shutdown
+                try
+                {
+                    MinecraftServer.getServer().close();
+                } catch ( Throwable t )
+                {
+                }
+            }
+            System.exit( 0 );
+        } catch ( Exception ex )
+        {
+            ex.printStackTrace();
+        }
+    }
+}

paper-server/src/main/java/org/spigotmc/SpigotConfig.java

@@ -200,4 +200,18 @@ public class SpigotConfig
         SpigotConfig.outdatedClientMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.outdated-client", SpigotConfig.outdatedClientMessage ) );
         SpigotConfig.outdatedServerMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.outdated-server", SpigotConfig.outdatedServerMessage ) );
     }
+
+    public static int timeoutTime = 60;
+    public static boolean restartOnCrash = true;
+    public static String restartScript = "./start.sh";
+    public static String restartMessage;
+    private static void watchdog()
+    {
+        SpigotConfig.timeoutTime = SpigotConfig.getInt( "settings.timeout-time", SpigotConfig.timeoutTime );
+        SpigotConfig.restartOnCrash = SpigotConfig.getBoolean( "settings.restart-on-crash", SpigotConfig.restartOnCrash );
+        SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
+        SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
+        SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
+        WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
+    }
 }

paper-server/src/main/java/org/spigotmc/WatchdogThread.java

@@ -0,0 +1,131 @@
+package org.spigotmc;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import net.minecraft.server.MinecraftServer;
+import org.bukkit.Bukkit;
+
+public class WatchdogThread extends Thread
+{
+
+    private static WatchdogThread instance;
+    private long timeoutTime;
+    private boolean restart;
+    private volatile long lastTick;
+    private volatile boolean stopping;
+
+    private WatchdogThread(long timeoutTime, boolean restart)
+    {
+        super( "Spigot Watchdog Thread" );
+        this.timeoutTime = timeoutTime;
+        this.restart = restart;
+    }
+
+    private static long monotonicMillis()
+    {
+        return System.nanoTime() / 1000000L;
+    }
+
+    public static void doStart(int timeoutTime, boolean restart)
+    {
+        if ( WatchdogThread.instance == null )
+        {
+            WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
+            WatchdogThread.instance.start();
+        } else
+        {
+            WatchdogThread.instance.timeoutTime = timeoutTime * 1000L;
+            WatchdogThread.instance.restart = restart;
+        }
+    }
+
+    public static void tick()
+    {
+        WatchdogThread.instance.lastTick = WatchdogThread.monotonicMillis();
+    }
+
+    public static void doStop()
+    {
+        if ( WatchdogThread.instance != null )
+        {
+            WatchdogThread.instance.stopping = true;
+        }
+    }
+
+    @Override
+    public void run()
+    {
+        while ( !this.stopping )
+        {
+            //
+            if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime )
+            {
+                Logger log = Bukkit.getServer().getLogger();
+                log.log( Level.SEVERE, "------------------------------" );
+                log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Spigot bug." );
+                log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
+                log.log( Level.SEVERE, "\t *Especially* if it looks like HTTP or MySQL operations are occurring" );
+                log.log( Level.SEVERE, "If you see a world save or edit, then it means you did far more than your server can handle at once" );
+                log.log( Level.SEVERE, "\t If this is the case, consider increasing timeout-time in spigot.yml but note that this will replace the crash with LARGE lag spikes" );
+                log.log( Level.SEVERE, "If you are unsure or still think this is a Spigot bug, please report to https://www.spigotmc.org/" );
+                log.log( Level.SEVERE, "Be sure to include ALL relevant console errors and Minecraft crash reports" );
+                log.log( Level.SEVERE, "Spigot version: " + Bukkit.getServer().getVersion() );
+                //
+                log.log( Level.SEVERE, "------------------------------" );
+                log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Spigot!):" );
+                WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
+                log.log( Level.SEVERE, "------------------------------" );
+                //
+                log.log( Level.SEVERE, "Entire Thread Dump:" );
+                ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
+                for ( ThreadInfo thread : threads )
+                {
+                    WatchdogThread.dumpThread( thread, log );
+                }
+                log.log( Level.SEVERE, "------------------------------" );
+
+                if ( this.restart && !MinecraftServer.getServer().hasStopped() )
+                {
+                    RestartCommand.restart();
+                }
+                break;
+            }
+
+            try
+            {
+                sleep( 10000 );
+            } catch ( InterruptedException ex )
+            {
+                this.interrupt();
+            }
+        }
+    }
+
+    private static void dumpThread(ThreadInfo thread, Logger log)
+    {
+        log.log( Level.SEVERE, "------------------------------" );
+        //
+        log.log( Level.SEVERE, "Current Thread: " + thread.getThreadName() );
+        log.log( Level.SEVERE, "\tPID: " + thread.getThreadId()
+                + " | Suspended: " + thread.isSuspended()
+                + " | Native: " + thread.isInNative()
+                + " | State: " + thread.getThreadState() );
+        if ( thread.getLockedMonitors().length != 0 )
+        {
+            log.log( Level.SEVERE, "\tThread is waiting on monitor(s):" );
+            for ( MonitorInfo monitor : thread.getLockedMonitors() )
+            {
+                log.log( Level.SEVERE, "\t\tLocked on:" + monitor.getLockedStackFrame() );
+            }
+        }
+        log.log( Level.SEVERE, "\tStack:" );
+        //
+        for ( StackTraceElement stack : thread.getStackTrace() )
+        {
+            log.log( Level.SEVERE, "\t\t" + stack );
+        }
+    }
+}