diff --git a/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch b/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch index 97079268da..82932a62e8 100644 --- a/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch +++ b/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch @@ -107,11 +107,10 @@ private static final int OVERLOADED_TICKS_THRESHOLD = 20; private static final long OVERLOADED_WARNING_INTERVAL_NANOS = 10L * TimeUtil.NANOSECONDS_PER_SECOND; private static final int OVERLOADED_TICKS_WARNING_INTERVAL = 100; -@@ -276,6 +301,19 @@ - private static final AtomicReference fatalException = new AtomicReference(); +@@ -277,6 +302,19 @@ private final SuppressedExceptionCollector suppressedExceptions; private final DiscontinuousFrame tickFrame; -+ + + // CraftBukkit start + public final WorldLoader.DataLoadContext worldLoader; + public org.bukkit.craftbukkit.CraftServer server; @@ -124,9 +123,10 @@ + public Commands vanillaCommandDispatcher; + private boolean forceTicks; + // CraftBukkit end - ++ public static S spin(Function serverFactory) { AtomicReference atomicreference = new AtomicReference(); + Thread thread = new Thread(() -> { @@ -290,14 +328,14 @@ thread.setPriority(8); } @@ -620,11 +620,10 @@ if (flush) { Iterator iterator1 = this.getAllLevels().iterator(); -@@ -626,20 +890,42 @@ - @Override +@@ -627,19 +891,41 @@ public void close() { this.stopServer(); -+ } + } + + // CraftBukkit start + private boolean hasStopped = false; @@ -633,7 +632,7 @@ + synchronized (this.stopLock) { + return this.hasStopped; + } - } ++ } + // CraftBukkit end public void stopServer() { @@ -688,10 +687,11 @@ this.nextTickTimeNanos += i; try { -@@ -830,6 +1118,12 @@ +@@ -830,6 +1118,13 @@ this.services.profileCache().clearExecutor(); } ++ org.spigotmc.WatchdogThread.doStop(); // Spigot + // CraftBukkit start - Restore terminal to original settings + try { + this.reader.getTerminal().restore(); @@ -701,7 +701,7 @@ this.onServerExit(); } -@@ -889,9 +1183,16 @@ +@@ -889,9 +1184,16 @@ } private boolean haveTime() { @@ -719,7 +719,7 @@ public static boolean throwIfFatalException() { RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get(); -@@ -903,7 +1204,7 @@ +@@ -903,7 +1205,7 @@ } public static void setFatalException(RuntimeException exception) { @@ -728,7 +728,7 @@ } @Override -@@ -977,7 +1278,7 @@ +@@ -977,7 +1279,7 @@ } } @@ -737,7 +737,15 @@ Profiler.get().incrementCounter("runTask"); super.doRunTask(ticktask); } -@@ -1041,11 +1342,13 @@ +@@ -1025,6 +1327,7 @@ + } + + public void tickServer(BooleanSupplier shouldKeepTicking) { ++ org.spigotmc.WatchdogThread.tick(); // Spigot + long i = Util.getNanos(); + int j = this.pauseWhileEmptySeconds() * 20; + +@@ -1041,11 +1344,13 @@ this.autoSave(); } @@ -751,7 +759,7 @@ ++this.tickCount; this.tickRateManager.tick(); this.tickChildren(shouldKeepTicking); -@@ -1055,7 +1358,7 @@ +@@ -1055,7 +1360,7 @@ } --this.ticksUntilAutosave; @@ -760,7 +768,7 @@ this.autoSave(); } -@@ -1071,10 +1374,13 @@ +@@ -1071,10 +1376,13 @@ this.smoothedTickTimeMillis = this.smoothedTickTimeMillis * 0.8F + (float) k / (float) TimeUtil.NANOSECONDS_PER_MILLISECOND * 0.19999999F; this.logTickMethodTime(i); gameprofilerfiller.pop(); @@ -775,7 +783,7 @@ MinecraftServer.LOGGER.debug("Autosave started"); ProfilerFiller gameprofilerfiller = Profiler.get(); -@@ -1082,6 +1388,7 @@ +@@ -1082,6 +1390,7 @@ this.saveEverything(true, false, false); gameprofilerfiller.pop(); MinecraftServer.LOGGER.debug("Autosave finished"); @@ -783,7 +791,7 @@ } private void logTickMethodTime(long tickStartTime) { -@@ -1154,11 +1461,34 @@ +@@ -1154,11 +1463,34 @@ this.getPlayerList().getPlayers().forEach((entityplayer) -> { entityplayer.connection.suspendFlushing(); }); @@ -796,7 +804,7 @@ + SpigotTimings.commandFunctionsTimer.stopTiming(); // Spigot gameprofilerfiller.popPush("levels"); Iterator iterator = this.getAllLevels().iterator(); - ++ + // CraftBukkit start + // Run tasks that are waiting on processing + SpigotTimings.processQueueTimer.startTiming(); // Spigot @@ -804,7 +812,7 @@ + this.processQueue.remove().run(); + } + SpigotTimings.processQueueTimer.stopTiming(); // Spigot -+ + + SpigotTimings.timeUpdateTimer.startTiming(); // Spigot + // Send time updates to everyone, it will get the right time from the world the player is in. + if (this.tickCount % 20 == 0) { @@ -818,7 +826,7 @@ while (iterator.hasNext()) { ServerLevel worldserver = (ServerLevel) iterator.next(); -@@ -1167,16 +1497,20 @@ +@@ -1167,16 +1499,20 @@ return s + " " + String.valueOf(worldserver.dimension().location()); }); @@ -839,7 +847,7 @@ } catch (Throwable throwable) { CrashReport crashreport = CrashReport.forThrowable(throwable, "Exception ticking world"); -@@ -1189,18 +1523,24 @@ +@@ -1189,18 +1525,24 @@ } gameprofilerfiller.popPush("connection"); @@ -864,7 +872,7 @@ gameprofilerfiller.popPush("send chunks"); iterator = this.playerList.getPlayers().iterator(); -@@ -1265,7 +1605,23 @@ +@@ -1265,7 +1607,23 @@ @Nullable public ServerLevel getLevel(ResourceKey key) { return (ServerLevel) this.levels.get(key); @@ -888,7 +896,7 @@ public Set> levelKeys() { return this.levels.keySet(); -@@ -1296,7 +1652,7 @@ +@@ -1296,7 +1654,7 @@ @DontObfuscate public String getServerModName() { @@ -897,7 +905,7 @@ } public SystemReport fillSystemReport(SystemReport details) { -@@ -1634,11 +1990,11 @@ +@@ -1634,11 +1992,11 @@ public CompletableFuture reloadResources(Collection dataPacks) { CompletableFuture completablefuture = CompletableFuture.supplyAsync(() -> { @@ -911,7 +919,7 @@ }, this).thenCompose((immutablelist) -> { MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist); List> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess()); -@@ -1654,6 +2010,7 @@ +@@ -1654,6 +2012,7 @@ }).thenAcceptAsync((minecraftserver_reloadableresources) -> { this.resources.close(); this.resources = minecraftserver_reloadableresources; @@ -919,7 +927,7 @@ this.packRepository.setSelected(dataPacks); WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures()); -@@ -1952,7 +2309,7 @@ +@@ -1952,7 +2311,7 @@ final List list = Lists.newArrayList(); final GameRules gamerules = this.getGameRules(); @@ -928,7 +936,7 @@ @Override public > void visit(GameRules.Key key, GameRules.Type type) { list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key))); -@@ -2058,7 +2415,7 @@ +@@ -2058,7 +2417,7 @@ try { label51: { @@ -937,7 +945,7 @@ try { arraylist = Lists.newArrayList(NativeModuleLister.listModules()); -@@ -2108,6 +2465,22 @@ +@@ -2108,6 +2467,22 @@ } @@ -960,7 +968,7 @@ private ProfilerFiller createProfiler() { if (this.willStartRecordingMetrics) { this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> { -@@ -2234,6 +2607,11 @@ +@@ -2234,6 +2609,11 @@ } } diff --git a/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch b/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch index 06c239d2bf..2e3ce8117f 100644 --- a/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch +++ b/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch @@ -190,6 +190,15 @@ } if (dedicatedserverproperties.enableQuery) { +@@ -197,7 +276,7 @@ + this.rconThread = RconThread.create(this); + } + +- if (this.getMaxTickLength() > 0L) { ++ if (false && this.getMaxTickLength() > 0L) { // Spigot - disable + Thread thread1 = new Thread(new ServerWatchdog(this)); + + thread1.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandlerWithName(DedicatedServer.LOGGER)); @@ -293,6 +372,7 @@ this.queryThreadGs4.stop(); } diff --git a/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java index bf626e0634..9aa353e33e 100644 --- a/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java +++ b/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java @@ -2139,7 +2139,7 @@ public final class CraftServer implements Server { @Override public boolean isPrimaryThread() { - return Thread.currentThread().equals(this.console.serverThread) || this.console.hasStopped(); // All bets are off if we have shut down (e.g. due to watchdog) + return Thread.currentThread().equals(this.console.serverThread) || this.console.hasStopped() || !org.spigotmc.AsyncCatcher.enabled; // All bets are off if we have shut down (e.g. due to watchdog) } @Override @@ -2580,6 +2580,11 @@ public final class CraftServer implements Server { { return org.spigotmc.SpigotConfig.config; } + + @Override + public void restart() { + org.spigotmc.RestartCommand.restart(); + } }; public org.bukkit.Server.Spigot spigot() diff --git a/paper-server/src/main/java/org/spigotmc/RestartCommand.java b/paper-server/src/main/java/org/spigotmc/RestartCommand.java new file mode 100644 index 0000000000..de8c703803 --- /dev/null +++ b/paper-server/src/main/java/org/spigotmc/RestartCommand.java @@ -0,0 +1,131 @@ +package org.spigotmc; + +import java.io.File; +import java.util.List; +import net.minecraft.server.MinecraftServer; +import net.minecraft.server.level.ServerPlayer; +import org.bukkit.command.Command; +import org.bukkit.command.CommandSender; +import org.bukkit.craftbukkit.util.CraftChatMessage; + +public class RestartCommand extends Command +{ + + public RestartCommand(String name) + { + super( name ); + this.description = "Restarts the server"; + this.usageMessage = "/restart"; + this.setPermission( "bukkit.command.restart" ); + } + + @Override + public boolean execute(CommandSender sender, String currentAlias, String[] args) + { + if ( this.testPermission( sender ) ) + { + MinecraftServer.getServer().processQueue.add( new Runnable() + { + @Override + public void run() + { + RestartCommand.restart(); + } + } ); + } + return true; + } + + public static void restart() + { + RestartCommand.restart( SpigotConfig.restartScript ); + } + + private static void restart(final String restartScript) + { + AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us + try + { + String[] split = restartScript.split( " " ); + if ( split.length > 0 && new File( split[0] ).isFile() ) + { + System.out.println( "Attempting to restart with " + restartScript ); + + // Disable Watchdog + WatchdogThread.doStop(); + + // Kick all players + for ( ServerPlayer p : (List) MinecraftServer.getServer().getPlayerList().players ) + { + p.connection.disconnect( CraftChatMessage.fromStringOrEmpty( SpigotConfig.restartMessage, true ) ); + } + // Give the socket a chance to send the packets + try + { + Thread.sleep( 100 ); + } catch ( InterruptedException ex ) + { + } + // Close the socket so we can rebind with the new process + MinecraftServer.getServer().getConnection().stop(); + + // Give time for it to kick in + try + { + Thread.sleep( 100 ); + } catch ( InterruptedException ex ) + { + } + + // Actually shutdown + try + { + MinecraftServer.getServer().close(); + } catch ( Throwable t ) + { + } + + // This will be done AFTER the server has completely halted + Thread shutdownHook = new Thread() + { + @Override + public void run() + { + try + { + String os = System.getProperty( "os.name" ).toLowerCase(java.util.Locale.ENGLISH); + if ( os.contains( "win" ) ) + { + Runtime.getRuntime().exec( "cmd /c start " + restartScript ); + } else + { + Runtime.getRuntime().exec( "sh " + restartScript ); + } + } catch ( Exception e ) + { + e.printStackTrace(); + } + } + }; + + shutdownHook.setDaemon( true ); + Runtime.getRuntime().addShutdownHook( shutdownHook ); + } else + { + System.out.println( "Startup script '" + SpigotConfig.restartScript + "' does not exist! Stopping server." ); + + // Actually shutdown + try + { + MinecraftServer.getServer().close(); + } catch ( Throwable t ) + { + } + } + System.exit( 0 ); + } catch ( Exception ex ) + { + ex.printStackTrace(); + } + } +} diff --git a/paper-server/src/main/java/org/spigotmc/SpigotConfig.java b/paper-server/src/main/java/org/spigotmc/SpigotConfig.java index 0c745f23a6..d8d7c94ba1 100644 --- a/paper-server/src/main/java/org/spigotmc/SpigotConfig.java +++ b/paper-server/src/main/java/org/spigotmc/SpigotConfig.java @@ -200,4 +200,18 @@ public class SpigotConfig SpigotConfig.outdatedClientMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.outdated-client", SpigotConfig.outdatedClientMessage ) ); SpigotConfig.outdatedServerMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.outdated-server", SpigotConfig.outdatedServerMessage ) ); } + + public static int timeoutTime = 60; + public static boolean restartOnCrash = true; + public static String restartScript = "./start.sh"; + public static String restartMessage; + private static void watchdog() + { + SpigotConfig.timeoutTime = SpigotConfig.getInt( "settings.timeout-time", SpigotConfig.timeoutTime ); + SpigotConfig.restartOnCrash = SpigotConfig.getBoolean( "settings.restart-on-crash", SpigotConfig.restartOnCrash ); + SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript ); + SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) ); + SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) ); + WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); + } } diff --git a/paper-server/src/main/java/org/spigotmc/WatchdogThread.java b/paper-server/src/main/java/org/spigotmc/WatchdogThread.java new file mode 100644 index 0000000000..065ed3823f --- /dev/null +++ b/paper-server/src/main/java/org/spigotmc/WatchdogThread.java @@ -0,0 +1,131 @@ +package org.spigotmc; + +import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; +import java.lang.management.ThreadInfo; +import java.util.logging.Level; +import java.util.logging.Logger; +import net.minecraft.server.MinecraftServer; +import org.bukkit.Bukkit; + +public class WatchdogThread extends Thread +{ + + private static WatchdogThread instance; + private long timeoutTime; + private boolean restart; + private volatile long lastTick; + private volatile boolean stopping; + + private WatchdogThread(long timeoutTime, boolean restart) + { + super( "Spigot Watchdog Thread" ); + this.timeoutTime = timeoutTime; + this.restart = restart; + } + + private static long monotonicMillis() + { + return System.nanoTime() / 1000000L; + } + + public static void doStart(int timeoutTime, boolean restart) + { + if ( WatchdogThread.instance == null ) + { + WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart ); + WatchdogThread.instance.start(); + } else + { + WatchdogThread.instance.timeoutTime = timeoutTime * 1000L; + WatchdogThread.instance.restart = restart; + } + } + + public static void tick() + { + WatchdogThread.instance.lastTick = WatchdogThread.monotonicMillis(); + } + + public static void doStop() + { + if ( WatchdogThread.instance != null ) + { + WatchdogThread.instance.stopping = true; + } + } + + @Override + public void run() + { + while ( !this.stopping ) + { + // + if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime ) + { + Logger log = Bukkit.getServer().getLogger(); + log.log( Level.SEVERE, "------------------------------" ); + log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Spigot bug." ); + log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" ); + log.log( Level.SEVERE, "\t *Especially* if it looks like HTTP or MySQL operations are occurring" ); + log.log( Level.SEVERE, "If you see a world save or edit, then it means you did far more than your server can handle at once" ); + log.log( Level.SEVERE, "\t If this is the case, consider increasing timeout-time in spigot.yml but note that this will replace the crash with LARGE lag spikes" ); + log.log( Level.SEVERE, "If you are unsure or still think this is a Spigot bug, please report to https://www.spigotmc.org/" ); + log.log( Level.SEVERE, "Be sure to include ALL relevant console errors and Minecraft crash reports" ); + log.log( Level.SEVERE, "Spigot version: " + Bukkit.getServer().getVersion() ); + // + log.log( Level.SEVERE, "------------------------------" ); + log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Spigot!):" ); + WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); + log.log( Level.SEVERE, "------------------------------" ); + // + log.log( Level.SEVERE, "Entire Thread Dump:" ); + ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true ); + for ( ThreadInfo thread : threads ) + { + WatchdogThread.dumpThread( thread, log ); + } + log.log( Level.SEVERE, "------------------------------" ); + + if ( this.restart && !MinecraftServer.getServer().hasStopped() ) + { + RestartCommand.restart(); + } + break; + } + + try + { + sleep( 10000 ); + } catch ( InterruptedException ex ) + { + this.interrupt(); + } + } + } + + private static void dumpThread(ThreadInfo thread, Logger log) + { + log.log( Level.SEVERE, "------------------------------" ); + // + log.log( Level.SEVERE, "Current Thread: " + thread.getThreadName() ); + log.log( Level.SEVERE, "\tPID: " + thread.getThreadId() + + " | Suspended: " + thread.isSuspended() + + " | Native: " + thread.isInNative() + + " | State: " + thread.getThreadState() ); + if ( thread.getLockedMonitors().length != 0 ) + { + log.log( Level.SEVERE, "\tThread is waiting on monitor(s):" ); + for ( MonitorInfo monitor : thread.getLockedMonitors() ) + { + log.log( Level.SEVERE, "\t\tLocked on:" + monitor.getLockedStackFrame() ); + } + } + log.log( Level.SEVERE, "\tStack:" ); + // + for ( StackTraceElement stack : thread.getStackTrace() ) + { + log.log( Level.SEVERE, "\t\t" + stack ); + } + } +}