Wait for Async Tasks during shutdown

Server.reload() had this logic to give time for tasks to shutdown,
however shutdown did not...

Adds a 5 second grace period for any async tasks to finish and warns
if any are still running after that delay just as reload does.

paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch

@@ -698,10 +698,12 @@
          if (flush) {
              Iterator iterator1 = this.getAllLevels().iterator();
  
-@@ -628,18 +927,44 @@
+@@ -626,20 +925,47 @@
+     @Override
+     public void close() {
          this.stopServer();
-     }
++    }
- 
++
 +    // CraftBukkit start
 +    private boolean hasStopped = false;
 +    private boolean hasLoggedStop = false; // Paper - Debugging
@@ -710,9 +712,9 @@
 +        synchronized (this.stopLock) {
 +            return this.hasStopped;
 +        }
-+    }
+     }
 +    // CraftBukkit end
-+
+ 
      public void stopServer() {
 +        // CraftBukkit start - prevent double stopping on multiple threads
 +        synchronized(this.stopLock) {
@@ -730,6 +732,7 @@
 +        // CraftBukkit start
 +        if (this.server != null) {
 +            this.server.disablePlugins();
++            this.server.waitForAsyncTasksShutdown(); // Paper - Wait for Async Tasks during shutdown
 +        }
 +        // CraftBukkit end
 +        if (io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper != null) io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper.shutdown(); // Paper - Plugin remapping
@@ -744,7 +747,7 @@
          }
  
          MinecraftServer.LOGGER.info("Saving worlds");
-@@ -693,6 +1018,15 @@
+@@ -693,6 +1019,15 @@
          } catch (IOException ioexception1) {
              MinecraftServer.LOGGER.error("Failed to unlock level {}", this.storageSource.getLevelId(), ioexception1);
          }
@@ -760,7 +763,7 @@
  
      }
  
-@@ -709,6 +1043,14 @@
+@@ -709,6 +1044,14 @@
      }
  
      public void halt(boolean waitForShutdown) {
@@ -775,7 +778,7 @@
          this.running = false;
          if (waitForShutdown) {
              try {
-@@ -720,6 +1062,64 @@
+@@ -720,6 +1063,64 @@
  
      }
  
@@ -840,7 +843,7 @@
      protected void runServer() {
          try {
              if (!this.initServer()) {
-@@ -727,9 +1127,16 @@
+@@ -727,9 +1128,16 @@
              }
  
              this.nextTickTimeNanos = Util.getNanos();
@@ -858,7 +861,7 @@
              while (this.running) {
                  long i;
  
-@@ -744,12 +1151,31 @@
+@@ -744,12 +1152,31 @@
                      if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) {
                          long k = j / i;
  
@@ -890,7 +893,7 @@
                  boolean flag = i == 0L;
  
                  if (this.debugCommandProfilerDelayStart) {
-@@ -757,6 +1183,8 @@
+@@ -757,6 +1184,8 @@
                      this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount);
                  }
  
@@ -899,7 +902,7 @@
                  this.nextTickTimeNanos += i;
  
                  try {
-@@ -830,6 +1258,13 @@
+@@ -830,6 +1259,13 @@
                      this.services.profileCache().clearExecutor();
                  }
  
@@ -913,7 +916,7 @@
                  this.onServerExit();
              }
  
-@@ -889,9 +1324,16 @@
+@@ -889,9 +1325,16 @@
      }
  
      private boolean haveTime() {
@@ -931,7 +934,7 @@
      public static boolean throwIfFatalException() {
          RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
  
-@@ -903,7 +1345,7 @@
+@@ -903,7 +1346,7 @@
      }
  
      public static void setFatalException(RuntimeException exception) {
@@ -940,7 +943,7 @@
      }
  
      @Override
-@@ -977,7 +1419,7 @@
+@@ -977,7 +1420,7 @@
          }
      }
  
@@ -949,7 +952,7 @@
          Profiler.get().incrementCounter("runTask");
          super.doRunTask(ticktask);
      }
-@@ -1025,6 +1467,7 @@
+@@ -1025,6 +1468,7 @@
      }
  
      public void tickServer(BooleanSupplier shouldKeepTicking) {
@@ -957,7 +960,7 @@
          long i = Util.getNanos();
          int j = this.pauseWhileEmptySeconds() * 20;
  
-@@ -1041,11 +1484,13 @@
+@@ -1041,11 +1485,13 @@
                      this.autoSave();
                  }
  
@@ -971,7 +974,7 @@
          ++this.tickCount;
          this.tickRateManager.tick();
          this.tickChildren(shouldKeepTicking);
-@@ -1055,12 +1500,18 @@
+@@ -1055,12 +1501,18 @@
          }
  
          --this.ticksUntilAutosave;
@@ -991,7 +994,7 @@
          gameprofilerfiller.push("tallying");
          long k = Util.getNanos() - i;
          int l = this.tickCount % 100;
-@@ -1069,12 +1520,17 @@
+@@ -1069,12 +1521,17 @@
          this.aggregatedTickTimesNanos += k;
          this.tickTimesNanos[l] = k;
          this.smoothedTickTimeMillis = this.smoothedTickTimeMillis * 0.8F + (float) k / (float) TimeUtil.NANOSECONDS_PER_MILLISECOND * 0.19999999F;
@@ -1010,7 +1013,7 @@
          MinecraftServer.LOGGER.debug("Autosave started");
          ProfilerFiller gameprofilerfiller = Profiler.get();
  
-@@ -1123,7 +1579,7 @@
+@@ -1123,7 +1580,7 @@
      private ServerStatus buildServerStatus() {
          ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus();
  
@@ -1019,7 +1022,7 @@
      }
  
      private ServerStatus.Players buildPlayerStatus() {
-@@ -1133,7 +1589,7 @@
+@@ -1133,7 +1590,7 @@
          if (this.hidesOnlinePlayers()) {
              return new ServerStatus.Players(i, list.size(), List.of());
          } else {
@@ -1028,7 +1031,7 @@
              ObjectArrayList<GameProfile> objectarraylist = new ObjectArrayList(j);
              int k = Mth.nextInt(this.random, 0, list.size() - j);
  
-@@ -1154,24 +1610,55 @@
+@@ -1154,24 +1611,55 @@
          this.getPlayerList().getPlayers().forEach((entityplayer) -> {
              entityplayer.connection.suspendFlushing();
          });
@@ -1084,7 +1087,7 @@
  
              gameprofilerfiller.push("tick");
  
-@@ -1186,6 +1673,7 @@
+@@ -1186,6 +1674,7 @@
  
              gameprofilerfiller.pop();
              gameprofilerfiller.pop();
@@ -1092,7 +1095,7 @@
          }
  
          gameprofilerfiller.popPush("connection");
-@@ -1267,6 +1755,22 @@
+@@ -1267,6 +1756,22 @@
          return (ServerLevel) this.levels.get(key);
      }
  
@@ -1115,7 +1118,7 @@
      public Set<ResourceKey<Level>> levelKeys() {
          return this.levels.keySet();
      }
-@@ -1296,7 +1800,7 @@
+@@ -1296,7 +1801,7 @@
  
      @DontObfuscate
      public String getServerModName() {
@@ -1124,7 +1127,7 @@
      }
  
      public SystemReport fillSystemReport(SystemReport details) {
-@@ -1347,7 +1851,7 @@
+@@ -1347,7 +1852,7 @@
  
      @Override
      public void sendSystemMessage(Component message) {
@@ -1133,7 +1136,7 @@
      }
  
      public KeyPair getKeyPair() {
-@@ -1481,10 +1985,20 @@
+@@ -1481,10 +1986,20 @@
  
      @Override
      public String getMotd() {
@@ -1155,7 +1158,7 @@
          this.motd = motd;
      }
  
-@@ -1507,7 +2021,7 @@
+@@ -1507,7 +2022,7 @@
      }
  
      public ServerConnectionListener getConnection() {
@@ -1164,7 +1167,7 @@
      }
  
      public boolean isReady() {
-@@ -1634,11 +2148,11 @@
+@@ -1634,11 +2149,11 @@
  
      public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
          CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@@ -1178,7 +1181,7 @@
          }, this).thenCompose((immutablelist) -> {
              MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
              List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
-@@ -1654,6 +2168,7 @@
+@@ -1654,6 +2169,7 @@
          }).thenAcceptAsync((minecraftserver_reloadableresources) -> {
              this.resources.close();
              this.resources = minecraftserver_reloadableresources;
@@ -1186,7 +1189,7 @@
              this.packRepository.setSelected(dataPacks);
              WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
  
-@@ -1952,7 +2467,7 @@
+@@ -1952,7 +2468,7 @@
              final List<String> list = Lists.newArrayList();
              final GameRules gamerules = this.getGameRules();
  
@@ -1195,7 +1198,7 @@
                  @Override
                  public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
                      list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
-@@ -2058,7 +2573,7 @@
+@@ -2058,7 +2574,7 @@
              try {
                  label51:
                  {
@@ -1204,7 +1207,7 @@
  
                      try {
                          arraylist = Lists.newArrayList(NativeModuleLister.listModules());
-@@ -2105,9 +2620,24 @@
+@@ -2105,9 +2621,24 @@
          if (bufferedwriter != null) {
              bufferedwriter.close();
          }
@@ -1229,7 +1232,7 @@
      private ProfilerFiller createProfiler() {
          if (this.willStartRecordingMetrics) {
              this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
-@@ -2225,18 +2755,24 @@
+@@ -2225,18 +2756,24 @@
      }
  
      public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) {
@@ -1258,7 +1261,7 @@
      }
  
      public boolean logIPs() {
-@@ -2379,4 +2915,30 @@
+@@ -2379,4 +2916,30 @@
      public static record ServerResourcePackInfo(UUID id, String url, String hash, boolean isRequired, @Nullable Component prompt) {
  
      }

paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java

@@ -1062,6 +1062,32 @@ public final class CraftServer implements Server {
         org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
     }
 
+    // Paper start - Wait for Async Tasks during shutdown
+    public void waitForAsyncTasksShutdown() {
+        int pollCount = 0;
+
+        // Wait for at most 5 seconds for plugins to close their threads
+        while (pollCount < 10*5 && getScheduler().getActiveWorkers().size() > 0) {
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {}
+            pollCount++;
+        }
+
+        List<BukkitWorker> overdueWorkers = getScheduler().getActiveWorkers();
+        for (BukkitWorker worker : overdueWorkers) {
+            Plugin plugin = worker.getOwner();
+            getLogger().log(Level.SEVERE, String.format(
+                "Nag author(s): '%s' of '%s' about the following: %s",
+                plugin.getPluginMeta().getAuthors(),
+                plugin.getPluginMeta().getDisplayName(),
+                "This plugin is not properly shutting down its async tasks when it is being shut down. This task may throw errors during the final shutdown logs and might not complete before process dies."
+            ));
+            if (console.isDebugging()) io.papermc.paper.util.TraceUtil.dumpTraceForThread(worker.getThread(), "still running"); // Paper - Debugging
+        }
+    }
+    // Paper end - Wait for Async Tasks during shutdown
+
     @Override
     public void reloadData() {
         ReloadCommand.reload(this.console);