Wait for Async Tasks during shutdown

Server.reload() had this logic to give time for tasks to shutdown,
however shutdown did not...

Adds a 5 second grace period for any async tasks to finish and warns
if any are still running after that delay just as reload does.
This commit is contained in:
Aikar 2020-05-10 22:16:17 -04:00
parent 7be16c7ef0
commit 74343ef1be
2 changed files with 64 additions and 35 deletions

View File

@ -698,10 +698,12 @@
if (flush) {
Iterator iterator1 = this.getAllLevels().iterator();
@@ -628,18 +927,44 @@
@@ -626,20 +925,47 @@
@Override
public void close() {
this.stopServer();
}
+ }
+
+ // CraftBukkit start
+ private boolean hasStopped = false;
+ private boolean hasLoggedStop = false; // Paper - Debugging
@ -710,9 +712,9 @@
+ synchronized (this.stopLock) {
+ return this.hasStopped;
+ }
+ }
}
+ // CraftBukkit end
+
public void stopServer() {
+ // CraftBukkit start - prevent double stopping on multiple threads
+ synchronized(this.stopLock) {
@ -730,6 +732,7 @@
+ // CraftBukkit start
+ if (this.server != null) {
+ this.server.disablePlugins();
+ this.server.waitForAsyncTasksShutdown(); // Paper - Wait for Async Tasks during shutdown
+ }
+ // CraftBukkit end
+ if (io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper != null) io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper.shutdown(); // Paper - Plugin remapping
@ -744,7 +747,7 @@
}
MinecraftServer.LOGGER.info("Saving worlds");
@@ -693,6 +1018,15 @@
@@ -693,6 +1019,15 @@
} catch (IOException ioexception1) {
MinecraftServer.LOGGER.error("Failed to unlock level {}", this.storageSource.getLevelId(), ioexception1);
}
@ -760,7 +763,7 @@
}
@@ -709,6 +1043,14 @@
@@ -709,6 +1044,14 @@
}
public void halt(boolean waitForShutdown) {
@ -775,7 +778,7 @@
this.running = false;
if (waitForShutdown) {
try {
@@ -720,6 +1062,64 @@
@@ -720,6 +1063,64 @@
}
@ -840,7 +843,7 @@
protected void runServer() {
try {
if (!this.initServer()) {
@@ -727,9 +1127,16 @@
@@ -727,9 +1128,16 @@
}
this.nextTickTimeNanos = Util.getNanos();
@ -858,7 +861,7 @@
while (this.running) {
long i;
@@ -744,12 +1151,31 @@
@@ -744,12 +1152,31 @@
if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) {
long k = j / i;
@ -890,7 +893,7 @@
boolean flag = i == 0L;
if (this.debugCommandProfilerDelayStart) {
@@ -757,6 +1183,8 @@
@@ -757,6 +1184,8 @@
this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount);
}
@ -899,7 +902,7 @@
this.nextTickTimeNanos += i;
try {
@@ -830,6 +1258,13 @@
@@ -830,6 +1259,13 @@
this.services.profileCache().clearExecutor();
}
@ -913,7 +916,7 @@
this.onServerExit();
}
@@ -889,9 +1324,16 @@
@@ -889,9 +1325,16 @@
}
private boolean haveTime() {
@ -931,7 +934,7 @@
public static boolean throwIfFatalException() {
RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
@@ -903,7 +1345,7 @@
@@ -903,7 +1346,7 @@
}
public static void setFatalException(RuntimeException exception) {
@ -940,7 +943,7 @@
}
@Override
@@ -977,7 +1419,7 @@
@@ -977,7 +1420,7 @@
}
}
@ -949,7 +952,7 @@
Profiler.get().incrementCounter("runTask");
super.doRunTask(ticktask);
}
@@ -1025,6 +1467,7 @@
@@ -1025,6 +1468,7 @@
}
public void tickServer(BooleanSupplier shouldKeepTicking) {
@ -957,7 +960,7 @@
long i = Util.getNanos();
int j = this.pauseWhileEmptySeconds() * 20;
@@ -1041,11 +1484,13 @@
@@ -1041,11 +1485,13 @@
this.autoSave();
}
@ -971,7 +974,7 @@
++this.tickCount;
this.tickRateManager.tick();
this.tickChildren(shouldKeepTicking);
@@ -1055,12 +1500,18 @@
@@ -1055,12 +1501,18 @@
}
--this.ticksUntilAutosave;
@ -991,7 +994,7 @@
gameprofilerfiller.push("tallying");
long k = Util.getNanos() - i;
int l = this.tickCount % 100;
@@ -1069,12 +1520,17 @@
@@ -1069,12 +1521,17 @@
this.aggregatedTickTimesNanos += k;
this.tickTimesNanos[l] = k;
this.smoothedTickTimeMillis = this.smoothedTickTimeMillis * 0.8F + (float) k / (float) TimeUtil.NANOSECONDS_PER_MILLISECOND * 0.19999999F;
@ -1010,7 +1013,7 @@
MinecraftServer.LOGGER.debug("Autosave started");
ProfilerFiller gameprofilerfiller = Profiler.get();
@@ -1123,7 +1579,7 @@
@@ -1123,7 +1580,7 @@
private ServerStatus buildServerStatus() {
ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus();
@ -1019,7 +1022,7 @@
}
private ServerStatus.Players buildPlayerStatus() {
@@ -1133,7 +1589,7 @@
@@ -1133,7 +1590,7 @@
if (this.hidesOnlinePlayers()) {
return new ServerStatus.Players(i, list.size(), List.of());
} else {
@ -1028,7 +1031,7 @@
ObjectArrayList<GameProfile> objectarraylist = new ObjectArrayList(j);
int k = Mth.nextInt(this.random, 0, list.size() - j);
@@ -1154,24 +1610,55 @@
@@ -1154,24 +1611,55 @@
this.getPlayerList().getPlayers().forEach((entityplayer) -> {
entityplayer.connection.suspendFlushing();
});
@ -1084,7 +1087,7 @@
gameprofilerfiller.push("tick");
@@ -1186,6 +1673,7 @@
@@ -1186,6 +1674,7 @@
gameprofilerfiller.pop();
gameprofilerfiller.pop();
@ -1092,7 +1095,7 @@
}
gameprofilerfiller.popPush("connection");
@@ -1267,6 +1755,22 @@
@@ -1267,6 +1756,22 @@
return (ServerLevel) this.levels.get(key);
}
@ -1115,7 +1118,7 @@
public Set<ResourceKey<Level>> levelKeys() {
return this.levels.keySet();
}
@@ -1296,7 +1800,7 @@
@@ -1296,7 +1801,7 @@
@DontObfuscate
public String getServerModName() {
@ -1124,7 +1127,7 @@
}
public SystemReport fillSystemReport(SystemReport details) {
@@ -1347,7 +1851,7 @@
@@ -1347,7 +1852,7 @@
@Override
public void sendSystemMessage(Component message) {
@ -1133,7 +1136,7 @@
}
public KeyPair getKeyPair() {
@@ -1481,10 +1985,20 @@
@@ -1481,10 +1986,20 @@
@Override
public String getMotd() {
@ -1155,7 +1158,7 @@
this.motd = motd;
}
@@ -1507,7 +2021,7 @@
@@ -1507,7 +2022,7 @@
}
public ServerConnectionListener getConnection() {
@ -1164,7 +1167,7 @@
}
public boolean isReady() {
@@ -1634,11 +2148,11 @@
@@ -1634,11 +2149,11 @@
public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@ -1178,7 +1181,7 @@
}, this).thenCompose((immutablelist) -> {
MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
@@ -1654,6 +2168,7 @@
@@ -1654,6 +2169,7 @@
}).thenAcceptAsync((minecraftserver_reloadableresources) -> {
this.resources.close();
this.resources = minecraftserver_reloadableresources;
@ -1186,7 +1189,7 @@
this.packRepository.setSelected(dataPacks);
WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
@@ -1952,7 +2467,7 @@
@@ -1952,7 +2468,7 @@
final List<String> list = Lists.newArrayList();
final GameRules gamerules = this.getGameRules();
@ -1195,7 +1198,7 @@
@Override
public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
@@ -2058,7 +2573,7 @@
@@ -2058,7 +2574,7 @@
try {
label51:
{
@ -1204,7 +1207,7 @@
try {
arraylist = Lists.newArrayList(NativeModuleLister.listModules());
@@ -2105,9 +2620,24 @@
@@ -2105,9 +2621,24 @@
if (bufferedwriter != null) {
bufferedwriter.close();
}
@ -1229,7 +1232,7 @@
private ProfilerFiller createProfiler() {
if (this.willStartRecordingMetrics) {
this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
@@ -2225,18 +2755,24 @@
@@ -2225,18 +2756,24 @@
}
public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) {
@ -1258,7 +1261,7 @@
}
public boolean logIPs() {
@@ -2379,4 +2915,30 @@
@@ -2379,4 +2916,30 @@
public static record ServerResourcePackInfo(UUID id, String url, String hash, boolean isRequired, @Nullable Component prompt) {
}

View File

@ -1062,6 +1062,32 @@ public final class CraftServer implements Server {
org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
}
// Paper start - Wait for Async Tasks during shutdown
public void waitForAsyncTasksShutdown() {
int pollCount = 0;
// Wait for at most 5 seconds for plugins to close their threads
while (pollCount < 10*5 && getScheduler().getActiveWorkers().size() > 0) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {}
pollCount++;
}
List<BukkitWorker> overdueWorkers = getScheduler().getActiveWorkers();
for (BukkitWorker worker : overdueWorkers) {
Plugin plugin = worker.getOwner();
getLogger().log(Level.SEVERE, String.format(
"Nag author(s): '%s' of '%s' about the following: %s",
plugin.getPluginMeta().getAuthors(),
plugin.getPluginMeta().getDisplayName(),
"This plugin is not properly shutting down its async tasks when it is being shut down. This task may throw errors during the final shutdown logs and might not complete before process dies."
));
if (console.isDebugging()) io.papermc.paper.util.TraceUtil.dumpTraceForThread(worker.getThread(), "still running"); // Paper - Debugging
}
}
// Paper end - Wait for Async Tasks during shutdown
@Override
public void reloadData() {
ReloadCommand.reload(this.console);