From c47e8c9505057b7a6f429a5c36065be1c783cfda Mon Sep 17 00:00:00 2001 From: Aikar Date: Thu, 26 Jul 2018 01:28:00 -0400 Subject: [PATCH] Async Chunk Load Current unfinished, not working progress --- .../minecraft/server/ChunkProviderServer.java | 118 +++++++++++++++--- .../minecraft/server/ChunkRegionLoader.java | 14 ++- .../java/net/minecraft/server/MCUtil.java | 35 ++++-- .../net/minecraft/server/MinecraftServer.java | 1 + .../net/minecraft/server/PlayerChunk.java | 14 ++- .../org/bukkit/craftbukkit/CraftWorld.java | 10 +- .../craftbukkit/chunkio/ChunkIOProvider.java | 42 +++++-- .../craftbukkit/chunkio/QueuedChunk.java | 8 ++ .../util/AsynchronousExecutor.java | 6 +- 9 files changed, 191 insertions(+), 57 deletions(-) diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java index 0e0c7b1abe..e72f43e2c8 100644 --- a/src/main/java/net/minecraft/server/ChunkProviderServer.java +++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java @@ -108,13 +108,33 @@ public class ChunkProviderServer implements IChunkProvider { @Nullable public Chunk getOrLoadChunkAt(int i, int j) { - Long2ObjectMap long2objectmap = this.chunks; - - synchronized (this.chunks) { - Chunk chunk = world.paperConfig.allowPermaChunkLoaders ? getLoadedChunkAt(i, j) : getChunkIfLoaded(i, j); // Paper - Configurable perma chunk loaders + // Paper start + return getOrLoadChunkAt(i, j, null); + } + public Chunk getOrLoadChunkAt(int i, int j, Runnable runnable) { + // Paper - remove synchronized + Chunk chunk = world.paperConfig.allowPermaChunkLoaders ? getLoadedChunkAt(i, j) : getChunkIfLoaded(i, j); // Paper - Configurable perma chunk loaders + if (!this.chunkLoader.chunkExists(i, j)) { + return null; + } + long key = ChunkCoordIntPair.asLong(i, j); + synchronized (pendingGenerates) { + if (pendingGenerates.containsKey(key)) { + return null; // let getChunkAt handle it + } + } - return chunk != null ? chunk : this.loadChunkAt(i, j); + if (chunk == null && runnable != null) { + ChunkIOExecutor.queueChunkLoad(world, (ChunkRegionLoader) this.chunkLoader, this, i, j, runnable); + return null; + } + if (chunk == null) { + chunk = ChunkIOExecutor.syncChunkLoad(world, (ChunkRegionLoader) chunkLoader, this, i, j); } + if (chunk != null && runnable != null) { + runnable.run(); + } + return chunk; } // CraftBukkit start @@ -123,20 +143,74 @@ public class ChunkProviderServer implements IChunkProvider { } // CraftBukkit end + // Paper sart + private final it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap> pendingGenerates = new it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap<>(); + public CompletableFuture generateChunk(int i, int j) { // Incase something else calls this + return generateChunk(i, j, null); + } + + private CompletableFuture postChunkgenMain(ProtoChunk proto) { + CompletableFuture pending = new CompletableFuture<>(); + MCUtil.ensureChunkMain(() -> { + try { + pending.complete(this.finishChunkgen(proto)); + } catch (Exception e) { + pending.completeExceptionally(e); + } + }); + return pending; + } + public CompletableFuture generateChunk(int i, int j, Consumer consumer) { + long key = ChunkCoordIntPair.asLong(i, j); + CompletableFuture pending; + boolean generate = false; + synchronized (pendingGenerates) { + pending = pendingGenerates.get(key); + if (pending == null) { + pending = new CompletableFuture<>(); + pendingGenerates.put(key, pending); + generate = true; + } + } + if (generate) { + this.generateChunk0(i, j).thenApply(chunk -> { + synchronized (pendingGenerates) { + pendingGenerates.remove(key); + } + return chunk; + }).thenAccept(pending::complete); + } + + if (consumer != null) { + pending.thenAccept(chunk -> MCUtil.ensureMain(() -> consumer.accept(chunk))); + } + return pending; + } + public Chunk getChunkAt(int i, int j) { - Chunk chunk = this.getOrLoadChunkAt(i, j); + return getChunkAt(i, j, null, true); + } + public Chunk getChunkAt(int i, int j, Runnable runnable) { + return getChunkAt(i, j, runnable, true); + } + public Chunk getChunkAt(int i, int j, Runnable runnable, boolean generate) { + // Paper end + //calling into self and blocking? + Chunk chunk = this.getOrLoadChunkAt(i, j, runnable); if (chunk != null) { return chunk; - } else { + } else if (generate) { try (co.aikar.timings.Timing timing = world.timings.chunkGeneration.startTiming()) { - ; // Spigot - chunk = (Chunk) this.generateChunk(i, j).get(); - return chunk; - } catch (ExecutionException | InterruptedException interruptedexception) { + // Paper start + CompletableFuture pending = generateChunk(i, j, runnable != null ? unused -> runnable.run() : null); + return runnable != null ? null : MCUtil.processChunkQueueWhileWaiting(pending); + } catch (Exception interruptedexception) { + // Paper end throw this.a(i, j, (Throwable) interruptedexception); } } + return null; // Paper } public IChunkAccess d(int i, int j) { @@ -160,7 +234,7 @@ public class ChunkProviderServer implements IChunkProvider { if (chunk != null) { consumer.accept(chunk); } else { - this.g.a(chunkcoordintpair).thenApply(this::a).thenAccept(consumer); + this.g.a(chunkcoordintpair).thenApply(proto -> postChunkgenMain(proto).thenAccept(consumer)); // Paper } } @@ -168,11 +242,13 @@ public class ChunkProviderServer implements IChunkProvider { } // CraftBukkit start - public CompletableFuture generateChunk(int i, int j) { - return this.generateChunk(i, j, false); + // Paper start - change name, make private + private CompletableFuture generateChunk0(int i, int j) { + return this.generateChunk0(i, j, false); } - public CompletableFuture generateChunk(int i, int j, boolean force) { + private CompletableFuture generateChunk0(int i, int j, boolean force) { + // Paper end this.g.b(); ChunkCoordIntPair chunkcoordintpair = new ChunkCoordIntPair(i, j); @@ -183,7 +259,11 @@ public class ChunkProviderServer implements IChunkProvider { // CraftBukkit end CompletableFuture completablefuture = this.g.c(); // CraftBukkit - decompile error - return completablefuture.thenApply(this::a); + // Paper start + CompletableFuture pending = new CompletableFuture<>(); + completablefuture.thenApply(proto -> postChunkgenMain(proto).thenAccept(pending::complete)); + return pending; + // Paper end } private ReportedException a(int i, int j, Throwable throwable) { @@ -196,7 +276,7 @@ public class ChunkProviderServer implements IChunkProvider { return new ReportedException(crashreport); } - private Chunk a(IChunkAccess ichunkaccess) { + private Chunk finishChunkgen(IChunkAccess ichunkaccess) { // Paper - rename method - Anything that accesses this should ensure it goes through process to ensure it is on main thread ChunkCoordIntPair chunkcoordintpair = ichunkaccess.getPos(); int i = chunkcoordintpair.x; int j = chunkcoordintpair.z; @@ -204,7 +284,7 @@ public class ChunkProviderServer implements IChunkProvider { Long2ObjectMap long2objectmap = this.chunks; Chunk chunk; - synchronized (this.chunks) { + //synchronized (this.chunks) { // Paper Chunk chunk1 = (Chunk) this.chunks.get(k); if (chunk1 != null) { @@ -222,7 +302,7 @@ public class ChunkProviderServer implements IChunkProvider { } this.chunks.put(k, chunk); - } + //} // Paper chunk.addEntities(); return chunk; diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java index bd52bf6561..e4447ecf58 100644 --- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java +++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java @@ -62,8 +62,14 @@ public class ChunkRegionLoader implements IChunkLoader, IAsyncChunkSaver { } @Nullable - private synchronized NBTTagCompound a(GeneratorAccess generatoraccess, int i, int j) throws IOException { - NBTTagCompound nbttagcompound = SupplierUtils.getIfExists(this.b.get(new ChunkCoordIntPair(i, j))); // Spigot + // Paper start - remove synchronized, manually sync on the map + private NBTTagCompound a(GeneratorAccess generatoraccess, int i, int j) throws IOException { + Supplier supplier; + synchronized (this) { + supplier = this.b.get(new ChunkCoordIntPair(i, j)); + } + NBTTagCompound nbttagcompound = SupplierUtils.getIfExists(supplier); // Spigot + // Paper end return nbttagcompound != null ? nbttagcompound : this.a(generatoraccess.o().getDimensionManager(), generatoraccess.s_(), i, j, generatoraccess); // CraftBukkit } @@ -71,7 +77,7 @@ public class ChunkRegionLoader implements IChunkLoader, IAsyncChunkSaver { // CraftBukkit start private boolean check(ChunkProviderServer cps, int x, int z) throws IOException { if (cps != null) { - com.google.common.base.Preconditions.checkState(org.bukkit.Bukkit.isPrimaryThread(), "primary thread"); + //com.google.common.base.Preconditions.checkState(org.bukkit.Bukkit.isPrimaryThread(), "primary thread"); // Paper if (cps.isLoaded(x, z)) { return true; } @@ -162,7 +168,7 @@ public class ChunkRegionLoader implements IChunkLoader, IAsyncChunkSaver { return null; } - public synchronized Object[] loadChunk(GeneratorAccess generatoraccess, int i, int j, Consumer consumer) throws IOException { + public Object[] loadChunk(GeneratorAccess generatoraccess, int i, int j, Consumer consumer) throws IOException { // Paper - remove synchronize // CraftBukkit end NBTTagCompound nbttagcompound = this.a(generatoraccess, i, j); diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java index 80927de08b..d936709b47 100644 --- a/src/main/java/net/minecraft/server/MCUtil.java +++ b/src/main/java/net/minecraft/server/MCUtil.java @@ -7,6 +7,7 @@ import com.mojang.authlib.GameProfile; import org.apache.commons.lang.exception.ExceptionUtils; import org.bukkit.Location; import org.bukkit.craftbukkit.CraftWorld; +import org.bukkit.craftbukkit.chunkio.ChunkIOExecutor; import org.bukkit.craftbukkit.util.Waitable; import org.spigotmc.AsyncCatcher; @@ -14,13 +15,13 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Queue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; -import java.util.regex.Pattern; public final class MCUtil { private static final Executor asyncExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("Paper Async Task Handler Thread - %1$d").build()); @@ -76,10 +77,14 @@ public final class MCUtil { MinecraftServer.getServer().postToMainThread(new DelayedRunnable(ticks, runnable)); } - public static void processQueue() { + + private static final Queue chunkMainQueue = new ConcurrentLinkedQueue<>(); + public static void processChunkMainQueue() { + if (!isMainThread()) { + return; + } Runnable runnable; - Queue processQueue = getProcessQueue(); - while ((runnable = processQueue.poll()) != null) { + while ((runnable = chunkMainQueue.poll()) != null) { try { runnable.run(); } catch (Exception e) { @@ -87,14 +92,15 @@ public final class MCUtil { } } } - public static T processQueueWhileWaiting(CompletableFuture future) { + public static T processChunkQueueWhileWaiting(CompletableFuture future) { try { if (isMainThread()) { while (!future.isDone()) { try { - return future.get(1, TimeUnit.MILLISECONDS); + return future.get(10000, TimeUnit.NANOSECONDS); } catch (TimeoutException ignored) { - processQueue(); + processChunkMainQueue(); + ChunkIOExecutor.tick(); } } } @@ -103,6 +109,13 @@ public final class MCUtil { throw new RuntimeException(e); } } + public static void ensureChunkMain(Runnable run) { + if (AsyncCatcher.enabled && Thread.currentThread() != MinecraftServer.getServer().primaryThread) { + chunkMainQueue.add(run); + } else { + run.run(); + } + } public static void ensureMain(Runnable run) { ensureMain(null, run); @@ -118,16 +131,12 @@ public final class MCUtil { if (reason != null) { new IllegalStateException("Asynchronous " + reason + "!").printStackTrace(); } - getProcessQueue().add(run); + MinecraftServer.getServer().processQueue.add(run); return; } run.run(); } - private static Queue getProcessQueue() { - return MinecraftServer.getServer().processQueue; - } - public static T ensureMain(Supplier run) { return ensureMain(null, run); } @@ -149,7 +158,7 @@ public final class MCUtil { return run.get(); } }; - getProcessQueue().add(wait); + MinecraftServer.getServer().processQueue.add(wait); try { return wait.get(); } catch (InterruptedException | ExecutionException e) { diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 90b1871196..390ecb4093 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -996,6 +996,7 @@ public abstract class MinecraftServer implements IAsyncTaskHandler, IMojangStati // CraftBukkit start // Run tasks that are waiting on processing MinecraftTimings.processQueueTimer.startTiming(); // Spigot + MCUtil.processChunkMainQueue(); // Paper while (!processQueue.isEmpty()) { processQueue.remove().run(); } diff --git a/src/main/java/net/minecraft/server/PlayerChunk.java b/src/main/java/net/minecraft/server/PlayerChunk.java index fcd9f5491f..4c67e7d0cf 100644 --- a/src/main/java/net/minecraft/server/PlayerChunk.java +++ b/src/main/java/net/minecraft/server/PlayerChunk.java @@ -49,7 +49,8 @@ public class PlayerChunk { public PlayerChunk(PlayerChunkMap playerchunkmap, int i, int j) { this.playerChunkMap = playerchunkmap; this.location = new ChunkCoordIntPair(i, j); - this.chunk = playerchunkmap.getWorld().getChunkProviderServer().getOrLoadChunkAt(i, j); + loadInProgress = true; // Paper + this.chunk = playerchunkmap.getWorld().getChunkProviderServer().getOrLoadChunkAt(i, j, loadedRunnable); // Paper this.chunkExists = this.chunk != null || ChunkIOExecutor.hasQueuedChunkLoad(playerChunkMap.getWorld(), i, j); // Paper markChunkUsed(); // Paper - delay chunk unloads } @@ -82,6 +83,7 @@ public class PlayerChunk { this.c.remove(entityplayer); if (this.c.isEmpty()) { + if (!this.done) ChunkIOExecutor.dropQueuedChunkLoad(this.playerChunkMap.getWorld(), this.location.x, this.location.z, this.loadedRunnable); // Paper this.playerChunkMap.b(this); } @@ -92,12 +94,14 @@ public class PlayerChunk { if (this.chunk != null) { return true; } else { - if (flag) { - this.chunk = this.playerChunkMap.getWorld().getChunkProviderServer().getChunkAt(this.location.x, this.location.z); - } else { - this.chunk = this.playerChunkMap.getWorld().getChunkProviderServer().getOrLoadChunkAt(this.location.x, this.location.z); + // Paper start + if (!loadInProgress) { + loadInProgress = true; + this.chunk = playerChunkMap.getWorld().getChunkProviderServer().getChunkAt(this.location.x, this.location.z, loadedRunnable, flag); markChunkUsed(); // Paper - delay chunk unloads } + // Paper end + return this.chunk != null; } diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java index f4dc7e4ac6..c1324515af 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -159,13 +159,7 @@ public class CraftWorld implements World { // Paper start - Async chunk load API public void getChunkAtAsync(final int x, final int z, final ChunkLoadCallback callback) { final ChunkProviderServer cps = this.world.getChunkProviderServer(); - callback.onLoad(cps.getChunkAt(x, z).bukkitChunk); // TODO: Add back async variant - /*cps.getChunkAt(x, z, new Runnable() { - @Override - public void run() { - callback.onLoad(cps.getChunkAt(x, z).bukkitChunk); - } - });*/ + cps.getChunkAt(x, z, () -> callback.onLoad(cps.getChunkAt(x, z).bukkitChunk)); } public void getChunkAtAsync(Block block, ChunkLoadCallback callback) { @@ -266,7 +260,7 @@ public class CraftWorld implements World { net.minecraft.server.Chunk chunk = null; - chunk = Futures.getUnchecked(world.getChunkProviderServer().generateChunk(x, z, true)); + chunk = MCUtil.processChunkQueueWhileWaiting(world.getChunkProviderServer().generateChunk(x, z)); // Paper PlayerChunk playerChunk = world.getPlayerChunkMap().getChunk(x, z); if (playerChunk != null) { playerChunk.chunk = chunk; diff --git a/src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOProvider.java b/src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOProvider.java index 2bbd5a7e2a..40b86c4334 100644 --- a/src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOProvider.java +++ b/src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOProvider.java @@ -6,11 +6,15 @@ import co.aikar.timings.Timing; import net.minecraft.server.Chunk; import net.minecraft.server.ChunkCoordIntPair; import net.minecraft.server.ChunkRegionLoader; +import net.minecraft.server.MCUtil; +import net.minecraft.server.MinecraftServer; import net.minecraft.server.NBTTagCompound; import org.bukkit.Server; import org.bukkit.craftbukkit.util.AsynchronousExecutor; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; class ChunkIOProvider implements AsynchronousExecutor.CallBackProvider { @@ -21,13 +25,21 @@ class ChunkIOProvider implements AsynchronousExecutor.CallBackProvider {}); - + if (data != null) { queuedChunk.compound = (NBTTagCompound) data[1]; return (Chunk) data[0]; + } else { + // Paper start + CompletableFuture pending = new CompletableFuture<>(); + MCUtil.ensureChunkMain(() -> { + System.out.println("GENERATING SEMI ASYNC" + queuedChunk); + queuedChunk.provider.generateChunk(queuedChunk.x, queuedChunk.z).thenApply(pending::complete); + }); + MCUtil.processChunkQueueWhileWaiting(pending); + return null; // We still need to return null so the code below doesn't use result which is already in the world now + // Paper end } - - return null; } catch (IOException ex) { throw new RuntimeException(ex); } @@ -35,12 +47,28 @@ class ChunkIOProvider implements AsynchronousExecutor.CallBackProvider {}); + if (data != null && data[0] != null) { + System.out.println("LOADED AFTER NULL" + queuedChunk); + queuedChunk.compound = (NBTTagCompound) data[1]; + chunk = (Chunk) data[0]; + } else { + System.out.println("GENERATING" + queuedChunk); + // Ok failed again, generate + MCUtil.processChunkQueueWhileWaiting(queuedChunk.provider.generateChunk(queuedChunk.x, queuedChunk.z, null)); + return; + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } } - try (Timing ignored = queuedChunk.provider.world.timings.chunkIOStage2.startTimingIfSync()) { // Paper queuedChunk.loader.loadEntities(queuedChunk.compound.getCompound("Level"), chunk); chunk.setLastSaved(queuedChunk.provider.world.getTime()); diff --git a/src/main/java/org/bukkit/craftbukkit/chunkio/QueuedChunk.java b/src/main/java/org/bukkit/craftbukkit/chunkio/QueuedChunk.java index 842d424f6a..c8f4db79f7 100644 --- a/src/main/java/org/bukkit/craftbukkit/chunkio/QueuedChunk.java +++ b/src/main/java/org/bukkit/craftbukkit/chunkio/QueuedChunk.java @@ -35,4 +35,12 @@ class QueuedChunk { return false; } + + @Override + public String toString() { + return "QueuedChunk{" + + "x=" + x + + ", z=" + z + + '}'; + } } diff --git a/src/main/java/org/bukkit/craftbukkit/util/AsynchronousExecutor.java b/src/main/java/org/bukkit/craftbukkit/util/AsynchronousExecutor.java index cf1258c559..b18c27060e 100644 --- a/src/main/java/org/bukkit/craftbukkit/util/AsynchronousExecutor.java +++ b/src/main/java/org/bukkit/craftbukkit/util/AsynchronousExecutor.java @@ -10,8 +10,10 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import net.minecraft.server.MCUtil; import org.apache.commons.lang.Validate; /** @@ -129,7 +131,8 @@ public final class AsynchronousExecutor { // We are the first into the lock while (state != STAGE_1_COMPLETE) { try { - this.wait(); + this.wait(0, 10000); // Paper + MCUtil.processChunkMainQueue(); // Paper } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Unable to handle interruption on " + parameter, e); @@ -185,6 +188,7 @@ public final class AsynchronousExecutor { final P parameter = this.parameter; final T object = this.object; + MCUtil.processChunkMainQueue(); // We may have an async chunk gen ready to be added to main first provider.callStage2(parameter, object); for (C callback : callbacks) { if (callback == this) { -- 2.18.0