Upstream/dynmap
1
0
Fork 0
mirror of https://github.com/webbukkit/dynmap.git synced 2024-12-25 01:57:53 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3725 from mastermc05/v3.0

Browse Source 
Async chunk loading on paper
This commit is contained in:
mikeprimm 2022-04-05 23:35:23 -05:00 committed by GitHub
commit 9c80489ec6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 444 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
DynmapCore/src/main/java/org/dynmap/common/chunk/GenericMapChunkCache.java
View File

@ -1,8 +1,10 @@
package org.dynmap.common.chunk;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Supplier;
import org.dynmap.DynmapChunk;
import org.dynmap.DynmapCore;
@ -37,6 +39,7 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
private int snapcnt;
private GenericChunk[] snaparray; /* Index = (x-x_min) + ((z-z_min)*x_dim) */
private boolean[][] isSectionNotEmpty; /* Indexed by snapshot index, then by section index */
private AtomicInteger loadingChunks = new AtomicInteger(0); //the amount of threads loading chunks at this moment, used by async loading
private static final BlockStep unstep[] = { BlockStep.X_MINUS, BlockStep.Y_MINUS, BlockStep.Z_MINUS,
BlockStep.X_PLUS, BlockStep.Y_PLUS, BlockStep.Z_PLUS };
@ -697,6 +700,14 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
protected abstract GenericChunk getLoadedChunk(DynmapChunk ch);
// Load generic chunk from unloaded chunk
protected abstract GenericChunk loadChunk(DynmapChunk ch);
// Load generic chunk from existing and already loaded chunk async
protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
throw new IllegalStateException("Not implemeted");
}
// Load generic chunks from unloaded chunk async
protected Supplier<GenericChunk> loadChunkAsync(DynmapChunk ch){
throw new IllegalStateException("Not implemeted");
}
/**
* Read NBT data from loaded chunks - needs to be called from server/world
@ -754,10 +765,82 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
return cnt;
}
/**
* Read NBT data from loaded chunks - do not needs to be called from server/world
* Will throw {@link IllegalStateException} if not supporting
*/
public void getLoadedChunksAsync() {
class SimplePair { //simple pair of the supplier that finishes read async, and a consumer that also finish his work async
final Supplier<GenericChunk> supplier;
final BiConsumer<GenericChunk, Long> consumer;
SimplePair(Supplier<GenericChunk> supplier, BiConsumer<GenericChunk, Long> consumer) {
this.supplier = supplier;
this.consumer = consumer;
}
}
if (!dw.isLoaded()) {
isempty = true;
unloadChunks();
return;
}
List<SimplePair> lastApply = new ArrayList<>();
for (DynmapChunk dynmapChunk : chunks) {
long startTime = System.nanoTime();
int chunkIndex = (dynmapChunk.x - x_min) + (dynmapChunk.z - z_min) * x_dim;
if (snaparray[chunkIndex] != null)
continue; // Skip if already processed
boolean vis = isChunkVisible(dynmapChunk);
/* Check if cached chunk snapshot found */
if (tryChunkCache(dynmapChunk, vis)) {
endChunkLoad(startTime, ChunkStats.CACHED_SNAPSHOT_HIT);
}
// If chunk is loaded and not being unloaded, we're grabbing its NBT data
else {
// Get generic chunk from already loaded chunk, if we can
Supplier<GenericChunk> supplier = getLoadedChunkAsync(dynmapChunk);
long startPause = System.nanoTime();
BiConsumer<GenericChunk, Long> consumer = (ss, reloadTime) -> {
if (ss == null) return;
long pause = reloadTime - startPause;
if (vis) { // If visible
prepChunkSnapshot(dynmapChunk, ss);
} else {
if (hidestyle == HiddenChunkStyle.FILL_STONE_PLAIN) {
ss = getStone();
} else if (hidestyle == HiddenChunkStyle.FILL_OCEAN) {
ss = getOcean();
} else {
ss = getEmpty();
}
}
snaparray[chunkIndex] = ss;
endChunkLoad(startTime - pause, ChunkStats.LOADED_CHUNKS);
};
lastApply.add(new SimplePair(supplier, consumer));
}
}
//impact on the main thread should be minimal, so we plan and finish the work after main thread finished it's part
lastApply.forEach(simplePair -> {
long reloadWork = System.nanoTime();
simplePair.consumer.accept(simplePair.supplier.get(), reloadWork);
});
}
@Override
public int loadChunks(int max_to_load) {
return getLoadedChunks() + readChunks(max_to_load);
}
/**
* Prepare the chunks async
*/
public void loadChunksAsync() {
getLoadedChunksAsync();
readChunksAsync();
}
public int readChunks(int max_to_load) {
@ -840,6 +923,96 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
return cnt;
}
public void readChunksAsync() {
class SimplePair { //pair of the chunk and the data which is readed async
private final Supplier<GenericChunk> supplier;
private final DynmapChunk chunk;
SimplePair(DynmapChunk chunk) {
this.chunk = chunk;
this.supplier = loadChunkAsync(chunk);
}
}
if (!dw.isLoaded()) {
isempty = true;
unloadChunks();
return;
}
List<DynmapChunk> chunks;
if (iterator == null) {
iterator = Collections.emptyListIterator();
chunks = new ArrayList<>(this.chunks);
} else {
chunks = new ArrayList<>();
iterator.forEachRemaining(chunks::add);
}
//if before increent was 0, means that we are the first, so we need to set this
if (loadingChunks.getAndIncrement() == 0) {
DynmapCore.setIgnoreChunkLoads(true);
}
try {
List<DynmapChunk> cached = new ArrayList<>();
List<SimplePair> notCached = new ArrayList<>();
iterator.forEachRemaining(chunks::add);
chunks.stream()
.filter(chunk -> snaparray[(chunk.x - x_min) + (chunk.z - z_min) * x_dim] == null)
.forEach(chunk -> {
if (cache.getSnapshot(dw.getName(), chunk.x, chunk.z) == null) {
notCached.add(new SimplePair(chunk));
} else {
cached.add(chunk);
}
});
cached.forEach(chunk -> {
long startTime = System.nanoTime();
tryChunkCache(chunk, isChunkVisible(chunk));
endChunkLoad(startTime, ChunkStats.CACHED_SNAPSHOT_HIT);
});
notCached.forEach(chunkSupplier -> {
long startTime = System.nanoTime();
GenericChunk chunk = chunkSupplier.supplier.get();
DynmapChunk dynmapChunk = chunkSupplier.chunk;
if (chunk != null) {
// If hidden
if (isChunkVisible(dynmapChunk)) {
// Prep snapshot
prepChunkSnapshot(dynmapChunk, chunk);
} else {
if (hidestyle == HiddenChunkStyle.FILL_STONE_PLAIN) {
chunk = getStone();
} else if (hidestyle == HiddenChunkStyle.FILL_OCEAN) {
chunk = getOcean();
} else {
chunk = getEmpty();
}
}
snaparray[(dynmapChunk.x - x_min) + (dynmapChunk.z - z_min) * x_dim] = chunk;
endChunkLoad(startTime, ChunkStats.UNLOADED_CHUNKS);
} else {
endChunkLoad(startTime, ChunkStats.UNGENERATED_CHUNKS);
}
});
isempty = true;
/* Fill missing chunks with empty dummy chunk */
for (int i = 0; i < snaparray.length; i++) {
if (snaparray[i] == null) {
snaparray[i] = getEmpty();
} else if (!snaparray[i].isEmpty) {
isempty = false;
}
}
} finally {
if (loadingChunks.decrementAndGet() == 0) {
DynmapCore.setIgnoreChunkLoads(false);
}
}
}
/**
* Test if done loading
*/

8
bukkit-helper-116-2/src/main/java/org/dynmap/bukkit/helper/v116_2/BukkitVersionHelperSpigot116_2.java
View File

@ -37,11 +37,13 @@ import net.minecraft.server.v1_16_R2.MinecraftServer;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot116_2 extends BukkitVersionHelperGeneric {
private final boolean unsafeAsync;
private Field watercolorfield;
public BukkitVersionHelperSpigot116_2() {
Class biomefog = getNMSClass("net.minecraft.server.BiomeFog");
watercolorfield = getPrivateField(biomefog, new String[] { "c" }, int.class);
this.unsafeAsync = true;
}
/**
@ -69,6 +71,12 @@ public class BukkitVersionHelperSpigot116_2 extends BukkitVersionHelperGeneric {
}
private Object[] biomelist;
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**
* Get list of defined biomebase objects
*/

8
bukkit-helper-116-3/src/main/java/org/dynmap/bukkit/helper/v116_3/BukkitVersionHelperSpigot116_3.java
View File

@ -37,11 +37,13 @@ import net.minecraft.server.v1_16_R2.BlockPosition;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot116_3 extends BukkitVersionHelperGeneric {
private final boolean unsafeAsync;
private Field watercolorfield;
public BukkitVersionHelperSpigot116_3() {
Class biomefog = getNMSClass("net.minecraft.server.BiomeFog");
watercolorfield = getPrivateField(biomefog, new String[] { "c" }, int.class);
this.unsafeAsync = true;
}
/**
@ -69,6 +71,12 @@ public class BukkitVersionHelperSpigot116_3 extends BukkitVersionHelperGeneric {
}
private Object[] biomelist;
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**
* Get list of defined biomebase objects
*/

8
bukkit-helper-116-4/src/main/java/org/dynmap/bukkit/helper/v116_4/BukkitVersionHelperSpigot116_4.java
View File

@ -26,11 +26,13 @@ import java.util.List;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot116_4 extends BukkitVersionHelperGeneric {
private final boolean unsafeAsync;
private Field watercolorfield;
public BukkitVersionHelperSpigot116_4() {
Class biomefog = getNMSClass("net.minecraft.server.BiomeFog");
watercolorfield = getPrivateField(biomefog, new String[] { "c" }, int.class);
this.unsafeAsync = true;
}
/**
@ -58,6 +60,12 @@ public class BukkitVersionHelperSpigot116_4 extends BukkitVersionHelperGeneric {
}
private Object[] biomelist;
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**
* Get list of defined biomebase objects
*/

8
bukkit-helper-116/src/main/java/org/dynmap/bukkit/helper/v116/BukkitVersionHelperSpigot116.java
View File

@ -36,11 +36,13 @@ import net.minecraft.server.v1_16_R1.BlockPosition;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot116 extends BukkitVersionHelperGeneric {
private final boolean unsafeAsync;
private Field watercolorfield;
public BukkitVersionHelperSpigot116() {
Class biomefog = getNMSClass("net.minecraft.server.BiomeFog");
watercolorfield = getPrivateField(biomefog, new String[] { "c" }, int.class);
this.unsafeAsync = true;
}
/**
@ -59,6 +61,12 @@ public class BukkitVersionHelperSpigot116 extends BukkitVersionHelperGeneric {
}
private Object[] biomelist;
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**
* Get list of defined biomebase objects
*/

8
bukkit-helper-117/src/main/java/org/dynmap/bukkit/helper/v117/BukkitVersionHelperSpigot117.java
View File

@ -59,7 +59,15 @@ import java.util.Map;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot117 extends BukkitVersionHelper {
private final boolean unsafeAsync;
public BukkitVersionHelperSpigot117() {
this.unsafeAsync = true;
}
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**

60
bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/AsyncChunkProvider118_2.java Normal file
View File

@ -0,0 +1,60 @@
package org.dynmap.bukkit.helper.v118_2;
import net.minecraft.nbt.NBTTagCompound;
import net.minecraft.server.level.WorldServer;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
import java.util.function.Predicate;
/**
* The provider used to work with paper libs
* Because paper libs need java 17 we can't interact with them directly
*/
public class AsyncChunkProvider118_2 {
private final Thread ioThread;
private final Method getChunk;
private final Predicate<NBTTagCompound> ifFailed;
AsyncChunkProvider118_2 () {
try {
Predicate<NBTTagCompound> ifFailed1 = null;
Method getChunk1 = null;
Thread ioThread1 = null;
try {
Class<?> threadClass = Class.forName("com.destroystokyo.paper.io.PaperFileIOThread");
Class<?>[] classes = threadClass.getClasses();
Class<?> holder = Arrays.stream(classes).filter(aClass -> aClass.getSimpleName().equals("Holder")).findAny().orElseThrow(RuntimeException::new);
ioThread1 = (Thread) holder.getField("INSTANCE").get(null);
getChunk1 = threadClass.getMethod("loadChunkDataAsync", WorldServer.class, int.class, int.class, int.class, Consumer.class, boolean.class, boolean.class, boolean.class);
NBTTagCompound failure = (NBTTagCompound) threadClass.getField("FAILURE_VALUE").get(null);
ifFailed1 = nbtTagCompound -> nbtTagCompound == failure;
} catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException | NoSuchMethodException e) {
e.printStackTrace();
}
ifFailed = Objects.requireNonNull(ifFailed1);
getChunk = Objects.requireNonNull(getChunk1);
ioThread = Objects.requireNonNull(ioThread1);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
public CompletableFuture<NBTTagCompound> getChunk(WorldServer world, int x, int y) throws InvocationTargetException, IllegalAccessException {
CompletableFuture<Object> future = new CompletableFuture<>();
getChunk.invoke(ioThread,world,x,y,5,(Consumer<Object>) future::complete, false, true, true);
return future.thenApply((resultFuture) -> {
if (resultFuture == null) return null;
try {
NBTTagCompound compound = (NBTTagCompound) resultFuture.getClass().getField("chunkData").get(resultFuture);
return ifFailed.test(compound) ? null : compound;
} catch (IllegalAccessException | NoSuchFieldException e) {
e.printStackTrace();
}
return null;
});
}
}

13
bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/BukkitVersionHelperSpigot118_2.java
View File

@ -64,9 +64,22 @@ import java.util.Set;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot118_2 extends BukkitVersionHelper {
private final boolean unsafeAsync;
public BukkitVersionHelperSpigot118_2() {
boolean unsafeAsync1;
try {
Class.forName("com.destroystokyo.paper.io.PaperFileIOThread");
unsafeAsync1 = false;
} catch (ClassNotFoundException e) {
unsafeAsync1 = true;
}
this.unsafeAsync = unsafeAsync1;
}
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**

55
bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/MapChunkCache118_2.java
View File

@ -1,8 +1,11 @@
package org.dynmap.bukkit.helper.v118_2;
import org.bukkit.Bukkit;
import org.bukkit.World;
import org.bukkit.craftbukkit.v1_18_R2.CraftServer;
import org.bukkit.craftbukkit.v1_18_R2.CraftWorld;
import org.dynmap.DynmapChunk;
import org.dynmap.bukkit.helper.BukkitVersionHelper;
import org.dynmap.bukkit.helper.BukkitWorld;
import org.dynmap.common.chunk.GenericChunk;
import org.dynmap.common.chunk.GenericChunkCache;
@ -14,12 +17,16 @@ import net.minecraft.world.level.chunk.storage.ChunkRegionLoader;
import net.minecraft.world.level.chunk.Chunk;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.Supplier;
/**
* Container for managing chunks - dependent upon using chunk snapshots, since rendering is off server thread
*/
public class MapChunkCache118_2 extends GenericMapChunkCache {
private static final AsyncChunkProvider118_2 provider = BukkitVersionHelper.helper.isUnsafeAsync() ? null : new AsyncChunkProvider118_2();
private World w;
/**
* Construct empty cache
@ -30,19 +37,45 @@ public class MapChunkCache118_2 extends GenericMapChunkCache {
// Load generic chunk from existing and already loaded chunk
protected GenericChunk getLoadedChunk(DynmapChunk chunk) {
return getLoadedChunk(chunk, false).get();
}
@Override
protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
return getLoadedChunk(ch, true);
}
@Override
protected Supplier<GenericChunk> loadChunkAsync(DynmapChunk chunk){
try {
CompletableFuture<NBTTagCompound> nbt = provider.getChunk(((CraftWorld) w).getHandle(), chunk.x, chunk.z);
return () -> {
NBTTagCompound compound = nbt.join();
return compound == null ? null : parseChunkFromNBT(new NBT.NBTCompound(compound));
};
} catch (InvocationTargetException | IllegalAccessException ignored) {
return () -> null;
}
}
private Supplier<GenericChunk> getLoadedChunk(DynmapChunk chunk, boolean async) {
CraftWorld cw = (CraftWorld) w;
NBTTagCompound nbt = null;
GenericChunk gc = null;
if (cw.isChunkLoaded(chunk.x, chunk.z)) {
Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z);
if ((c != null) && c.o) { // c.loaded
nbt = ChunkRegionLoader.a(cw.getHandle(), c);
if (!cw.isChunkLoaded(chunk.x, chunk.z)) return () -> null;
Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
if ((c == null) || c.o) return () -> null; // c.loaded
if (async) { //the data of the chunk may change while we write, better to write it sync
CompletableFuture<NBTTagCompound> nbt = CompletableFuture.supplyAsync(() -> ChunkRegionLoader.a(cw.getHandle(), c), ((CraftServer) Bukkit.getServer()).getServer());
return () -> {
NBTTagCompound compound = nbt.join();
return compound == null ? null : parseChunkFromNBT(new NBT.NBTCompound(compound));
};
} else {
NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
GenericChunk genericChunk;
if (nbt != null) genericChunk = parseChunkFromNBT(new NBT.NBTCompound(nbt));
else genericChunk = null;
return () -> genericChunk;
}
if (nbt != null) {
gc = parseChunkFromNBT(new NBT.NBTCompound(nbt));
}
}
return gc;
}
// Load generic chunk from unloaded chunk
protected GenericChunk loadChunk(DynmapChunk chunk) {

6
bukkit-helper-118/src/main/java/org/dynmap/bukkit/helper/v118/BukkitVersionHelperSpigot118.java
View File

@ -64,9 +64,15 @@ import java.util.Set;
* Helper for isolation of bukkit version specific issues
*/
public class BukkitVersionHelperSpigot118 extends BukkitVersionHelper {
private final boolean unsafeAsync;
public BukkitVersionHelperSpigot118() {
this.unsafeAsync = true;
}
@Override
public boolean isUnsafeAsync() {
return unsafeAsync;
}
/**

4
bukkit-helper/src/main/java/org/dynmap/bukkit/helper/BukkitVersionHelper.java
View File

@ -34,6 +34,10 @@ public abstract class BukkitVersionHelper {
protected BukkitVersionHelper() {
}
/**
* Get if it's unsafe to load chunks async
*/
public abstract boolean isUnsafeAsync();
/**
* Get list of defined biomebase objects
*/

6
bukkit-helper/src/main/java/org/dynmap/bukkit/helper/BukkitVersionHelperCB.java
View File

@ -54,6 +54,12 @@ public class BukkitVersionHelperCB extends BukkitVersionHelperGeneric {
isBadUnload = HDBlockModels.checkVersionRange(mcver, "1.9-");
Log.verboseinfo("MCVER=" + mcver + ", isBadUnload=" + isBadUnload);
}
@Override
public boolean isUnsafeAsync() {
return true;
}
@Override
protected String getNMSPackage() {
Server srv = Bukkit.getServer();

5
bukkit-helper/src/main/java/org/dynmap/bukkit/helper/BukkitVersionHelperGlowstone.java
View File

@ -33,6 +33,11 @@ public class BukkitVersionHelperGlowstone extends BukkitVersionHelper {
}
}
@Override
public boolean isUnsafeAsync() {
return true;
}
@Override
public Object[] getBiomeBaseList() {
return new Object[0];

19
spigot/src/main/java/org/dynmap/bukkit/DynmapPlugin.java
View File

@ -104,6 +104,7 @@ import org.dynmap.common.DynmapPlayer;
import org.dynmap.common.DynmapServerInterface;
import org.dynmap.common.chunk.GenericChunkCache;
import org.dynmap.common.DynmapListenerManager.EventType;
import org.dynmap.common.chunk.GenericMapChunkCache;
import org.dynmap.hdmap.HDMap;
import org.dynmap.markers.MarkerAPI;
import org.dynmap.modsupport.ModSupportImpl;
@ -514,6 +515,7 @@ public class DynmapPlugin extends JavaPlugin implements DynmapAPI {
final MapChunkCache cc = c;
while(!cc.isDoneLoading()) {
if (BukkitVersionHelper.helper.isUnsafeAsync()) {
Future<Boolean> f = core.getServer().callSyncMethod(new Callable<Boolean>() {
public Boolean call() throws Exception {
boolean exhausted = true;
@ -552,8 +554,23 @@ public class DynmapPlugin extends JavaPlugin implements DynmapAPI {
Log.severe(ix);
return null;
}
if ((delay != null) && delay.booleanValue()) {
try { Thread.sleep(25); } catch (InterruptedException ix) {}
try {
Thread.sleep(25);
} catch (InterruptedException ix) {
}
}
} else {
if (prev_tick != cur_tick) {
prev_tick = cur_tick;
cur_tick_starttime = System.nanoTime();
}
if (cc instanceof GenericMapChunkCache) {
((GenericMapChunkCache) cc).loadChunksAsync();
} else {
cc.loadChunks(Integer.MAX_VALUE);
}
}
}
/* If cancelled due to world unload return nothing */