Merge pull request #3794 from mastermc05/v3.0

Fix and improve async chunk load

DynmapCore/src/main/java/org/dynmap/common/chunk/GenericMapChunkCache.java

@@ -766,7 +766,7 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 	}
 
 	/**
-	 * Read NBT data from loaded chunks - do not needs to be called from server/world
+	 * Read NBT data from loaded chunks - do not needs to be called from server/world <p>
 	 * Will throw {@link IllegalStateException} if not supporting
 	 */
 	public void getLoadedChunksAsync() {
@@ -836,7 +836,9 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 	}
 
 	/**
-	 * Prepare the chunks async
+	 * Loads all chunks in the world asynchronously.
+	 * <p>
+	 * If it is not supported, it will throw {@link IllegalStateException}
 	 */
 	public void loadChunksAsync() {
 		getLoadedChunksAsync();
@@ -923,6 +925,12 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 		return cnt;
 	}
 
+	/**
+	 * It loads chunks from the cache or from the world, and if the chunk is not visible, it fills it with stone, ocean or
+	 * empty chunk
+	 * <p>
+	 * if it's not supported, will throw {@link IllegalStateException}
+	 */
 	public void readChunksAsync() {
 		class SimplePair { //pair of the chunk and the data which is readed async
 			private final Supplier<GenericChunk> supplier;

bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/AsyncChunkProvider118_2.java

@@ -1,31 +1,54 @@
 package org.dynmap.bukkit.helper.v118_2;
 
 import net.minecraft.nbt.NBTTagCompound;
+import net.minecraft.server.MinecraftServer;
 import net.minecraft.server.level.WorldServer;
+import net.minecraft.world.level.chunk.Chunk;
+import net.minecraft.world.level.chunk.IChunkAccess;
+import net.minecraft.world.level.chunk.storage.ChunkRegionLoader;
+import org.bukkit.Bukkit;
+import org.bukkit.craftbukkit.v1_18_R2.CraftServer;
+import org.bukkit.craftbukkit.v1_18_R2.CraftWorld;
+import org.dynmap.MapManager;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * The provider used to work with paper libs
  * Because paper libs need java 17 we can't interact with them directly
  */
+@SuppressWarnings({"JavaReflectionMemberAccess"}) //java don't know about paper
 public class AsyncChunkProvider118_2 {
     private final Thread ioThread;
     private final Method getChunk;
     private final Predicate<NBTTagCompound> ifFailed;
+    private final Method getAsyncSaveData;
+    private final Method save;
+    private int currTick = MinecraftServer.currentTick;
+    private int currChunks = 0;
+
     AsyncChunkProvider118_2 () {
         try {
             Predicate<NBTTagCompound> ifFailed1 = null;
-            Method getChunk1 = null;
+            Method getChunk1 = null, getAsyncSaveData1 = null, save1 = null;
             Thread ioThread1 = null;
             try {
                 Class<?> threadClass = Class.forName("com.destroystokyo.paper.io.PaperFileIOThread");
+                Class<?> asyncChunkData = Arrays.stream(ChunkRegionLoader.class.getClasses())
+                        .filter(c -> c.getSimpleName().equals("AsyncSaveData"))
+                        .findFirst()
+                        .orElseThrow(RuntimeException::new);
+                getAsyncSaveData1 = ChunkRegionLoader.class.getMethod("getAsyncSaveData", WorldServer.class, IChunkAccess.class);
+                save1 = ChunkRegionLoader.class.getMethod("saveChunk", WorldServer.class, IChunkAccess.class, asyncChunkData);
                 Class<?>[] classes = threadClass.getClasses();
                 Class<?> holder = Arrays.stream(classes).filter(aClass -> aClass.getSimpleName().equals("Holder")).findAny().orElseThrow(RuntimeException::new);
                 ioThread1 = (Thread) holder.getField("INSTANCE").get(null);
@@ -35,6 +58,8 @@ public class AsyncChunkProvider118_2 {
             } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException | NoSuchMethodException e) {
                 e.printStackTrace();
             }
+            getAsyncSaveData = Objects.requireNonNull(getAsyncSaveData1);
+            save = Objects.requireNonNull(save1);
             ifFailed = Objects.requireNonNull(ifFailed1);
             getChunk = Objects.requireNonNull(getChunk1);
             ioThread = Objects.requireNonNull(ioThread1);
@@ -57,4 +82,38 @@ public class AsyncChunkProvider118_2 {
             return null;
         });
     }
+
+    public synchronized Supplier<NBTTagCompound> getLoadedChunk(CraftWorld world, int x, int z) {
+        if (!world.isChunkLoaded(x, z)) return () -> null;
+        Chunk c = world.getHandle().getChunkIfLoaded(x, z); //already safe async on vanilla
+        if ((c == null) || !c.o) return () -> null;    // c.loaded
+        if (currTick != MinecraftServer.currentTick) {
+            currTick = MinecraftServer.currentTick;
+            currChunks = 0;
+        }
+        //prepare data synchronously
+        CompletableFuture<?> future = CompletableFuture.supplyAsync(() -> {
+            try {
+                return getAsyncSaveData.invoke(null, world.getHandle(), c);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+                throw new RuntimeException(e);
+            }
+        }, ((CraftServer) Bukkit.getServer()).getServer());
+        //we shouldn't stress main thread
+        if (++currChunks > MapManager.mapman.getMaxChunkLoadsPerTick()) {
+            try {
+                Thread.sleep(25); //hold the lock so other threads also won't stress main thread
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        //save data asynchronously
+        return () -> {
+            try {
+                return (NBTTagCompound) save.invoke(null, world.getHandle(), c, future.get());
+            } catch (ReflectiveOperationException | ExecutionException | InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        };
+    }
 }

bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/MapChunkCache118_2.java

@@ -1,10 +1,12 @@
 package org.dynmap.bukkit.helper.v118_2;
 
+import net.minecraft.server.MinecraftServer;
 import org.bukkit.Bukkit;
 import org.bukkit.World;
 import org.bukkit.craftbukkit.v1_18_R2.CraftServer;
 import org.bukkit.craftbukkit.v1_18_R2.CraftWorld;
 import org.dynmap.DynmapChunk;
+import org.dynmap.MapManager;
 import org.dynmap.bukkit.helper.BukkitVersionHelper;
 import org.dynmap.bukkit.helper.BukkitWorld;
 import org.dynmap.common.chunk.GenericChunk;
@@ -20,6 +22,7 @@ import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 /**
@@ -37,11 +40,20 @@ public class MapChunkCache118_2 extends GenericMapChunkCache {
 
     // Load generic chunk from existing and already loaded chunk
     protected GenericChunk getLoadedChunk(DynmapChunk chunk) {
-        return getLoadedChunk(chunk, false).get();
+        CraftWorld cw = (CraftWorld) w;
+        if (!cw.isChunkLoaded(chunk.x, chunk.z)) return null;
+        Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
+        if ((c == null) || !c.o) return null;    // c.loaded
+        NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
+        return nbt != null ? parseChunkFromNBT(new NBT.NBTCompound(nbt)) : null;
     }
     @Override
     protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
-        return getLoadedChunk(ch, true);
+        Supplier<NBTTagCompound> nbtSupplier = provider.getLoadedChunk((CraftWorld) w, ch.x, ch.z);
+        return () -> {
+            NBTTagCompound nbt = nbtSupplier.get();
+            return nbt == null ? null : parseChunkFromNBT(new NBT.NBTCompound(nbt));
+        };
     }
 
     @Override
@@ -57,26 +69,6 @@ public class MapChunkCache118_2 extends GenericMapChunkCache {
         }
     }
 
-    private Supplier<GenericChunk> getLoadedChunk(DynmapChunk chunk, boolean async) {
-        CraftWorld cw = (CraftWorld) w;
-        if (!cw.isChunkLoaded(chunk.x, chunk.z)) return () -> null;
-        Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
-        if ((c == null) || c.o) return () -> null;    // c.loaded
-        if (async) { //the data of the chunk may change while we write, better to write it sync
-            CompletableFuture<NBTTagCompound> nbt = CompletableFuture.supplyAsync(() -> ChunkRegionLoader.a(cw.getHandle(), c), ((CraftServer) Bukkit.getServer()).getServer());
-            return () -> {
-                    NBTTagCompound compound = nbt.join();
-                    return compound == null ? null : parseChunkFromNBT(new NBT.NBTCompound(compound));
-            };
-        } else {
-            NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
-            GenericChunk genericChunk;
-            if (nbt != null) genericChunk = parseChunkFromNBT(new NBT.NBTCompound(nbt));
-            else genericChunk = null;
-            return () -> genericChunk;
-        }
-
-    }
     // Load generic chunk from unloaded chunk
     protected GenericChunk loadChunk(DynmapChunk chunk) {
         CraftWorld cw = (CraftWorld) w;

bukkit-helper-119/src/main/java/org/dynmap/bukkit/helper/v119/AsyncChunkProvider119.java

@@ -1,31 +1,53 @@
 package org.dynmap.bukkit.helper.v119;
 
 import net.minecraft.nbt.NBTTagCompound;
+import net.minecraft.server.MinecraftServer;
 import net.minecraft.server.level.WorldServer;
+import net.minecraft.world.level.chunk.Chunk;
+import net.minecraft.world.level.chunk.IChunkAccess;
+import net.minecraft.world.level.chunk.storage.ChunkRegionLoader;
+import org.bukkit.Bukkit;
+import org.bukkit.craftbukkit.v1_19_R1.CraftServer;
+import org.bukkit.craftbukkit.v1_19_R1.CraftWorld;
+import org.dynmap.MapManager;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * The provider used to work with paper libs
  * Because paper libs need java 17 we can't interact with them directly
  */
+@SuppressWarnings({"JavaReflectionMemberAccess"}) //java don't know about paper
 public class AsyncChunkProvider119 {
     private final Thread ioThread;
     private final Method getChunk;
     private final Predicate<NBTTagCompound> ifFailed;
-    AsyncChunkProvider119 () {
+    private final Method getAsyncSaveData;
+    private final Method save;
+    private int currTick = MinecraftServer.currentTick;
+    private int currChunks = 0;
+
+    AsyncChunkProvider119() {
         try {
             Predicate<NBTTagCompound> ifFailed1 = null;
-            Method getChunk1 = null;
+            Method getChunk1 = null, getAsyncSaveData1 = null, save1 = null;
             Thread ioThread1 = null;
             try {
                 Class<?> threadClass = Class.forName("com.destroystokyo.paper.io.PaperFileIOThread");
+                Class<?> asyncChunkData = Arrays.stream(ChunkRegionLoader.class.getClasses())
+                        .filter(c -> c.getSimpleName().equals("AsyncSaveData"))
+                        .findFirst()
+                        .orElseThrow(RuntimeException::new);
+                getAsyncSaveData1 = ChunkRegionLoader.class.getMethod("getAsyncSaveData", WorldServer.class, IChunkAccess.class);
+                save1 = ChunkRegionLoader.class.getMethod("saveChunk", WorldServer.class, IChunkAccess.class, asyncChunkData);
                 Class<?>[] classes = threadClass.getClasses();
                 Class<?> holder = Arrays.stream(classes).filter(aClass -> aClass.getSimpleName().equals("Holder")).findAny().orElseThrow(RuntimeException::new);
                 ioThread1 = (Thread) holder.getField("INSTANCE").get(null);
@@ -35,6 +57,8 @@ public class AsyncChunkProvider119 {
             } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException | NoSuchMethodException e) {
                 e.printStackTrace();
             }
+            getAsyncSaveData = Objects.requireNonNull(getAsyncSaveData1);
+            save = Objects.requireNonNull(save1);
             ifFailed = Objects.requireNonNull(ifFailed1);
             getChunk = Objects.requireNonNull(getChunk1);
             ioThread = Objects.requireNonNull(ioThread1);
@@ -57,4 +81,38 @@ public class AsyncChunkProvider119 {
             return null;
         });
     }
+
+    public synchronized Supplier<NBTTagCompound> getLoadedChunk(CraftWorld world, int x, int z) {
+        if (!world.isChunkLoaded(x, z)) return () -> null;
+        Chunk c = world.getHandle().getChunkIfLoaded(x, z); //already safe async on vanilla
+        if ((c == null) || !c.o) return () -> null;    // c.loaded
+        if (currTick != MinecraftServer.currentTick) {
+            currTick = MinecraftServer.currentTick;
+            currChunks = 0;
+        }
+        //prepare data synchronously
+        CompletableFuture<?> future = CompletableFuture.supplyAsync(() -> {
+            try {
+                return getAsyncSaveData.invoke(null, world.getHandle(), c);
+            } catch (ReflectiveOperationException e) {
+                throw new RuntimeException(e);
+            }
+        }, ((CraftServer) Bukkit.getServer()).getServer());
+        //we shouldn't stress main thread
+        if (++currChunks > MapManager.mapman.getMaxChunkLoadsPerTick()) {
+            try {
+                Thread.sleep(25); //hold the lock so other threads also won't stress main thread
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        //save data asynchronously
+        return () -> {
+            try {
+                return (NBTTagCompound) save.invoke(null, world.getHandle(), c, future.get());
+            } catch (ReflectiveOperationException | ExecutionException | InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        };
+    }
 }

bukkit-helper-119/src/main/java/org/dynmap/bukkit/helper/v119/MapChunkCache119.java

@@ -1,8 +1,6 @@
 package org.dynmap.bukkit.helper.v119;
 
-import org.bukkit.Bukkit;
 import org.bukkit.World;
-import org.bukkit.craftbukkit.v1_19_R1.CraftServer;
 import org.bukkit.craftbukkit.v1_19_R1.CraftWorld;
 import org.dynmap.DynmapChunk;
 import org.dynmap.bukkit.helper.BukkitVersionHelper;
@@ -16,7 +14,6 @@ import net.minecraft.world.level.ChunkCoordIntPair;
 import net.minecraft.world.level.chunk.storage.ChunkRegionLoader;
 import net.minecraft.world.level.chunk.Chunk;
 
-import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import java.util.NoSuchElementException;
@@ -38,14 +35,24 @@ public class MapChunkCache119 extends GenericMapChunkCache {
     }
 
     // Load generic chunk from existing and already loaded chunk
-    protected GenericChunk getLoadedChunk(DynmapChunk chunk) {
-        return getLoadedChunk(chunk, false).get();
-    }
     @Override
-    protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
+    protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk chunk) {
-        return getLoadedChunk(ch, true);
+        Supplier<NBTTagCompound> supplier = provider.getLoadedChunk((CraftWorld) w, chunk.x, chunk.z);
+        return () -> {
+            NBTTagCompound nbt = supplier.get();
+            return nbt != null ? parseChunkFromNBT(new NBT.NBTCompound(nbt)) : null;
+        };
+    }
+    protected GenericChunk getLoadedChunk(DynmapChunk chunk) {
+        CraftWorld cw = (CraftWorld) w;
+        if (!cw.isChunkLoaded(chunk.x, chunk.z)) return null;
+        Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z);
+        if (c == null || !c.o) return null;    // c.loaded
+        NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
+        return nbt != null ? parseChunkFromNBT(new NBT.NBTCompound(nbt)) : null;
     }
 
+    // Load generic chunk from unloaded chunk
     @Override
     protected Supplier<GenericChunk> loadChunkAsync(DynmapChunk chunk){
         try {
@@ -59,27 +66,6 @@ public class MapChunkCache119 extends GenericMapChunkCache {
         }
     }
 
-    private Supplier<GenericChunk> getLoadedChunk(DynmapChunk chunk, boolean async) {
-        CraftWorld cw = (CraftWorld) w;
-        if (!cw.isChunkLoaded(chunk.x, chunk.z)) return () -> null;
-        Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
-        if ((c == null) || c.o) return () -> null;    // c.loaded
-        if (async) { //the data of the chunk may change while we write, better to write it sync
-            CompletableFuture<NBTTagCompound> nbt = CompletableFuture.supplyAsync(() -> ChunkRegionLoader.a(cw.getHandle(), c), ((CraftServer) Bukkit.getServer()).getServer());
-            return () -> {
-                    NBTTagCompound compound = nbt.join();
-                    return compound == null ? null : parseChunkFromNBT(new NBT.NBTCompound(compound));
-            };
-        } else {
-            NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
-            GenericChunk genericChunk;
-            if (nbt != null) genericChunk = parseChunkFromNBT(new NBT.NBTCompound(nbt));
-            else genericChunk = null;
-            return () -> genericChunk;
-        }
-
-    }
-    // Load generic chunk from unloaded chunk
     protected GenericChunk loadChunk(DynmapChunk chunk) {
         CraftWorld cw = (CraftWorld) w;
         NBTTagCompound nbt = null;