Use async Chunk generation, if possible, using PaperLib

This routes all world generation requests through PaperLib, which will generate Chunks asynchronously if the server allows it (Paper does, Spigot doesn't). This means changes to which chunks are still needed, and which can be unloaded, as well; the code keeps a list of Chunks that are needed for others, and will unload them only when the target chunk has been generated. Unloads by the server itself get prevented while the chunk is needed; else the server could decide on a tick that chunk has no players nearby and needs to be unloaded.

pom.xml

@@ -17,8 +17,8 @@
 
 	<repositories>
 		<repository>
-                   <id>papermc</id>
+            <id>papermc</id>
-                   <url>https://papermc.io/repo/repository/maven-public/</url>
+            <url>https://papermc.io/repo/repository/maven-public/</url>
 		</repository>
 		<repository>
 			<id>dynmap-repo</id>
@@ -29,10 +29,10 @@
 	<dependencies>
 		<!--Spigot-API-->
 		<dependency>
-                   <groupId>com.destroystokyo.paper</groupId>
+            <groupId>com.destroystokyo.paper</groupId>
-                   <artifactId>paper-api</artifactId>
+            <artifactId>paper-api</artifactId>
-                   <version>1.13.2-R0.1-SNAPSHOT</version>
+            <version>1.13.2-R0.1-SNAPSHOT</version>
-                   <scope>provided</scope>
+            <scope>provided</scope>
 		</dependency>
 		<!--Bukkit API-->
 		<dependency>
@@ -46,6 +46,12 @@
 			<artifactId>dynmap-api</artifactId>
 			<version>2.5</version>
 		</dependency>
+        <dependency>
+            <groupId>io.papermc</groupId>
+            <artifactId>paperlib</artifactId>
+            <version>1.0.2</version>
+            <scope>compile</scope>
+         </dependency>        
 	</dependencies>
 
 	<build>
@@ -61,6 +67,43 @@
 					<target>1.8</target>
 				</configuration>
 			</plugin>
-		</plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.1.1</version>
+                <configuration>
+                    <dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+                    <relocations>
+                        <relocation>
+                            <pattern>io.papermc.lib</pattern>
+                            <shadedPattern>com.wimbli.WorldBorder.paperlib</shadedPattern> <!-- Replace this -->
+                        </relocation>
+                    </relocations>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                              <excludes>
+                                <exclude>commons-lang:commons-lang</exclude>
+                                <exclude>com.googlecode.json-simple:json-simple</exclude>
+                                <exclude>junit:junit</exclude>
+                                <exclude>org.hamcrest:hamcrest-core</exclude>
+                                <exclude>com.google.guava:guava</exclude>
+                                <exclude>com.google.code.gson:gson</exclude>
+                                <exclude>org.yaml:snakeyaml</exclude>
+                                <exclude>org.bukkit:bukkit</exclude>
+                                <exclude>us.dynmap:dynmap-api</exclude>
+                              </excludes>
+                            </artifactSet>                    
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
 	</build>
 </project>

src/main/java/com/wimbli/WorldBorder/WBListener.java

@@ -1,5 +1,6 @@
 package com.wimbli.WorldBorder;
 
+import org.bukkit.Chunk;
 import org.bukkit.event.EventHandler;
 import org.bukkit.event.EventPriority;
 import org.bukkit.event.Listener;
@@ -7,6 +8,7 @@ import org.bukkit.event.player.PlayerTeleportEvent;
 import org.bukkit.event.player.PlayerPortalEvent;
 import org.bukkit.event.world.ChunkLoadEvent;
 import org.bukkit.Location;
+import org.bukkit.event.world.ChunkUnloadEvent;
 
 
 public class WBListener implements Listener
@@ -68,4 +70,24 @@ public class WBListener implements Listener
 		Config.logWarn("Border-checking task was not running! Something on your server apparently killed it. It will now be restarted.");
 		Config.StartBorderTimer();
 	}
+
+    /*
+     * Check if there is a fill task running, and if yes, if it's for the
+     * world that the unload event refers to and if the chunk should be
+     * kept in memory because generation still needs it.
+     */
+    @EventHandler
+    public void onChunkUnload(ChunkUnloadEvent e)
+    {
+        if (Config.fillTask!=null)
+        {
+            Chunk chunk=e.getChunk();
+            if (e.getWorld() == Config.fillTask.getWorld() 
+            &&  Config.fillTask.chunkOnUnloadPreventionList(chunk.getX(), chunk.getZ()))
+            {
+                e.setCancelled(true);
+            }
+        }
+    }
+
 }

src/main/java/com/wimbli/WorldBorder/WorldFillTask.java

@@ -1,8 +1,6 @@
 package com.wimbli.WorldBorder;
 
 import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Set;
 
 import org.bukkit.Bukkit;
@@ -13,8 +11,10 @@ import org.bukkit.World;
 
 import com.wimbli.WorldBorder.Events.WorldBorderFillFinishedEvent;
 import com.wimbli.WorldBorder.Events.WorldBorderFillStartEvent;
+import io.papermc.lib.PaperLib;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 
 
 public class WorldFillTask implements Runnable
@@ -49,8 +49,6 @@ public class WorldFillTask implements Runnable
 	private transient int length = -1;
 	private transient int current = 0;
 	private transient boolean insideBorder = true;
-	private List<CoordXZ> storedChunks = new LinkedList<>();
-	private Set<CoordXZ> originalChunks = new HashSet<>();
 	private transient CoordXZ lastChunk = new CoordXZ(0, 0);
 
 	// for reporting progress back to user occasionally
@@ -60,8 +58,51 @@ public class WorldFillTask implements Runnable
 	private transient int reportTotal = 0;
 	private transient int reportNum = 0;
     
-    private transient boolean canUsePaperAPI = false;
+    // A map that holds to-be-loaded chunks, and their coordinates
-    private Set<CompletableFuture<Chunk>> pendingChunks;
+    private transient Map<CompletableFuture<Chunk>, CoordXZ> pendingChunks;
+    
+    // and a set of "Chunk a needed for Chunk b" dependencies, which
+    // unfortunately can't be a Map as a chunk might be needed for
+    // several others.
+    private transient Set<UnloadDependency> preventUnload;
+    
+    private class UnloadDependency
+    {
+        int neededX, neededZ;
+        int forX, forZ;
+        
+        UnloadDependency(int neededX, int neededZ, int forX, int forZ)
+        {
+            this.neededX=neededX;
+            this.neededZ=neededZ;
+            this.forX=forX;
+            this.forZ=forZ;
+        }
+        
+        @Override
+        public boolean equals(Object other)
+        {
+            if (other == null || !(other instanceof UnloadDependency))
+            {
+                return false;
+            }
+            return this.neededX == ((UnloadDependency) other).neededX
+               &&  this.neededZ == ((UnloadDependency) other).neededZ
+               &&  this.forX    == ((UnloadDependency) other).forX
+               &&  this.forZ    == ((UnloadDependency) other).forZ;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            int hash = 7;
+            hash = 79 * hash + this.neededX;
+            hash = 79 * hash + this.neededZ;
+            hash = 79 * hash + this.forX;
+            hash = 79 * hash + this.forZ;
+            return hash;
+        }
+    }
 
 	public WorldFillTask(Server theServer, Player player, String worldName, int fillDistance, int chunksPerRun, int tickFrequency, boolean forceLoad)
 	{
@@ -99,10 +140,8 @@ public class WorldFillTask implements Runnable
 			return;
 		}
         
-        canUsePaperAPI = checkForPaperAPI();
+        pendingChunks = new HashMap<>();
-        if (canUsePaperAPI) {
+        preventUnload = new HashSet<>();
-            pendingChunks = new HashSet<>();
-        }
 
 		this.border.setRadiusX(border.getRadiusX() + fillDistance);
 		this.border.setRadiusZ(border.getRadiusZ() + fillDistance);
@@ -118,17 +157,10 @@ public class WorldFillTask implements Runnable
 		//this.reportTarget = (this.border.getShape()) ? ((int) Math.ceil(chunkWidthX * chunkWidthZ / 4 * Math.PI + 2 * chunkWidthX)) : (chunkWidthX * chunkWidthZ);
 		//                                                                       Area of the ellipse                 just to be safe      area of the rectangle
 		
-		
-		// keep track of the chunks which are already loaded when the task starts, to not unload them
-		Chunk[] originals = world.getLoadedChunks();
-		for (Chunk original : originals)
-		{
-			originalChunks.add(new CoordXZ(original.getX(), original.getZ()));
-		}
-
 		this.readyToGo = true;
 		Bukkit.getServer().getPluginManager().callEvent(new WorldBorderFillStartEvent(this));
 	}
+    
 	// for backwards compatibility
 	public WorldFillTask(Server theServer, Player player, String worldName, int fillDistance, int chunksPerRun, int tickFrequency)
 	{
@@ -140,16 +172,6 @@ public class WorldFillTask implements Runnable
 		if (ID == -1) this.stop();
 		this.taskID = ID;
 	}
-    
-    private boolean checkForPaperAPI() {
-        try {
-            Class.forName("com.destroystokyo.paper.PaperConfig");
-            return true;
-        } catch (ClassNotFoundException e) {        
-            return false;
-        }
-    }
-
 
 	@Override
 	public void run()
@@ -177,39 +199,88 @@ public class WorldFillTask implements Runnable
 		// this is set so it only does one iteration at a time, no matter how frequently the timer fires
 		readyToGo = false;
 		// and this is tracked to keep one iteration from dragging on too long and possibly choking the system if the user specified a really high frequency
+		long loopStartTime = Config.Now();
+
+        // Process async results from last time. We don't make a difference
+        // whether they were really async, or sync.
         
-        if (canUsePaperAPI) {
+        // First, Check which chunk generations have been finished.
-            Set<CompletableFuture<Chunk>> newPendingChunks = new HashSet<>();
+        // Mark those chunks as existing and unloadable, and remove
-            for (CompletableFuture<Chunk> cf: pendingChunks) {
+        // them from the pending set.
-                if (cf.isDone()) {
+        int chunksProcessedLastTick = 0;
-                    try {
+        Map<CompletableFuture<Chunk>, CoordXZ> newPendingChunks = new HashMap<>();
-                        Chunk chunk=cf.get();
+        Set<CoordXZ> chunksToUnload = new HashSet<>();
-                        // System.out.println(chunk);
+        for (CompletableFuture<Chunk> cf: pendingChunks.keySet())
-                        if (chunk==null)
+        {
-                            continue;
+            if (cf.isDone())
-                        CoordXZ xz = new CoordXZ(chunk.getX(), chunk.getZ());
+            {
-                        worldData.chunkExistsNow(xz.x, xz.z);
+                ++chunksProcessedLastTick;
-                        storedChunks.add(xz);
+                // If cf.get() returned the chunk reliably, pendingChunks could
-                    } catch (InterruptedException | ExecutionException ex) {
+                // be a set and we wouldn't have to map CFs to coords ...
-                        Config.log(ex.getMessage());
+                CoordXZ xz=pendingChunks.get(cf);
-                    }
+                worldData.chunkExistsNow(xz.x, xz.z);
-                } else {
+                chunksToUnload.add(xz);
-                    newPendingChunks.add(cf);
-                }
             }
-            pendingChunks=newPendingChunks;
+            else
-            if (pendingChunks.size() > chunksPerRun*2) {
+            {
-                readyToGo = true;
+                newPendingChunks.put(cf, pendingChunks.get(cf));
-                return;
+            }
+        }
+        pendingChunks = newPendingChunks;
+
+        // Next, check which chunks had been loaded because a to-be-generated
+        // chunk needed them, and don't have to remain in memory any more.
+        Set<UnloadDependency> newPreventUnload = new HashSet<>();
+        for (UnloadDependency dependency: preventUnload) 
+        {
+            if (worldData.doesChunkExist(dependency.forX, dependency.forZ)) 
+            {
+                chunksToUnload.add(new CoordXZ(dependency.neededX, dependency.neededZ));
+            }
+            else
+            {
+                newPreventUnload.add(dependency);
+            }
+        }
+        preventUnload = newPreventUnload;
+
+        // Unload all chunks that aren't needed anymore. NB a chunk could have
+        // been needed for two different others, or been generated and needed
+        // for one other chunk, so it might be in the unload set wrongly.
+        // The ChunkUnloadListener checks this anyway, but it doesn't hurt to
+        // save a few µs by not even requesting the unload.
+
+        for (CoordXZ unload: chunksToUnload)
+        {
+            if (!chunkOnUnloadPreventionList(unload.x, unload.z))
+            {
+                world.unloadChunkRequest(unload.x, unload.z);
             }
         }
 
-		long loopStartTime = Config.Now();
+        // Put some damper on chunksPerRun. We don't want the queue to be too
-		for (int loop = 0; loop < chunksPerRun; loop++)
+        // full; only fill it to a bit more than what we can
+        // process per tick. This ensures the background task can run at
+        // full speed and we recover from a temporary drop in generation rate,
+        // but doesn't push user-induced chunk generations behind a very
+        // long queue of fill-generations.
+
+        int chunksToProcess = chunksPerRun;
+        if (chunksProcessedLastTick > 0 || pendingChunks.size() > 0)
+        {
+            // Note we generally queue 3 chunks, so real numbers are 1/3 of chunksProcessedLastTick and pendingchunks.size
+            int chunksExpectedToGetProcessed = (chunksProcessedLastTick - pendingChunks.size()) / 3 + 3;
+            if (chunksExpectedToGetProcessed < chunksToProcess)
+                chunksToProcess = chunksExpectedToGetProcessed;
+        }
+
+		for (int loop = 0; loop < chunksToProcess; loop++)
 		{
 			// in case the task has been paused while we're repeating...
 			if (paused || pausedForMemory)
+            {
 				return;
+            }
 
 			long now = Config.Now();
 
@@ -228,7 +299,9 @@ public class WorldFillTask implements Runnable
 			while (!border.insideBorder(CoordXZ.chunkToBlock(x) + 8, CoordXZ.chunkToBlock(z) + 8))
 			{
 				if (!moveToNext())
+                {
 					return;
+                }
 			}
 			insideBorder = true;
 
@@ -241,7 +314,9 @@ public class WorldFillTask implements Runnable
 					rLoop++;
 					insideBorder = true;
 					if (!moveToNext())
+                    {
 						return;
+                    }
 					if (rLoop > 255)
 					{	// only skim through max 256 chunks (~8 region files) at a time here, to allow process to take a break if needed
 						readyToGo = true;
@@ -250,51 +325,26 @@ public class WorldFillTask implements Runnable
 				}
 			}
 
-			// load the target chunk and generate it if necessary
+            pendingChunks.put(PaperLib.getChunkAtAsync(world, x, z, true), new CoordXZ(x, z));
-            if (canUsePaperAPI) {
-                pendingChunks.add(world.getChunkAtAsync(x, z, true));
-            } else {
-                world.loadChunk(x, z, true);
-                worldData.chunkExistsNow(x, z);
-    			storedChunks.add(new CoordXZ(x, z));
-            }
 
 			// There need to be enough nearby chunks loaded to make the server populate a chunk with trees, snow, etc.
 			// So, we keep the last few chunks loaded, and need to also temporarily load an extra inside chunk (neighbor closest to center of map)
 			int popX = !isZLeg ? x : (x + (isNeg ? -1 : 1));
 			int popZ = isZLeg ? z : (z + (!isNeg ? -1 : 1));
+
+            pendingChunks.put(PaperLib.getChunkAtAsync(world, popX, popZ, false), new CoordXZ(popX, popZ));
+            preventUnload.add(new UnloadDependency(popX, popZ, x, z));
             
-            if (canUsePaperAPI) {
-                pendingChunks.add(world.getChunkAtAsync(popX, popZ, false));
-            } else {
-                world.loadChunk(popX, popZ, false);
-    			storedChunks.add(new CoordXZ(popX, popZ));
-            }
-
 			// make sure the previous chunk in our spiral is loaded as well (might have already existed and been skipped over)
-			if (!storedChunks.contains(lastChunk) && !originalChunks.contains(lastChunk))
+            pendingChunks.put(PaperLib.getChunkAtAsync(world, lastChunk.x, lastChunk.z, false), new CoordXZ(lastChunk.x, lastChunk.z)); // <-- new CoordXZ as lastChunk isn't immutable
-			{
+            preventUnload.add(new UnloadDependency(lastChunk.x, lastChunk.z, x, z));
-                if (canUsePaperAPI) {
-                    pendingChunks.add(world.getChunkAtAsync(lastChunk.x, lastChunk.z, false));
-                } else {
-                    world.loadChunk(lastChunk.x, lastChunk.z, false);
-                    storedChunks.add(new CoordXZ(lastChunk.x, lastChunk.z));
-                }
-			}
-
-			// If enough stored chunks are buffered in, go ahead and unload the oldest to free up memory
-			while (storedChunks.size() > 8)
-			{
-				CoordXZ coord = storedChunks.remove(0);
-				if (!originalChunks.contains(coord))
-					world.unloadChunkRequest(coord.x, coord.z);
-			}
 
 			// move on to next chunk
 			if (!moveToNext())
+            {
 				return;
+            }
 		}
-
 		// ready for the next iteration to run
 		readyToGo = true;
 	}
@@ -401,13 +451,15 @@ public class WorldFillTask implements Runnable
 			server.getScheduler().cancelTask(taskID);
 		server = null;
 
-		// go ahead and unload any chunks we still have loaded
+        // go ahead and unload any chunks we still have loaded
-		while(!storedChunks.isEmpty())
+        // Set preventUnload to emptry first so the ChunkUnloadEvent Listener
-		{
+        // doesn't get in our way
-			CoordXZ coord = storedChunks.remove(0);
+        Set<UnloadDependency> tempPreventUnload = preventUnload;
-			if (!originalChunks.contains(coord))
+        preventUnload = null;
-				world.unloadChunkRequest(coord.x, coord.z);
+        for (UnloadDependency entry: tempPreventUnload)
-		}
+        {
+            world.unloadChunkRequest(entry.neededX, entry.neededZ);
+        }
 	}
 
 	// is this task still valid/workable?
@@ -442,6 +494,26 @@ public class WorldFillTask implements Runnable
 	{
 		return this.paused || this.pausedForMemory;
 	}
+    
+    public boolean chunkOnUnloadPreventionList(int x, int z)
+    {
+        if (preventUnload != null)
+        {
+            for (UnloadDependency entry: preventUnload)
+            {
+                if (entry.neededX == x && entry.neededZ == z)
+                {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    
+    public World getWorld()
+    {
+        return world;
+    }
 
 	// let the user know how things are coming along
 	private void reportProgress()