Add autogenerate-to-visibilitylimits option to allow automatic priming/generating of chunks within defined visibility area

Avoid rendering tiles with nothing but empty chunks
Limit size of hash-file cache, and move hash files to world-specific tile directories

configuration.txt

@@ -302,7 +302,7 @@ worlds:
   #    - x: -15000
   #      y: 64
   #      z: -5000
-  #  # Use visbilitylimits to restrict which areas of maps on your world to render (zero or more rectangles can be defined)
+  #  # Use visibilitylimits to restrict which areas of maps on your world to render (zero or more rectangles can be defined)
   #  visibilitylimits:
   #    - x0: -1000
   #      z0: -1000
@@ -314,6 +314,10 @@ worlds:
   #      z1: -500
   #  # Use hidestyle to control how hidden-but-existing chunks are to be rendered (air=empty air (same as ungenerated), stone=a flat stone plain, ocean=a flat ocean)
   #  hidestyle: stone
+  #  # Use 'autogenerate-to-visibilitylimits: true' to choose to force the generation of ungenerated chunks while rendering maps on this world, for any chunks within the defined
+  #  # visibilitylimits (limits must be set).  This will result in initializing all game world areas within the visible ranges that have not yet been initialized.
+  #  # Note: BE SURE YOU WANT TO DO THIS - there isn't a good way to "ungenerate" terrain chunks once generated (although tools like WorldEdit can regenerate them) 
+  #  autogenerate-to-visibilitylimits: false
   #   Use 'template: mycustomtemplate' to use the properties specified in the template 'mycustomtemplate' to this world. Default it is set to the environment-name (normal or nether).
   #  template: mycustomtemplate
   #   Rest of comes from template - uncomment to tailor for world specifically

src/main/java/org/dynmap/DynmapWorld.java

@@ -27,6 +27,7 @@ public class DynmapWorld {
     public ConfigurationNode configuration;
     public List<Location> seedloc;
     public List<MapChunkCache.VisibilityLimit> visibility_limits;
+    public boolean do_autogenerate;
     public MapChunkCache.HiddenChunkStyle hiddenchunkstyle;
     public int servertime;
     public boolean sendposition;

src/main/java/org/dynmap/MapManager.java

@@ -108,6 +108,7 @@ public class MapManager {
         MapTile tile = null;
         int rendercnt = 0;
         CommandSender sender;
+        long starttime;
 
         /* Full world, all maps render */
         FullWorldRenderState(DynmapWorld dworld, Location l, CommandSender sender) {
@@ -143,8 +144,9 @@ public class MapManager {
                 /* If render queue is empty, start next map */
                 if(renderQueue.isEmpty()) {
                     if(map_index >= 0) { /* Finished a map? */
+                        double msecpertile = (double)(tstart - starttime) / (double)((rendercnt>0)?rendercnt:1);
                         sender.sendMessage("Full render of map '" + world.maps.get(map_index).getClass().getSimpleName() + "' of world '" +
-                                 world.world.getName() + "' completed - " + rendercnt + " tiles rendered.");
+                                 world.world.getName() + "' completed - " + rendercnt + " tiles rendered (" + String.format("%.2f", msecpertile) + " msec/tile).");
                     }                	
                     found.clear();
                     rendered.clear();
@@ -156,6 +158,7 @@ public class MapManager {
                         return;
                     }
                     map = world.maps.get(map_index);
+                    starttime = System.currentTimeMillis();
 
                     /* Now, prime the render queue */
                     for (MapTile mt : map.getTiles(loc)) {
@@ -192,10 +195,11 @@ public class MapManager {
                 return; /* Cancelled/aborted */
             }
             if(tile0 != null) {    /* Single tile? */
-                render(cache, tile);    /* Just render */
+                if(cache.isEmpty() == false)
+                    render(cache, tile);    /* Just render */
             }
             else {
-                if (render(cache, tile)) {
+                if ((cache.isEmpty() == false) && render(cache, tile)) {
                     found.remove(tile);
                     rendered.add(tile);
                     for (MapTile adjTile : map.getAdjecentTiles(tile)) {
@@ -206,10 +210,13 @@ public class MapManager {
                     }
                 }
                 found.remove(tile);
-                rendercnt++;
+                if(!cache.isEmpty()) {
-                if((rendercnt % 100) == 0) {
+                    rendercnt++;
-                    sender.sendMessage("Full render of map '" + world.maps.get(map_index).getClass().getSimpleName() + "' on world '" +
+                    if((rendercnt % 100) == 0) {
-                            w.getName() + "' in progress - " + rendercnt + " tiles rendered, " + renderQueue.size() + " tiles pending.");
+                        double msecpertile = (double)(System.currentTimeMillis() - starttime) / (double)rendercnt;
+                        sender.sendMessage("Full render of map '" + world.maps.get(map_index).getClass().getSimpleName() + "' on world '" +
+                            w.getName() + "' in progress - " + rendercnt + " tiles rendered (" + String.format("%.2f", msecpertile) + " msec/tile).");
+                    }
                 }
             }
             /* And unload what we loaded */
@@ -384,6 +391,11 @@ public class MapManager {
                 dynmapWorld.seedloc.add(new Location(w, (lim.x0+lim.x1)/2, 64, (lim.z0+lim.z1)/2));
             }            
         }
+        dynmapWorld.do_autogenerate = worldConfiguration.getBoolean("autogenerate-to-visibilitylimits", false);
+        if(dynmapWorld.do_autogenerate && (dynmapWorld.visibility_limits == null)) {
+            Log.info("Warning: Automatic world generation to visible limits option requires that visiblelimits be set - option disabled");
+            dynmapWorld.do_autogenerate = false;
+        }
         String hiddenchunkstyle = worldConfiguration.getString("hidestyle", "stone");
         if(hiddenchunkstyle.equals("air"))
             dynmapWorld.hiddenchunkstyle = MapChunkCache.HiddenChunkStyle.FILL_AIR;
@@ -516,6 +528,7 @@ public class MapManager {
                 c.setVisibleRange(limit);
             }
             c.setHiddenFillStyle(w.hiddenchunkstyle);
+            c.setAutoGenerateVisbileRanges(w.do_autogenerate);
         }
         c.setChunks(w.world, chunks);
         if(c.setChunkDataTypes(blockdata, biome, highesty, rawbiome) == false)

src/main/java/org/dynmap/TileHashManager.java

@@ -3,6 +3,7 @@ import java.io.File;
 import java.io.RandomAccessFile;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.Map;
 import java.io.IOException;
 import java.util.zip.CRC32;
 
@@ -45,8 +46,15 @@ public class TileHashManager {
         }
         
         public File getHashFile(File tiledir) {
-            if(hf == null)
+            if(hf == null) {
-                hf = new File(tiledir, key + (subtype.equals("")?"":("." + subtype)) + "_" + x + "_" + y + ".hash");
+                String k;
+                int idx = key.indexOf('.'); /* Find first '.' - world name split */
+                if(idx > 0)
+                    k = key.substring(0, idx) + File.separatorChar + key.substring(idx+1);
+                else
+                    k = key;
+                hf = new File(tiledir, k + (subtype.equals("")?"":("." + subtype)) + "_" + x + "_" + y + ".hash");
+            }
             return hf;
         }
         /* Write to file */
@@ -88,8 +96,21 @@ public class TileHashManager {
                 crcbuf[off+i] = (byte)((crc >> ((3-i)*8)) & 0xFF);
         }
     }
+    
+    public static class LRULinkedHashMap<T, K> extends LinkedHashMap<T, K> {
+        private int limit;
+        public LRULinkedHashMap(int lim) {
+            super(16, (float)0.75, true);
+            limit = lim;
+        }
+        protected boolean removeEldestEntry(Map.Entry<T, K> last) {
+            return(size() >= limit);
+        }
+    }
+    
+    private static final int MAX_CACHED_TILEHASHFILES = 25;
     private Object lock = new Object();
-    private LinkedHashMap<TileHashFile, byte[]> tilehash = new LinkedHashMap<TileHashFile, byte[]>(16, (float) 0.75, true);
+    private LRULinkedHashMap<TileHashFile, byte[]> tilehash = new LRULinkedHashMap<TileHashFile, byte[]>(MAX_CACHED_TILEHASHFILES);
     private CRC32 crc32 = new CRC32();
     
     public TileHashManager(File tileroot, boolean enabled) {

src/main/java/org/dynmap/utils/LegacyMapChunkCache.java

@@ -28,6 +28,8 @@ public class LegacyMapChunkCache implements MapChunkCache {
     private World w;
     private List<DynmapChunk> chunks;
     private ListIterator<DynmapChunk> iterator;
+    private boolean do_generate;
+    private boolean isempty = true;
 
     private int x_min, x_max, z_min, z_max;
     private int x_dim;
@@ -276,7 +278,11 @@ public class LegacyMapChunkCache implements MapChunkCache {
                     }
                 }
                 boolean wasLoaded = w.isChunkLoaded(chunk.x, chunk.z);
-                boolean didload = w.loadChunk(chunk.x, chunk.z, false);
+                boolean didload = w.loadChunk(chunk.x, chunk.z, do_generate && vis);
+                boolean didgenerate = false;
+                /* If we didn't load, and we're supposed to generate, do it */
+                if((!didload) && do_generate && vis)
+                    didgenerate = didload = w.loadChunk(chunk.x, chunk.z, true);
                 /* If it did load, make cache of it */
                 if(didload) {
                     LegacyChunkSnapshot ss = null;
@@ -306,17 +312,20 @@ public class LegacyMapChunkCache implements MapChunkCache {
                 if ((!wasLoaded) && didload) {
                     /* It looks like bukkit "leaks" entities - they don't get removed from the world-level table
                      * when chunks are unloaded but not saved - removing them seems to do the trick */
-                    Chunk cc = w.getChunkAt(chunk.x, chunk.z);
+                    if(!didgenerate) {
-                    if(cc != null) {
+                        Chunk cc = w.getChunkAt(chunk.x, chunk.z);
-                        for(Entity e: cc.getEntities())
+                        if(cc != null) {
-                            e.remove();
+                            for(Entity e: cc.getEntities())
+                                e.remove();
+                        }
                     }
                     /* Since we only remember ones we loaded, and we're synchronous, no player has
                      * moved, so it must be safe (also prevent chunk leak, which appears to happen
                      * because isChunkInUse defined "in use" as being within 256 blocks of a player,
                      * while the actual in-use chunk area for a player where the chunks are managed
-                     * by the MC base server is 21x21 (or about a 160 block radius) */
+                     * by the MC base server is 21x21 (or about a 160 block radius).
-                    w.unloadChunk(chunk.x, chunk.z, false, false);
+                     * Also, if we did generate it, need to save it */
+                    w.unloadChunk(chunk.x, chunk.z, didgenerate, false);
                     /* And pop preserved chunk - this is a bad leak in Bukkit for map traversals like us */
                     try {
                         if(poppreservedchunk != null)
@@ -330,10 +339,13 @@ public class LegacyMapChunkCache implements MapChunkCache {
         }
         /* If done, finish table */
         if(iterator.hasNext() == false) {
+            isempty = true;
             /* Fill missing chunks with empty dummy chunk */
             for(int i = 0; i < snaparray.length; i++) {
                 if(snaparray[i] == null)
                     snaparray[i] = EMPTY;
+                else if(snaparray[i] != EMPTY)
+                    isempty = false;
             }
         }
         return cnt;
@@ -346,6 +358,12 @@ public class LegacyMapChunkCache implements MapChunkCache {
             return !iterator.hasNext();
         return false;
     }
+    /**
+     * Test if all empty blocks
+     */
+    public boolean isEmpty() {
+        return isempty;
+    }
     /**
      * Unload chunks
      */
@@ -435,6 +453,16 @@ public class LegacyMapChunkCache implements MapChunkCache {
             visible_limits = new ArrayList<VisibilityLimit>();
         visible_limits.add(limit);
     }
+    /**
+     * Set autogenerate - must be done after at least one visible range has been set
+     */
+    public void setAutoGenerateVisbileRanges(boolean do_generate) {
+        if(do_generate && ((visible_limits == null) || (visible_limits.size() == 0))) {
+            Log.severe("Cannot setAutoGenerateVisibleRanges() without visible ranges defined");
+            return;
+        }
+        this.do_generate = do_generate;
+    }
     @Override
     public boolean setChunkDataTypes(boolean blockdata, boolean biome, boolean highestblocky, boolean rawbiome) {
         if(biome || rawbiome)   /* Legacy doesn't support these */

src/main/java/org/dynmap/utils/MapChunkCache.java

@@ -37,6 +37,10 @@ public interface MapChunkCache {
      * Test if done loading
      */
     boolean isDoneLoading();
+    /**
+     * Test if all empty blocks
+     */
+    boolean isEmpty();
     /**
      * Unload chunks
      */
@@ -87,4 +91,8 @@ public interface MapChunkCache {
      * Coordinates are block coordinates
      */
     public void setVisibleRange(VisibilityLimit limit);
+    /**
+     * Set autogenerate - must be done after at least one visible range has been set
+     */
+    public void setAutoGenerateVisbileRanges(boolean do_generate);
 }

src/main/java/org/dynmap/utils/NewMapChunkCache.java

@@ -31,6 +31,8 @@ public class NewMapChunkCache implements MapChunkCache {
     private boolean biome, biomeraw, highesty, blockdata;
     private HiddenChunkStyle hidestyle = HiddenChunkStyle.FILL_AIR;
     private List<VisibilityLimit> visible_limits = null;
+    private boolean do_generate = false;
+    private boolean isempty = true;
 
     private ChunkSnapshot[] snaparray; /* Index = (x-x_min) + ((z-z_min)*x_dim) */
     
@@ -296,7 +298,11 @@ public class NewMapChunkCache implements MapChunkCache {
                 }
             }
             boolean wasLoaded = w.isChunkLoaded(chunk.x, chunk.z);
-            boolean didload = w.loadChunk(chunk.x, chunk.z, false);
+            boolean didload = w.loadChunk(chunk.x, chunk.z, do_generate && vis);
+            boolean didgenerate = false;
+            /* If we didn't load, and we're supposed to generate, do it */
+            if((!didload) && do_generate && vis)
+                didgenerate = didload = w.loadChunk(chunk.x, chunk.z, true);
             /* If it did load, make cache of it */
             if(didload) {
                 ChunkSnapshot ss = null;
@@ -329,17 +335,20 @@ public class NewMapChunkCache implements MapChunkCache {
             if ((!wasLoaded) && didload) {
                 /* It looks like bukkit "leaks" entities - they don't get removed from the world-level table
                  * when chunks are unloaded but not saved - removing them seems to do the trick */
-                Chunk cc = w.getChunkAt(chunk.x, chunk.z);
+                if(!didgenerate) {
-                if(cc != null) {
+                    Chunk cc = w.getChunkAt(chunk.x, chunk.z);
-                    for(Entity e: cc.getEntities())
+                    if(cc != null) {
-                        e.remove();
+                        for(Entity e: cc.getEntities())
+                            e.remove();
+                    }
                 }
                 /* Since we only remember ones we loaded, and we're synchronous, no player has
                  * moved, so it must be safe (also prevent chunk leak, which appears to happen
                  * because isChunkInUse defined "in use" as being within 256 blocks of a player,
                  * while the actual in-use chunk area for a player where the chunks are managed
-                 * by the MC base server is 21x21 (or about a 160 block radius) */
+                 * by the MC base server is 21x21 (or about a 160 block radius).
-                w.unloadChunk(chunk.x, chunk.z, false, false);
+                 * Also, if we did generate it, need to save it */
+                w.unloadChunk(chunk.x, chunk.z, didgenerate, false);
                 /* And pop preserved chunk - this is a bad leak in Bukkit for map traversals like us */
                 try {
                     if(poppreservedchunk != null)
@@ -351,10 +360,13 @@ public class NewMapChunkCache implements MapChunkCache {
             cnt++;
         }
         if(iterator.hasNext() == false) {   /* If we're done */
+            isempty = true;
             /* Fill missing chunks with empty dummy chunk */
             for(int i = 0; i < snaparray.length; i++) {
                 if(snaparray[i] == null)
                     snaparray[i] = EMPTY;
+                else if(snaparray[i] != EMPTY)
+                    isempty = false;
             }
         }
         return cnt;
@@ -367,6 +379,12 @@ public class NewMapChunkCache implements MapChunkCache {
             return !iterator.hasNext();
         return false;
     }
+    /**
+     * Test if all empty blocks
+     */
+    public boolean isEmpty() {
+        return isempty;
+    }
     /**
      * Unload chunks
      */
@@ -436,6 +454,16 @@ public class NewMapChunkCache implements MapChunkCache {
     public void setHiddenFillStyle(HiddenChunkStyle style) {
         this.hidestyle = style;
     }
+    /**
+     * Set autogenerate - must be done after at least one visible range has been set
+     */
+    public void setAutoGenerateVisbileRanges(boolean do_generate) {
+        if(do_generate && ((visible_limits == null) || (visible_limits.size() == 0))) {
+            Log.severe("Cannot setAutoGenerateVisibleRanges() without visible ranges defined");
+            return;
+        }
+        this.do_generate = do_generate;
+    }
     /**
      * Add visible area limit - can be called more than once 
      * Needs to be set before chunks are loaded