From 91c66d9f5861f46dc48a6340c943382be1e8a999 Mon Sep 17 00:00:00 2001 From: Brettflan Date: Fri, 10 Feb 2012 20:20:47 -0600 Subject: [PATCH] Fill process now checks the world region files directly to determine which chunks have been previously generated and skips over them, so existing worlds where much of the world is already generated should get through the fill process much faster. Fill process should run much more smoothly regardless of what frequency you've specified. If the fill process is paused due to low memory, it now tries to prod Java into cleaning up memory, so the process usually is able to start right back up almost immediately. A few other minor tweaks to the fill process. Moved world region file handling code out to new separate WorldFileData class, which is used by both the trim process and the fill process. --- src/com/wimbli/WorldBorder/WorldFileData.java | 259 ++++++++++++++++++ src/com/wimbli/WorldBorder/WorldFillTask.java | 75 ++++- src/com/wimbli/WorldBorder/WorldTrimTask.java | 79 ++---- 3 files changed, 341 insertions(+), 72 deletions(-) create mode 100644 src/com/wimbli/WorldBorder/WorldFileData.java diff --git a/src/com/wimbli/WorldBorder/WorldFileData.java b/src/com/wimbli/WorldBorder/WorldFileData.java new file mode 100644 index 0000000..b25d59e --- /dev/null +++ b/src/com/wimbli/WorldBorder/WorldFileData.java @@ -0,0 +1,259 @@ +package com.wimbli.WorldBorder; + +import java.io.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.bukkit.entity.Player; +import org.bukkit.World; + +// image output stuff, for debugging method at bottom of this file +import java.awt.*; +import java.awt.image.*; +import javax.imageio.*; + + +public class WorldFileData +{ + private transient World world; + private transient File regionFolder = null; + private transient File[] regionFiles = null; + private transient Player notifyPlayer = null; + private transient Map> regionChunkExistence = Collections.synchronizedMap(new HashMap>()); + + // Use this static method to create a new instance of this class. If null is returned, there was a problem so any process relying on this should be cancelled. + public static WorldFileData create(World world, Player notifyPlayer) + { + WorldFileData newData = new WorldFileData(world, notifyPlayer); + + newData.regionFolder = new File(newData.world.getWorldFolder(), "region"); + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + String mainRegionFolder = newData.regionFolder.getPath(); + newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + String subRegionFolder = newData.regionFolder.getPath(); + newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + newData.sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+newData.regionFolder.getPath()); + return null; + } + } + } + newData.regionFiles = newData.regionFolder.listFiles(new RegionFileFilter()); + + if (newData.regionFiles == null || newData.regionFiles.length == 0) + { + newData.sendMessage("Could not find any region files. Looked in: "+newData.regionFolder.getPath()); + return null; + } + + return newData; + } + + // the constructor is private; use create() method above to create an instance of this class. + private WorldFileData(World world, Player notifyPlayer) + { + this.world = world; + this.notifyPlayer = notifyPlayer; + } + + + // number of region files this world has + public int regionFileCount() + { + return regionFiles.length; + } + + // folder where world's region files are located + public File regionFolder() + { + return regionFolder; + } + + // return entire list of region files + public File[] regionFiles() + { + return regionFiles.clone(); + } + + // return a region file by index + public File regionFile(int index) + { + if (regionFiles.length < index) + return null; + return regionFiles[index]; + } + + // get the X and Z world coordinates of the region from the filename + public CoordXZ regionFileCoordinates(int index) + { + File regionFile = this.regionFile(index); + String[] coords = regionFile.getName().split("\\."); + int x, z; + try + { + x = Integer.parseInt(coords[1]); + z = Integer.parseInt(coords[2]); + return new CoordXZ (x, z); + } + catch(Exception ex) + { + sendMessage("Error! Region file found with abnormal name: "+regionFile.getName()); + return null; + } + } + + + // Find out if the chunk at the given coordinates exists. + public boolean doesChunkExist(int x, int z) + { + CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z)); + List regionChunks = this.getRegionData(region); +// Bukkit.getLogger().info("x: "+x+" z: "+z+" offset: "+coordToRegionOffset(x, z)); + return regionChunks.get(coordToRegionOffset(x, z)).booleanValue(); + } + + // Find out if the chunk at the given coordinates has been fully generated. + // Minecraft only fully generates a chunk when adjacent chunks are also loaded. + public boolean isChunkFullyGenerated(int x, int z) + { // if all adjacent chunks exist, it should be a safe enough bet that this one is fully generated + return + ! ( + ! doesChunkExist(x, z) + || ! doesChunkExist(x+1, z) + || ! doesChunkExist(x-1, z) + || ! doesChunkExist(x, z+1) + || ! doesChunkExist(x, z-1) + ); + } + + // Method to let us know a chunk has been generated, to update our region map. + public void chunkExistsNow(int x, int z) + { + CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z)); + List regionChunks = this.getRegionData(region); + regionChunks.set(coordToRegionOffset(x, z), true); + } + + + + // region is 32 X 32 chunks; chunk pointers are stored in region file at position: x + z*32 (max 1024) + // input x and z values can be world-based chunk coordinates or local-to-region chunk coordinates either one + private int coordToRegionOffset(int x, int z) + { + // "%" modulus is used to convert potential world coordinates to definitely be local region coordinates + x = x % 32; + z = z % 32; + // similarly, for local coordinates, we need to wrap negative values around + if (x < 0) x += 32; + if (z < 0) z += 32; + // return offset position for the now definitely local x and z values + return (x + (z * 32)); + } + + private List getRegionData(CoordXZ region) + { + List data = regionChunkExistence.get(region); + if (data != null) + return data; + + // data for the specified region isn't loaded yet, so init it as empty and try to find the file and load the data + data = new ArrayList(1024); + for (int i = 0; i < 1024; i++) + { + data.add(Boolean.FALSE); + } + + for (int i = 0; i < regionFiles.length; i++) + { + CoordXZ coord = regionFileCoordinates(i); + // is this region file the one we're looking for? + if ( ! coord.equals(region)) + continue; + + int counter = 0; + try + { + RandomAccessFile regionData = new RandomAccessFile(this.regionFile(i), "r"); + // first 4096 bytes of region file consists of 4-byte int pointers to chunk data in the file + for (int j = 0; j < 1024; j++) + { + // if chunk pointer data is 0, chunk doesn't exist yet; otherwise, it does + if (regionData.readInt() != 0) + data.set(j, true); + counter++; + } + } + catch (FileNotFoundException ex) + { + sendMessage("Error! Could not open region file to find generated chunks: "+this.regionFile(i).getName()); + } + catch (IOException ex) + { + sendMessage("Error! Could not read region file to find generated chunks: "+this.regionFile(i).getName()); + } + } + regionChunkExistence.put(region, data); +// testImage(region, data); + return data; + } + + // send a message to the server console/log and possibly to an in-game player + private void sendMessage(String text) + { + Config.Log("[WorldData] " + text); + if (notifyPlayer != null && notifyPlayer.isOnline()) + notifyPlayer.sendMessage("[WorldData] " + text); + } + + // filter for region files + private static class RegionFileFilter implements FileFilter + { + @Override + public boolean accept(File file) + { + return ( + file.exists() + && file.isFile() + && file.getName().toLowerCase().endsWith(".mcr") + ); + } + } + + + +// crude chunk map PNG image output, for debugging + private void testImage(CoordXZ region, List data) { + int width = 32; + int height = 32; + BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + Graphics2D g2 = bi.createGraphics(); + int current = 0; + g2.setColor(Color.BLACK); + + for (int x = 0; x < 32; x++) + { + for (int z = 0; z < 32; z++) + { + if (data.get(current).booleanValue()) + g2.fillRect(x,z, x+1, z+1); + current++; + } + } + + File f = new File("region_"+region.x+"_"+region.z+"_.png"); + Config.Log(f.getAbsolutePath()); + try { + // png is an image format (like gif or jpg) + ImageIO.write(bi, "png", f); + } catch (IOException ex) { + Config.Log("[SEVERE]"+ex.getLocalizedMessage()); + } + } +} diff --git a/src/com/wimbli/WorldBorder/WorldFillTask.java b/src/com/wimbli/WorldBorder/WorldFillTask.java index c33631a..6e383eb 100644 --- a/src/com/wimbli/WorldBorder/WorldFillTask.java +++ b/src/com/wimbli/WorldBorder/WorldFillTask.java @@ -17,6 +17,7 @@ public class WorldFillTask implements Runnable private transient Server server = null; private transient World world = null; private transient BorderData border = null; + private transient WorldFileData worldData = null; private transient boolean readyToGo = false; private transient boolean paused = false; private transient boolean pausedForMemory = false; @@ -43,9 +44,11 @@ public class WorldFillTask implements Runnable private transient boolean insideBorder = true; private List storedChunks = new LinkedList(); private Set originalChunks = new HashSet(); + private transient CoordXZ lastChunk = new CoordXZ(0, 0); // for reporting progress back to user occasionally private transient long lastReport = Config.Now(); + private transient int reportSkipped = 0; private transient int reportTarget = 0; private transient int reportTotal = 0; private transient int reportNum = 0; @@ -78,6 +81,14 @@ public class WorldFillTask implements Runnable return; } + // load up a new WorldFileData for the world in question, used to scan region files for which chunks are already fully generated and such + worldData = WorldFileData.create(world, notifyPlayer); + if (worldData == null) + { + this.stop(); + return; + } + this.border.setRadius(border.getRadius() + fillDistance); this.x = CoordXZ.blockToChunk((int)border.getX()); this.z = CoordXZ.blockToChunk((int)border.getZ()); @@ -117,6 +128,7 @@ public class WorldFillTask implements Runnable return; pausedForMemory = false; + readyToGo = true; sendMessage("Available memory is sufficient, automatically continuing."); } @@ -125,6 +137,8 @@ public class WorldFillTask implements Runnable // this is set so it only does one iteration at a time, no matter how frequently the timer fires readyToGo = false; + // and this is tracked to keep one iteration from dragging on too long and possibly choking the system if the user specified a really high frequency + long loopStartTime = Config.Now(); for (int loop = 0; loop < chunksPerRun; loop++) { @@ -132,10 +146,19 @@ public class WorldFillTask implements Runnable if (paused || pausedForMemory) return; + long now = Config.Now(); + // every 5 seconds or so, give basic progress report to let user know how it's going - if (Config.Now() > lastReport + 5000) + if (now > lastReport + 5000) reportProgress(); + // if this iteration has been running for 250ms (~5 ticks) or more, stop to take a breather + if (now > loopStartTime + 250) + { + readyToGo = true; + return; + } + // if we've made it at least partly outside the border, skip past any such chunks while (!border.insideBorder(CoordXZ.chunkToBlock(x) + 8, CoordXZ.chunkToBlock(z) + 8)) { @@ -144,16 +167,18 @@ public class WorldFillTask implements Runnable } insideBorder = true; - // skip past any chunks which are currently loaded (they're definitely already generated) - while (world.isChunkLoaded(x, z)) + // skip past any chunks which are confirmed as fully generated using our super-special isChunkFullyGenerated routine + while (worldData.isChunkFullyGenerated(x, z)) { + reportSkipped++; insideBorder = true; if (!moveToNext()) return; } - // load the target chunk and generate it if necessary (no way to check if chunk has been generated first, simply have to load it) + // load the target chunk and generate it if necessary world.loadChunk(x, z, true); + worldData.chunkExistsNow(x, z); // There need to be enough nearby chunks loaded to make the server populate a chunk with trees, snow, etc. // So, we keep the last few chunks loaded, and need to also temporarily load an extra inside chunk (neighbor closest to center of map) @@ -161,19 +186,23 @@ public class WorldFillTask implements Runnable int popZ = isZLeg ? z : (z + (!isNeg ? -1 : 1)); world.loadChunk(popX, popZ, false); + // make sure the previous chunk in our spiral is loaded as well (might have already existed and been skipped over) + if (!storedChunks.contains(lastChunk) && !originalChunks.contains(lastChunk)) + { + world.loadChunk(lastChunk.x, lastChunk.z, false); + storedChunks.add(new CoordXZ(lastChunk.x, lastChunk.z)); + } + // Store the coordinates of these latest 2 chunks we just loaded, so we can unload them after a bit... - storedChunks.add(new CoordXZ(x, z)); storedChunks.add(new CoordXZ(popX, popZ)); + storedChunks.add(new CoordXZ(x, z)); // If enough stored chunks are buffered in, go ahead and unload the oldest to free up memory - if (storedChunks.size() > 6) + while (storedChunks.size() > 8) { CoordXZ coord = storedChunks.remove(0); if (!originalChunks.contains(coord)) world.unloadChunkRequest(coord.x, coord.z); - coord = storedChunks.remove(0); - if (!originalChunks.contains(coord)) - world.unloadChunkRequest(coord.x, coord.z); } // move on to next chunk @@ -188,6 +217,9 @@ public class WorldFillTask implements Runnable // step through chunks in spiral pattern from center; returns false if we're done, otherwise returns true public boolean moveToNext() { + if (paused || pausedForMemory) + return false; + reportNum++; // keep track of progress in case we need to save to config for restoring progress after server restart @@ -220,6 +252,10 @@ public class WorldFillTask implements Runnable } } + // keep track of the last chunk we were at + lastChunk.x = x; + lastChunk.z = z; + // move one chunk further in the appropriate direction if (isZLeg) z += (isNeg) ? -1 : 1; @@ -323,22 +359,30 @@ public class WorldFillTask implements Runnable } // let the user know how things are coming along + private int reportCounter = 0; private void reportProgress() { lastReport = Config.Now(); double perc = ((double)(reportTotal + reportNum) / (double)reportTarget) * 100; - sendMessage(reportNum + " more map chunks processed (" + (reportTotal + reportNum) + " total, " + Config.coord.format(perc) + "%" + ")"); + if (perc > 100) perc = 100; + sendMessage(reportNum + " more chunks processed (" + (reportTotal + reportNum) + " total with " + reportSkipped + " skipped, ~" + Config.coord.format(perc) + "%" + ")"); reportTotal += reportNum; reportNum = 0; - // try to keep memory usage in check and keep things speedy as much as possible... - world.save(); + reportCounter++; + // go ahead and save world to disk every 30 seconds or so, just in case; can take a couple of seconds or more, so we don't want to run it too often + if (reportCounter >= 6) + { + reportCounter = 0; + sendMessage("Saving the world to disk, just to be on the safe side."); + world.save(); + } } // send a message to the server console/log and possibly to an in-game player private void sendMessage(String text) { - // Due to the apparent chunk generation memory leak, we need to track memory availability + // Due to chunk generation eating up memory and Java being too slow about GC, we need to track memory availability int availMem = Config.AvailableMemory(); Config.Log("[Fill] " + text + " (free mem: " + availMem + " MB)"); @@ -349,10 +393,13 @@ public class WorldFillTask implements Runnable { // running low on memory, auto-pause pausedForMemory = true; Config.StoreFillTask(); - text = "Available memory is very low, task is pausing. Will automatically continue if/when sufficient memory is freed up.\n Alternately, if you restart the server, this task will automatically continue once the server is back up."; + text = "Available memory is very low, task is pausing. A cleanup will be attempted now, and the task will automatically continue if/when sufficient memory is freed up.\n Alternatively, if you restart the server, this task will automatically continue once the server is back up."; Config.Log("[Fill] " + text); if (notifyPlayer != null) notifyPlayer.sendMessage("[Fill] " + text); + // prod Java with a request to go ahead and do GC to clean unloaded chunks from memory; this seems to work wonders almost immediately + // yes, explicit calls to System.gc() are normally bad, but in this case it otherwise can take a long long long time for Java to recover memory + System.gc(); } } diff --git a/src/com/wimbli/WorldBorder/WorldTrimTask.java b/src/com/wimbli/WorldBorder/WorldTrimTask.java index 3fb6262..67e8424 100644 --- a/src/com/wimbli/WorldBorder/WorldTrimTask.java +++ b/src/com/wimbli/WorldBorder/WorldTrimTask.java @@ -14,8 +14,7 @@ public class WorldTrimTask implements Runnable // general task-related reference data private transient Server server = null; private transient World world = null; - private transient File regionFolder = null; - private transient File[] regionFiles = null; + private transient WorldFileData worldData = null; private transient BorderData border = null; private transient boolean readyToGo = false; private transient boolean paused = false; @@ -67,34 +66,15 @@ public class WorldTrimTask implements Runnable this.border.setRadius(border.getRadius() + trimDistance); - regionFolder = new File(this.world.getWorldFolder(), "region"); - if (!regionFolder.exists() || !regionFolder.isDirectory()) + worldData = WorldFileData.create(world, notifyPlayer); + if (worldData == null) { - String mainRegionFolder = regionFolder.getPath(); - regionFolder = new File(this.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds - if (!regionFolder.exists() || !regionFolder.isDirectory()) - { - String subRegionFolder = regionFolder.getPath(); - regionFolder = new File(this.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is - if (!regionFolder.exists() || !regionFolder.isDirectory()) - { - sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+regionFolder.getPath()); - this.stop(); - return; - } - } - } - regionFiles = regionFolder.listFiles(new RegionFileFilter()); - - if (regionFiles == null || regionFiles.length == 0) - { - sendMessage("Could not find any region files. Looked in: "+regionFolder.getPath()); this.stop(); return; } // each region file covers up to 1024 chunks; with all operations we might need to do, let's figure 3X that - this.reportTarget = regionFiles.length * 3072; + this.reportTarget = worldData.regionFileCount() * 3072; // queue up the first file if (!nextFile()) @@ -147,9 +127,10 @@ public class WorldTrimTask implements Runnable trimChunks = regionChunks; unloadChunks(); reportTrimmedRegions++; - if (!regionFiles[currentRegion].delete()) + File regionFile = worldData.regionFile(currentRegion); + if (!regionFile.delete()) { - sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFile.getName()); wipeChunks(); } nextFile(); @@ -189,7 +170,7 @@ public class WorldTrimTask implements Runnable trimChunks = new ArrayList(1024); // have we already handled all region files? - if (currentRegion >= regionFiles.length) + if (currentRegion >= worldData.regionFileCount()) { // hey, we're done paused = true; readyToGo = false; @@ -199,19 +180,13 @@ public class WorldTrimTask implements Runnable counter += 16; - // get the X and Z coordinates of the current region from the filename - String[] coords = regionFiles[currentRegion].getName().split("\\."); - try - { - regionX = Integer.parseInt(coords[1]); - regionZ = Integer.parseInt(coords[2]); - } - catch(Exception ex) - { - sendMessage("Error! Region file found with abnormal name: "+regionFiles[currentRegion].getName()); + // get the X and Z coordinates of the current region + CoordXZ coord = worldData.regionFileCoordinates(currentRegion); + if (coord == null) return false; - } + regionX = coord.x; + regionZ = coord.z; return true; } @@ -279,12 +254,13 @@ public class WorldTrimTask implements Runnable // by the way, this method was created based on the divulged region file format: http://mojang.com/2011/02/16/minecraft-save-file-format-in-beta-1-3/ private void wipeChunks() { - if (!regionFiles[currentRegion].canWrite()) + File regionFile = worldData.regionFile(currentRegion); + if (!regionFile.canWrite()) { - regionFiles[currentRegion].setWritable(true); - if (!regionFiles[currentRegion].canWrite()) + regionFile.setWritable(true); + if (!regionFile.canWrite()) { - sendMessage("Error! region file is locked and can't be trimmed: "+regionFiles[currentRegion].getName()); + sendMessage("Error! region file is locked and can't be trimmed: "+regionFile.getName()); return; } } @@ -296,7 +272,7 @@ public class WorldTrimTask implements Runnable try { - RandomAccessFile unChunk = new RandomAccessFile(regionFiles[currentRegion], "rwd"); + RandomAccessFile unChunk = new RandomAccessFile(regionFile, "rwd"); for (CoordXZ wipe : trimChunks) { // wipe this extraneous chunk's pointer... note that this method isn't perfect since the actual chunk data is left orphaned, // but Minecraft will overwrite the orphaned data sector if/when another chunk is created in the region, so it's not so bad @@ -309,11 +285,11 @@ public class WorldTrimTask implements Runnable } catch (FileNotFoundException ex) { - sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFile.getName()); } catch (IOException ex) { - sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFile.getName()); } counter += trimChunks.size(); } @@ -387,17 +363,4 @@ public class WorldTrimTask implements Runnable if (notifyPlayer != null) notifyPlayer.sendMessage("[Trim] " + text); } - - // filter for region files - private static class RegionFileFilter implements FileFilter - { - public boolean accept(File file) - { - return ( - file.exists() - && file.isFile() - && file.getName().toLowerCase().endsWith(".mcr") - ); - } - } }