diff --git a/src/com/wimbli/WorldBorder/WorldFileData.java b/src/com/wimbli/WorldBorder/WorldFileData.java new file mode 100644 index 0000000..b25d59e --- /dev/null +++ b/src/com/wimbli/WorldBorder/WorldFileData.java @@ -0,0 +1,259 @@ +package com.wimbli.WorldBorder; + +import java.io.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.bukkit.entity.Player; +import org.bukkit.World; + +// image output stuff, for debugging method at bottom of this file +import java.awt.*; +import java.awt.image.*; +import javax.imageio.*; + + +public class WorldFileData +{ + private transient World world; + private transient File regionFolder = null; + private transient File[] regionFiles = null; + private transient Player notifyPlayer = null; + private transient Map> regionChunkExistence = Collections.synchronizedMap(new HashMap>()); + + // Use this static method to create a new instance of this class. If null is returned, there was a problem so any process relying on this should be cancelled. + public static WorldFileData create(World world, Player notifyPlayer) + { + WorldFileData newData = new WorldFileData(world, notifyPlayer); + + newData.regionFolder = new File(newData.world.getWorldFolder(), "region"); + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + String mainRegionFolder = newData.regionFolder.getPath(); + newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + String subRegionFolder = newData.regionFolder.getPath(); + newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is + if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory()) + { + newData.sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+newData.regionFolder.getPath()); + return null; + } + } + } + newData.regionFiles = newData.regionFolder.listFiles(new RegionFileFilter()); + + if (newData.regionFiles == null || newData.regionFiles.length == 0) + { + newData.sendMessage("Could not find any region files. Looked in: "+newData.regionFolder.getPath()); + return null; + } + + return newData; + } + + // the constructor is private; use create() method above to create an instance of this class. + private WorldFileData(World world, Player notifyPlayer) + { + this.world = world; + this.notifyPlayer = notifyPlayer; + } + + + // number of region files this world has + public int regionFileCount() + { + return regionFiles.length; + } + + // folder where world's region files are located + public File regionFolder() + { + return regionFolder; + } + + // return entire list of region files + public File[] regionFiles() + { + return regionFiles.clone(); + } + + // return a region file by index + public File regionFile(int index) + { + if (regionFiles.length < index) + return null; + return regionFiles[index]; + } + + // get the X and Z world coordinates of the region from the filename + public CoordXZ regionFileCoordinates(int index) + { + File regionFile = this.regionFile(index); + String[] coords = regionFile.getName().split("\\."); + int x, z; + try + { + x = Integer.parseInt(coords[1]); + z = Integer.parseInt(coords[2]); + return new CoordXZ (x, z); + } + catch(Exception ex) + { + sendMessage("Error! Region file found with abnormal name: "+regionFile.getName()); + return null; + } + } + + + // Find out if the chunk at the given coordinates exists. + public boolean doesChunkExist(int x, int z) + { + CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z)); + List regionChunks = this.getRegionData(region); +// Bukkit.getLogger().info("x: "+x+" z: "+z+" offset: "+coordToRegionOffset(x, z)); + return regionChunks.get(coordToRegionOffset(x, z)).booleanValue(); + } + + // Find out if the chunk at the given coordinates has been fully generated. + // Minecraft only fully generates a chunk when adjacent chunks are also loaded. + public boolean isChunkFullyGenerated(int x, int z) + { // if all adjacent chunks exist, it should be a safe enough bet that this one is fully generated + return + ! ( + ! doesChunkExist(x, z) + || ! doesChunkExist(x+1, z) + || ! doesChunkExist(x-1, z) + || ! doesChunkExist(x, z+1) + || ! doesChunkExist(x, z-1) + ); + } + + // Method to let us know a chunk has been generated, to update our region map. + public void chunkExistsNow(int x, int z) + { + CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z)); + List regionChunks = this.getRegionData(region); + regionChunks.set(coordToRegionOffset(x, z), true); + } + + + + // region is 32 X 32 chunks; chunk pointers are stored in region file at position: x + z*32 (max 1024) + // input x and z values can be world-based chunk coordinates or local-to-region chunk coordinates either one + private int coordToRegionOffset(int x, int z) + { + // "%" modulus is used to convert potential world coordinates to definitely be local region coordinates + x = x % 32; + z = z % 32; + // similarly, for local coordinates, we need to wrap negative values around + if (x < 0) x += 32; + if (z < 0) z += 32; + // return offset position for the now definitely local x and z values + return (x + (z * 32)); + } + + private List getRegionData(CoordXZ region) + { + List data = regionChunkExistence.get(region); + if (data != null) + return data; + + // data for the specified region isn't loaded yet, so init it as empty and try to find the file and load the data + data = new ArrayList(1024); + for (int i = 0; i < 1024; i++) + { + data.add(Boolean.FALSE); + } + + for (int i = 0; i < regionFiles.length; i++) + { + CoordXZ coord = regionFileCoordinates(i); + // is this region file the one we're looking for? + if ( ! coord.equals(region)) + continue; + + int counter = 0; + try + { + RandomAccessFile regionData = new RandomAccessFile(this.regionFile(i), "r"); + // first 4096 bytes of region file consists of 4-byte int pointers to chunk data in the file + for (int j = 0; j < 1024; j++) + { + // if chunk pointer data is 0, chunk doesn't exist yet; otherwise, it does + if (regionData.readInt() != 0) + data.set(j, true); + counter++; + } + } + catch (FileNotFoundException ex) + { + sendMessage("Error! Could not open region file to find generated chunks: "+this.regionFile(i).getName()); + } + catch (IOException ex) + { + sendMessage("Error! Could not read region file to find generated chunks: "+this.regionFile(i).getName()); + } + } + regionChunkExistence.put(region, data); +// testImage(region, data); + return data; + } + + // send a message to the server console/log and possibly to an in-game player + private void sendMessage(String text) + { + Config.Log("[WorldData] " + text); + if (notifyPlayer != null && notifyPlayer.isOnline()) + notifyPlayer.sendMessage("[WorldData] " + text); + } + + // filter for region files + private static class RegionFileFilter implements FileFilter + { + @Override + public boolean accept(File file) + { + return ( + file.exists() + && file.isFile() + && file.getName().toLowerCase().endsWith(".mcr") + ); + } + } + + + +// crude chunk map PNG image output, for debugging + private void testImage(CoordXZ region, List data) { + int width = 32; + int height = 32; + BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + Graphics2D g2 = bi.createGraphics(); + int current = 0; + g2.setColor(Color.BLACK); + + for (int x = 0; x < 32; x++) + { + for (int z = 0; z < 32; z++) + { + if (data.get(current).booleanValue()) + g2.fillRect(x,z, x+1, z+1); + current++; + } + } + + File f = new File("region_"+region.x+"_"+region.z+"_.png"); + Config.Log(f.getAbsolutePath()); + try { + // png is an image format (like gif or jpg) + ImageIO.write(bi, "png", f); + } catch (IOException ex) { + Config.Log("[SEVERE]"+ex.getLocalizedMessage()); + } + } +} diff --git a/src/com/wimbli/WorldBorder/WorldFillTask.java b/src/com/wimbli/WorldBorder/WorldFillTask.java index c33631a..6e383eb 100644 --- a/src/com/wimbli/WorldBorder/WorldFillTask.java +++ b/src/com/wimbli/WorldBorder/WorldFillTask.java @@ -17,6 +17,7 @@ public class WorldFillTask implements Runnable private transient Server server = null; private transient World world = null; private transient BorderData border = null; + private transient WorldFileData worldData = null; private transient boolean readyToGo = false; private transient boolean paused = false; private transient boolean pausedForMemory = false; @@ -43,9 +44,11 @@ public class WorldFillTask implements Runnable private transient boolean insideBorder = true; private List storedChunks = new LinkedList(); private Set originalChunks = new HashSet(); + private transient CoordXZ lastChunk = new CoordXZ(0, 0); // for reporting progress back to user occasionally private transient long lastReport = Config.Now(); + private transient int reportSkipped = 0; private transient int reportTarget = 0; private transient int reportTotal = 0; private transient int reportNum = 0; @@ -78,6 +81,14 @@ public class WorldFillTask implements Runnable return; } + // load up a new WorldFileData for the world in question, used to scan region files for which chunks are already fully generated and such + worldData = WorldFileData.create(world, notifyPlayer); + if (worldData == null) + { + this.stop(); + return; + } + this.border.setRadius(border.getRadius() + fillDistance); this.x = CoordXZ.blockToChunk((int)border.getX()); this.z = CoordXZ.blockToChunk((int)border.getZ()); @@ -117,6 +128,7 @@ public class WorldFillTask implements Runnable return; pausedForMemory = false; + readyToGo = true; sendMessage("Available memory is sufficient, automatically continuing."); } @@ -125,6 +137,8 @@ public class WorldFillTask implements Runnable // this is set so it only does one iteration at a time, no matter how frequently the timer fires readyToGo = false; + // and this is tracked to keep one iteration from dragging on too long and possibly choking the system if the user specified a really high frequency + long loopStartTime = Config.Now(); for (int loop = 0; loop < chunksPerRun; loop++) { @@ -132,10 +146,19 @@ public class WorldFillTask implements Runnable if (paused || pausedForMemory) return; + long now = Config.Now(); + // every 5 seconds or so, give basic progress report to let user know how it's going - if (Config.Now() > lastReport + 5000) + if (now > lastReport + 5000) reportProgress(); + // if this iteration has been running for 250ms (~5 ticks) or more, stop to take a breather + if (now > loopStartTime + 250) + { + readyToGo = true; + return; + } + // if we've made it at least partly outside the border, skip past any such chunks while (!border.insideBorder(CoordXZ.chunkToBlock(x) + 8, CoordXZ.chunkToBlock(z) + 8)) { @@ -144,16 +167,18 @@ public class WorldFillTask implements Runnable } insideBorder = true; - // skip past any chunks which are currently loaded (they're definitely already generated) - while (world.isChunkLoaded(x, z)) + // skip past any chunks which are confirmed as fully generated using our super-special isChunkFullyGenerated routine + while (worldData.isChunkFullyGenerated(x, z)) { + reportSkipped++; insideBorder = true; if (!moveToNext()) return; } - // load the target chunk and generate it if necessary (no way to check if chunk has been generated first, simply have to load it) + // load the target chunk and generate it if necessary world.loadChunk(x, z, true); + worldData.chunkExistsNow(x, z); // There need to be enough nearby chunks loaded to make the server populate a chunk with trees, snow, etc. // So, we keep the last few chunks loaded, and need to also temporarily load an extra inside chunk (neighbor closest to center of map) @@ -161,19 +186,23 @@ public class WorldFillTask implements Runnable int popZ = isZLeg ? z : (z + (!isNeg ? -1 : 1)); world.loadChunk(popX, popZ, false); + // make sure the previous chunk in our spiral is loaded as well (might have already existed and been skipped over) + if (!storedChunks.contains(lastChunk) && !originalChunks.contains(lastChunk)) + { + world.loadChunk(lastChunk.x, lastChunk.z, false); + storedChunks.add(new CoordXZ(lastChunk.x, lastChunk.z)); + } + // Store the coordinates of these latest 2 chunks we just loaded, so we can unload them after a bit... - storedChunks.add(new CoordXZ(x, z)); storedChunks.add(new CoordXZ(popX, popZ)); + storedChunks.add(new CoordXZ(x, z)); // If enough stored chunks are buffered in, go ahead and unload the oldest to free up memory - if (storedChunks.size() > 6) + while (storedChunks.size() > 8) { CoordXZ coord = storedChunks.remove(0); if (!originalChunks.contains(coord)) world.unloadChunkRequest(coord.x, coord.z); - coord = storedChunks.remove(0); - if (!originalChunks.contains(coord)) - world.unloadChunkRequest(coord.x, coord.z); } // move on to next chunk @@ -188,6 +217,9 @@ public class WorldFillTask implements Runnable // step through chunks in spiral pattern from center; returns false if we're done, otherwise returns true public boolean moveToNext() { + if (paused || pausedForMemory) + return false; + reportNum++; // keep track of progress in case we need to save to config for restoring progress after server restart @@ -220,6 +252,10 @@ public class WorldFillTask implements Runnable } } + // keep track of the last chunk we were at + lastChunk.x = x; + lastChunk.z = z; + // move one chunk further in the appropriate direction if (isZLeg) z += (isNeg) ? -1 : 1; @@ -323,22 +359,30 @@ public class WorldFillTask implements Runnable } // let the user know how things are coming along + private int reportCounter = 0; private void reportProgress() { lastReport = Config.Now(); double perc = ((double)(reportTotal + reportNum) / (double)reportTarget) * 100; - sendMessage(reportNum + " more map chunks processed (" + (reportTotal + reportNum) + " total, " + Config.coord.format(perc) + "%" + ")"); + if (perc > 100) perc = 100; + sendMessage(reportNum + " more chunks processed (" + (reportTotal + reportNum) + " total with " + reportSkipped + " skipped, ~" + Config.coord.format(perc) + "%" + ")"); reportTotal += reportNum; reportNum = 0; - // try to keep memory usage in check and keep things speedy as much as possible... - world.save(); + reportCounter++; + // go ahead and save world to disk every 30 seconds or so, just in case; can take a couple of seconds or more, so we don't want to run it too often + if (reportCounter >= 6) + { + reportCounter = 0; + sendMessage("Saving the world to disk, just to be on the safe side."); + world.save(); + } } // send a message to the server console/log and possibly to an in-game player private void sendMessage(String text) { - // Due to the apparent chunk generation memory leak, we need to track memory availability + // Due to chunk generation eating up memory and Java being too slow about GC, we need to track memory availability int availMem = Config.AvailableMemory(); Config.Log("[Fill] " + text + " (free mem: " + availMem + " MB)"); @@ -349,10 +393,13 @@ public class WorldFillTask implements Runnable { // running low on memory, auto-pause pausedForMemory = true; Config.StoreFillTask(); - text = "Available memory is very low, task is pausing. Will automatically continue if/when sufficient memory is freed up.\n Alternately, if you restart the server, this task will automatically continue once the server is back up."; + text = "Available memory is very low, task is pausing. A cleanup will be attempted now, and the task will automatically continue if/when sufficient memory is freed up.\n Alternatively, if you restart the server, this task will automatically continue once the server is back up."; Config.Log("[Fill] " + text); if (notifyPlayer != null) notifyPlayer.sendMessage("[Fill] " + text); + // prod Java with a request to go ahead and do GC to clean unloaded chunks from memory; this seems to work wonders almost immediately + // yes, explicit calls to System.gc() are normally bad, but in this case it otherwise can take a long long long time for Java to recover memory + System.gc(); } } diff --git a/src/com/wimbli/WorldBorder/WorldTrimTask.java b/src/com/wimbli/WorldBorder/WorldTrimTask.java index 3fb6262..67e8424 100644 --- a/src/com/wimbli/WorldBorder/WorldTrimTask.java +++ b/src/com/wimbli/WorldBorder/WorldTrimTask.java @@ -14,8 +14,7 @@ public class WorldTrimTask implements Runnable // general task-related reference data private transient Server server = null; private transient World world = null; - private transient File regionFolder = null; - private transient File[] regionFiles = null; + private transient WorldFileData worldData = null; private transient BorderData border = null; private transient boolean readyToGo = false; private transient boolean paused = false; @@ -67,34 +66,15 @@ public class WorldTrimTask implements Runnable this.border.setRadius(border.getRadius() + trimDistance); - regionFolder = new File(this.world.getWorldFolder(), "region"); - if (!regionFolder.exists() || !regionFolder.isDirectory()) + worldData = WorldFileData.create(world, notifyPlayer); + if (worldData == null) { - String mainRegionFolder = regionFolder.getPath(); - regionFolder = new File(this.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds - if (!regionFolder.exists() || !regionFolder.isDirectory()) - { - String subRegionFolder = regionFolder.getPath(); - regionFolder = new File(this.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is - if (!regionFolder.exists() || !regionFolder.isDirectory()) - { - sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+regionFolder.getPath()); - this.stop(); - return; - } - } - } - regionFiles = regionFolder.listFiles(new RegionFileFilter()); - - if (regionFiles == null || regionFiles.length == 0) - { - sendMessage("Could not find any region files. Looked in: "+regionFolder.getPath()); this.stop(); return; } // each region file covers up to 1024 chunks; with all operations we might need to do, let's figure 3X that - this.reportTarget = regionFiles.length * 3072; + this.reportTarget = worldData.regionFileCount() * 3072; // queue up the first file if (!nextFile()) @@ -147,9 +127,10 @@ public class WorldTrimTask implements Runnable trimChunks = regionChunks; unloadChunks(); reportTrimmedRegions++; - if (!regionFiles[currentRegion].delete()) + File regionFile = worldData.regionFile(currentRegion); + if (!regionFile.delete()) { - sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFile.getName()); wipeChunks(); } nextFile(); @@ -189,7 +170,7 @@ public class WorldTrimTask implements Runnable trimChunks = new ArrayList(1024); // have we already handled all region files? - if (currentRegion >= regionFiles.length) + if (currentRegion >= worldData.regionFileCount()) { // hey, we're done paused = true; readyToGo = false; @@ -199,19 +180,13 @@ public class WorldTrimTask implements Runnable counter += 16; - // get the X and Z coordinates of the current region from the filename - String[] coords = regionFiles[currentRegion].getName().split("\\."); - try - { - regionX = Integer.parseInt(coords[1]); - regionZ = Integer.parseInt(coords[2]); - } - catch(Exception ex) - { - sendMessage("Error! Region file found with abnormal name: "+regionFiles[currentRegion].getName()); + // get the X and Z coordinates of the current region + CoordXZ coord = worldData.regionFileCoordinates(currentRegion); + if (coord == null) return false; - } + regionX = coord.x; + regionZ = coord.z; return true; } @@ -279,12 +254,13 @@ public class WorldTrimTask implements Runnable // by the way, this method was created based on the divulged region file format: http://mojang.com/2011/02/16/minecraft-save-file-format-in-beta-1-3/ private void wipeChunks() { - if (!regionFiles[currentRegion].canWrite()) + File regionFile = worldData.regionFile(currentRegion); + if (!regionFile.canWrite()) { - regionFiles[currentRegion].setWritable(true); - if (!regionFiles[currentRegion].canWrite()) + regionFile.setWritable(true); + if (!regionFile.canWrite()) { - sendMessage("Error! region file is locked and can't be trimmed: "+regionFiles[currentRegion].getName()); + sendMessage("Error! region file is locked and can't be trimmed: "+regionFile.getName()); return; } } @@ -296,7 +272,7 @@ public class WorldTrimTask implements Runnable try { - RandomAccessFile unChunk = new RandomAccessFile(regionFiles[currentRegion], "rwd"); + RandomAccessFile unChunk = new RandomAccessFile(regionFile, "rwd"); for (CoordXZ wipe : trimChunks) { // wipe this extraneous chunk's pointer... note that this method isn't perfect since the actual chunk data is left orphaned, // but Minecraft will overwrite the orphaned data sector if/when another chunk is created in the region, so it's not so bad @@ -309,11 +285,11 @@ public class WorldTrimTask implements Runnable } catch (FileNotFoundException ex) { - sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFile.getName()); } catch (IOException ex) { - sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFiles[currentRegion].getName()); + sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFile.getName()); } counter += trimChunks.size(); } @@ -387,17 +363,4 @@ public class WorldTrimTask implements Runnable if (notifyPlayer != null) notifyPlayer.sendMessage("[Trim] " + text); } - - // filter for region files - private static class RegionFileFilter implements FileFilter - { - public boolean accept(File file) - { - return ( - file.exists() - && file.isFile() - && file.getName().toLowerCase().endsWith(".mcr") - ); - } - } }