Fill process now checks the world region files directly to determine which chunks have been previously generated and skips over them, so existing worlds where much of the world is already generated should get through the fill process much faster.

Fill process should run much more smoothly regardless of what frequency you've specified.
If the fill process is paused due to low memory, it now tries to prod Java into cleaning up memory, so the process usually is able to start right back up almost immediately.
A few other minor tweaks to the fill process.

Moved world region file handling code out to new separate WorldFileData class, which is used by both the trim process and the fill process.
This commit is contained in:
Brettflan 2012-02-10 20:20:47 -06:00
parent 9ee2d3aa50
commit 91c66d9f58
3 changed files with 341 additions and 72 deletions

View File

@ -0,0 +1,259 @@
package com.wimbli.WorldBorder;
import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.bukkit.entity.Player;
import org.bukkit.World;
// image output stuff, for debugging method at bottom of this file
import java.awt.*;
import java.awt.image.*;
import javax.imageio.*;
public class WorldFileData
{
private transient World world;
private transient File regionFolder = null;
private transient File[] regionFiles = null;
private transient Player notifyPlayer = null;
private transient Map<CoordXZ, List<Boolean>> regionChunkExistence = Collections.synchronizedMap(new HashMap<CoordXZ, List<Boolean>>());
// Use this static method to create a new instance of this class. If null is returned, there was a problem so any process relying on this should be cancelled.
public static WorldFileData create(World world, Player notifyPlayer)
{
WorldFileData newData = new WorldFileData(world, notifyPlayer);
newData.regionFolder = new File(newData.world.getWorldFolder(), "region");
if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory())
{
String mainRegionFolder = newData.regionFolder.getPath();
newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds
if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory())
{
String subRegionFolder = newData.regionFolder.getPath();
newData.regionFolder = new File(newData.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is
if (!newData.regionFolder.exists() || !newData.regionFolder.isDirectory())
{
newData.sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+newData.regionFolder.getPath());
return null;
}
}
}
newData.regionFiles = newData.regionFolder.listFiles(new RegionFileFilter());
if (newData.regionFiles == null || newData.regionFiles.length == 0)
{
newData.sendMessage("Could not find any region files. Looked in: "+newData.regionFolder.getPath());
return null;
}
return newData;
}
// the constructor is private; use create() method above to create an instance of this class.
private WorldFileData(World world, Player notifyPlayer)
{
this.world = world;
this.notifyPlayer = notifyPlayer;
}
// number of region files this world has
public int regionFileCount()
{
return regionFiles.length;
}
// folder where world's region files are located
public File regionFolder()
{
return regionFolder;
}
// return entire list of region files
public File[] regionFiles()
{
return regionFiles.clone();
}
// return a region file by index
public File regionFile(int index)
{
if (regionFiles.length < index)
return null;
return regionFiles[index];
}
// get the X and Z world coordinates of the region from the filename
public CoordXZ regionFileCoordinates(int index)
{
File regionFile = this.regionFile(index);
String[] coords = regionFile.getName().split("\\.");
int x, z;
try
{
x = Integer.parseInt(coords[1]);
z = Integer.parseInt(coords[2]);
return new CoordXZ (x, z);
}
catch(Exception ex)
{
sendMessage("Error! Region file found with abnormal name: "+regionFile.getName());
return null;
}
}
// Find out if the chunk at the given coordinates exists.
public boolean doesChunkExist(int x, int z)
{
CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z));
List<Boolean> regionChunks = this.getRegionData(region);
// Bukkit.getLogger().info("x: "+x+" z: "+z+" offset: "+coordToRegionOffset(x, z));
return regionChunks.get(coordToRegionOffset(x, z)).booleanValue();
}
// Find out if the chunk at the given coordinates has been fully generated.
// Minecraft only fully generates a chunk when adjacent chunks are also loaded.
public boolean isChunkFullyGenerated(int x, int z)
{ // if all adjacent chunks exist, it should be a safe enough bet that this one is fully generated
return
! (
! doesChunkExist(x, z)
|| ! doesChunkExist(x+1, z)
|| ! doesChunkExist(x-1, z)
|| ! doesChunkExist(x, z+1)
|| ! doesChunkExist(x, z-1)
);
}
// Method to let us know a chunk has been generated, to update our region map.
public void chunkExistsNow(int x, int z)
{
CoordXZ region = new CoordXZ(CoordXZ.chunkToRegion(x), CoordXZ.chunkToRegion(z));
List<Boolean> regionChunks = this.getRegionData(region);
regionChunks.set(coordToRegionOffset(x, z), true);
}
// region is 32 X 32 chunks; chunk pointers are stored in region file at position: x + z*32 (max 1024)
// input x and z values can be world-based chunk coordinates or local-to-region chunk coordinates either one
private int coordToRegionOffset(int x, int z)
{
// "%" modulus is used to convert potential world coordinates to definitely be local region coordinates
x = x % 32;
z = z % 32;
// similarly, for local coordinates, we need to wrap negative values around
if (x < 0) x += 32;
if (z < 0) z += 32;
// return offset position for the now definitely local x and z values
return (x + (z * 32));
}
private List<Boolean> getRegionData(CoordXZ region)
{
List<Boolean> data = regionChunkExistence.get(region);
if (data != null)
return data;
// data for the specified region isn't loaded yet, so init it as empty and try to find the file and load the data
data = new ArrayList<Boolean>(1024);
for (int i = 0; i < 1024; i++)
{
data.add(Boolean.FALSE);
}
for (int i = 0; i < regionFiles.length; i++)
{
CoordXZ coord = regionFileCoordinates(i);
// is this region file the one we're looking for?
if ( ! coord.equals(region))
continue;
int counter = 0;
try
{
RandomAccessFile regionData = new RandomAccessFile(this.regionFile(i), "r");
// first 4096 bytes of region file consists of 4-byte int pointers to chunk data in the file
for (int j = 0; j < 1024; j++)
{
// if chunk pointer data is 0, chunk doesn't exist yet; otherwise, it does
if (regionData.readInt() != 0)
data.set(j, true);
counter++;
}
}
catch (FileNotFoundException ex)
{
sendMessage("Error! Could not open region file to find generated chunks: "+this.regionFile(i).getName());
}
catch (IOException ex)
{
sendMessage("Error! Could not read region file to find generated chunks: "+this.regionFile(i).getName());
}
}
regionChunkExistence.put(region, data);
// testImage(region, data);
return data;
}
// send a message to the server console/log and possibly to an in-game player
private void sendMessage(String text)
{
Config.Log("[WorldData] " + text);
if (notifyPlayer != null && notifyPlayer.isOnline())
notifyPlayer.sendMessage("[WorldData] " + text);
}
// filter for region files
private static class RegionFileFilter implements FileFilter
{
@Override
public boolean accept(File file)
{
return (
file.exists()
&& file.isFile()
&& file.getName().toLowerCase().endsWith(".mcr")
);
}
}
// crude chunk map PNG image output, for debugging
private void testImage(CoordXZ region, List<Boolean> data) {
int width = 32;
int height = 32;
BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
Graphics2D g2 = bi.createGraphics();
int current = 0;
g2.setColor(Color.BLACK);
for (int x = 0; x < 32; x++)
{
for (int z = 0; z < 32; z++)
{
if (data.get(current).booleanValue())
g2.fillRect(x,z, x+1, z+1);
current++;
}
}
File f = new File("region_"+region.x+"_"+region.z+"_.png");
Config.Log(f.getAbsolutePath());
try {
// png is an image format (like gif or jpg)
ImageIO.write(bi, "png", f);
} catch (IOException ex) {
Config.Log("[SEVERE]"+ex.getLocalizedMessage());
}
}
}

View File

@ -17,6 +17,7 @@ public class WorldFillTask implements Runnable
private transient Server server = null;
private transient World world = null;
private transient BorderData border = null;
private transient WorldFileData worldData = null;
private transient boolean readyToGo = false;
private transient boolean paused = false;
private transient boolean pausedForMemory = false;
@ -43,9 +44,11 @@ public class WorldFillTask implements Runnable
private transient boolean insideBorder = true;
private List<CoordXZ> storedChunks = new LinkedList<CoordXZ>();
private Set<CoordXZ> originalChunks = new HashSet<CoordXZ>();
private transient CoordXZ lastChunk = new CoordXZ(0, 0);
// for reporting progress back to user occasionally
private transient long lastReport = Config.Now();
private transient int reportSkipped = 0;
private transient int reportTarget = 0;
private transient int reportTotal = 0;
private transient int reportNum = 0;
@ -78,6 +81,14 @@ public class WorldFillTask implements Runnable
return;
}
// load up a new WorldFileData for the world in question, used to scan region files for which chunks are already fully generated and such
worldData = WorldFileData.create(world, notifyPlayer);
if (worldData == null)
{
this.stop();
return;
}
this.border.setRadius(border.getRadius() + fillDistance);
this.x = CoordXZ.blockToChunk((int)border.getX());
this.z = CoordXZ.blockToChunk((int)border.getZ());
@ -117,6 +128,7 @@ public class WorldFillTask implements Runnable
return;
pausedForMemory = false;
readyToGo = true;
sendMessage("Available memory is sufficient, automatically continuing.");
}
@ -125,6 +137,8 @@ public class WorldFillTask implements Runnable
// this is set so it only does one iteration at a time, no matter how frequently the timer fires
readyToGo = false;
// and this is tracked to keep one iteration from dragging on too long and possibly choking the system if the user specified a really high frequency
long loopStartTime = Config.Now();
for (int loop = 0; loop < chunksPerRun; loop++)
{
@ -132,10 +146,19 @@ public class WorldFillTask implements Runnable
if (paused || pausedForMemory)
return;
long now = Config.Now();
// every 5 seconds or so, give basic progress report to let user know how it's going
if (Config.Now() > lastReport + 5000)
if (now > lastReport + 5000)
reportProgress();
// if this iteration has been running for 250ms (~5 ticks) or more, stop to take a breather
if (now > loopStartTime + 250)
{
readyToGo = true;
return;
}
// if we've made it at least partly outside the border, skip past any such chunks
while (!border.insideBorder(CoordXZ.chunkToBlock(x) + 8, CoordXZ.chunkToBlock(z) + 8))
{
@ -144,16 +167,18 @@ public class WorldFillTask implements Runnable
}
insideBorder = true;
// skip past any chunks which are currently loaded (they're definitely already generated)
while (world.isChunkLoaded(x, z))
// skip past any chunks which are confirmed as fully generated using our super-special isChunkFullyGenerated routine
while (worldData.isChunkFullyGenerated(x, z))
{
reportSkipped++;
insideBorder = true;
if (!moveToNext())
return;
}
// load the target chunk and generate it if necessary (no way to check if chunk has been generated first, simply have to load it)
// load the target chunk and generate it if necessary
world.loadChunk(x, z, true);
worldData.chunkExistsNow(x, z);
// There need to be enough nearby chunks loaded to make the server populate a chunk with trees, snow, etc.
// So, we keep the last few chunks loaded, and need to also temporarily load an extra inside chunk (neighbor closest to center of map)
@ -161,19 +186,23 @@ public class WorldFillTask implements Runnable
int popZ = isZLeg ? z : (z + (!isNeg ? -1 : 1));
world.loadChunk(popX, popZ, false);
// make sure the previous chunk in our spiral is loaded as well (might have already existed and been skipped over)
if (!storedChunks.contains(lastChunk) && !originalChunks.contains(lastChunk))
{
world.loadChunk(lastChunk.x, lastChunk.z, false);
storedChunks.add(new CoordXZ(lastChunk.x, lastChunk.z));
}
// Store the coordinates of these latest 2 chunks we just loaded, so we can unload them after a bit...
storedChunks.add(new CoordXZ(x, z));
storedChunks.add(new CoordXZ(popX, popZ));
storedChunks.add(new CoordXZ(x, z));
// If enough stored chunks are buffered in, go ahead and unload the oldest to free up memory
if (storedChunks.size() > 6)
while (storedChunks.size() > 8)
{
CoordXZ coord = storedChunks.remove(0);
if (!originalChunks.contains(coord))
world.unloadChunkRequest(coord.x, coord.z);
coord = storedChunks.remove(0);
if (!originalChunks.contains(coord))
world.unloadChunkRequest(coord.x, coord.z);
}
// move on to next chunk
@ -188,6 +217,9 @@ public class WorldFillTask implements Runnable
// step through chunks in spiral pattern from center; returns false if we're done, otherwise returns true
public boolean moveToNext()
{
if (paused || pausedForMemory)
return false;
reportNum++;
// keep track of progress in case we need to save to config for restoring progress after server restart
@ -220,6 +252,10 @@ public class WorldFillTask implements Runnable
}
}
// keep track of the last chunk we were at
lastChunk.x = x;
lastChunk.z = z;
// move one chunk further in the appropriate direction
if (isZLeg)
z += (isNeg) ? -1 : 1;
@ -323,22 +359,30 @@ public class WorldFillTask implements Runnable
}
// let the user know how things are coming along
private int reportCounter = 0;
private void reportProgress()
{
lastReport = Config.Now();
double perc = ((double)(reportTotal + reportNum) / (double)reportTarget) * 100;
sendMessage(reportNum + " more map chunks processed (" + (reportTotal + reportNum) + " total, " + Config.coord.format(perc) + "%" + ")");
if (perc > 100) perc = 100;
sendMessage(reportNum + " more chunks processed (" + (reportTotal + reportNum) + " total with " + reportSkipped + " skipped, ~" + Config.coord.format(perc) + "%" + ")");
reportTotal += reportNum;
reportNum = 0;
// try to keep memory usage in check and keep things speedy as much as possible...
world.save();
reportCounter++;
// go ahead and save world to disk every 30 seconds or so, just in case; can take a couple of seconds or more, so we don't want to run it too often
if (reportCounter >= 6)
{
reportCounter = 0;
sendMessage("Saving the world to disk, just to be on the safe side.");
world.save();
}
}
// send a message to the server console/log and possibly to an in-game player
private void sendMessage(String text)
{
// Due to the apparent chunk generation memory leak, we need to track memory availability
// Due to chunk generation eating up memory and Java being too slow about GC, we need to track memory availability
int availMem = Config.AvailableMemory();
Config.Log("[Fill] " + text + " (free mem: " + availMem + " MB)");
@ -349,10 +393,13 @@ public class WorldFillTask implements Runnable
{ // running low on memory, auto-pause
pausedForMemory = true;
Config.StoreFillTask();
text = "Available memory is very low, task is pausing. Will automatically continue if/when sufficient memory is freed up.\n Alternately, if you restart the server, this task will automatically continue once the server is back up.";
text = "Available memory is very low, task is pausing. A cleanup will be attempted now, and the task will automatically continue if/when sufficient memory is freed up.\n Alternatively, if you restart the server, this task will automatically continue once the server is back up.";
Config.Log("[Fill] " + text);
if (notifyPlayer != null)
notifyPlayer.sendMessage("[Fill] " + text);
// prod Java with a request to go ahead and do GC to clean unloaded chunks from memory; this seems to work wonders almost immediately
// yes, explicit calls to System.gc() are normally bad, but in this case it otherwise can take a long long long time for Java to recover memory
System.gc();
}
}

View File

@ -14,8 +14,7 @@ public class WorldTrimTask implements Runnable
// general task-related reference data
private transient Server server = null;
private transient World world = null;
private transient File regionFolder = null;
private transient File[] regionFiles = null;
private transient WorldFileData worldData = null;
private transient BorderData border = null;
private transient boolean readyToGo = false;
private transient boolean paused = false;
@ -67,34 +66,15 @@ public class WorldTrimTask implements Runnable
this.border.setRadius(border.getRadius() + trimDistance);
regionFolder = new File(this.world.getWorldFolder(), "region");
if (!regionFolder.exists() || !regionFolder.isDirectory())
worldData = WorldFileData.create(world, notifyPlayer);
if (worldData == null)
{
String mainRegionFolder = regionFolder.getPath();
regionFolder = new File(this.world.getWorldFolder(), "DIM-1"+File.separator+"region"); // nether worlds
if (!regionFolder.exists() || !regionFolder.isDirectory())
{
String subRegionFolder = regionFolder.getPath();
regionFolder = new File(this.world.getWorldFolder(), "DIM1"+File.separator+"region"); // "the end" worlds; not sure why "DIM1" vs "DIM-1", but that's how it is
if (!regionFolder.exists() || !regionFolder.isDirectory())
{
sendMessage("Could not validate folder for world's region files. Looked in: "+mainRegionFolder+" -and- "+subRegionFolder+" -and- "+regionFolder.getPath());
this.stop();
return;
}
}
}
regionFiles = regionFolder.listFiles(new RegionFileFilter());
if (regionFiles == null || regionFiles.length == 0)
{
sendMessage("Could not find any region files. Looked in: "+regionFolder.getPath());
this.stop();
return;
}
// each region file covers up to 1024 chunks; with all operations we might need to do, let's figure 3X that
this.reportTarget = regionFiles.length * 3072;
this.reportTarget = worldData.regionFileCount() * 3072;
// queue up the first file
if (!nextFile())
@ -147,9 +127,10 @@ public class WorldTrimTask implements Runnable
trimChunks = regionChunks;
unloadChunks();
reportTrimmedRegions++;
if (!regionFiles[currentRegion].delete())
File regionFile = worldData.regionFile(currentRegion);
if (!regionFile.delete())
{
sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFiles[currentRegion].getName());
sendMessage("Error! Region file which is outside the border could not be deleted: "+regionFile.getName());
wipeChunks();
}
nextFile();
@ -189,7 +170,7 @@ public class WorldTrimTask implements Runnable
trimChunks = new ArrayList<CoordXZ>(1024);
// have we already handled all region files?
if (currentRegion >= regionFiles.length)
if (currentRegion >= worldData.regionFileCount())
{ // hey, we're done
paused = true;
readyToGo = false;
@ -199,19 +180,13 @@ public class WorldTrimTask implements Runnable
counter += 16;
// get the X and Z coordinates of the current region from the filename
String[] coords = regionFiles[currentRegion].getName().split("\\.");
try
{
regionX = Integer.parseInt(coords[1]);
regionZ = Integer.parseInt(coords[2]);
}
catch(Exception ex)
{
sendMessage("Error! Region file found with abnormal name: "+regionFiles[currentRegion].getName());
// get the X and Z coordinates of the current region
CoordXZ coord = worldData.regionFileCoordinates(currentRegion);
if (coord == null)
return false;
}
regionX = coord.x;
regionZ = coord.z;
return true;
}
@ -279,12 +254,13 @@ public class WorldTrimTask implements Runnable
// by the way, this method was created based on the divulged region file format: http://mojang.com/2011/02/16/minecraft-save-file-format-in-beta-1-3/
private void wipeChunks()
{
if (!regionFiles[currentRegion].canWrite())
File regionFile = worldData.regionFile(currentRegion);
if (!regionFile.canWrite())
{
regionFiles[currentRegion].setWritable(true);
if (!regionFiles[currentRegion].canWrite())
regionFile.setWritable(true);
if (!regionFile.canWrite())
{
sendMessage("Error! region file is locked and can't be trimmed: "+regionFiles[currentRegion].getName());
sendMessage("Error! region file is locked and can't be trimmed: "+regionFile.getName());
return;
}
}
@ -296,7 +272,7 @@ public class WorldTrimTask implements Runnable
try
{
RandomAccessFile unChunk = new RandomAccessFile(regionFiles[currentRegion], "rwd");
RandomAccessFile unChunk = new RandomAccessFile(regionFile, "rwd");
for (CoordXZ wipe : trimChunks)
{ // wipe this extraneous chunk's pointer... note that this method isn't perfect since the actual chunk data is left orphaned,
// but Minecraft will overwrite the orphaned data sector if/when another chunk is created in the region, so it's not so bad
@ -309,11 +285,11 @@ public class WorldTrimTask implements Runnable
}
catch (FileNotFoundException ex)
{
sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFiles[currentRegion].getName());
sendMessage("Error! Could not open region file to wipe individual chunks: "+regionFile.getName());
}
catch (IOException ex)
{
sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFiles[currentRegion].getName());
sendMessage("Error! Could not modify region file to wipe individual chunks: "+regionFile.getName());
}
counter += trimChunks.size();
}
@ -387,17 +363,4 @@ public class WorldTrimTask implements Runnable
if (notifyPlayer != null)
notifyPlayer.sendMessage("[Trim] " + text);
}
// filter for region files
private static class RegionFileFilter implements FileFilter
{
public boolean accept(File file)
{
return (
file.exists()
&& file.isFile()
&& file.getName().toLowerCase().endsWith(".mcr")
);
}
}
}