Upstream
/
dynmap
mirror of https://github.com/webbukkit/dynmap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4011 from stormboomer/Stormboomer-Performance-MySQL-Storage 

Drasticly improve zoom tile calculation for larger maps when using MySQL/MariaDB storage Engine
This commit is contained in:
mikeprimm 2023-08-29 18:32:58 -05:00 committed by GitHub
parent 31d1b400f0 0120c135c4
commit ebb9dc00d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 123 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java
View File

@ -809,29 +809,21 @@ public class MySQLMapStorage extends MapStorage {
}
try {
c = getConnection();
boolean done = false;
int limit = 100;
int offset = 0;
while (!done) {
// Query tiles for given mapkey
Statement stmt = c.createStatement();
ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d LIMIT %d OFFSET %d;", tableTiles, mapkey, limit, offset));
int cnt = 0;
while (rs.next()) {
StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var);
final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format"));
if(cb != null)
cb.tileFound(st, encoding);
if(cbBase != null && st.zoom == 0)
cbBase.tileFound(st, encoding);
st.cleanup();
cnt++;
}
rs.close();
stmt.close();
if (cnt < limit) done = true;
offset += cnt;
Statement stmt = c.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY, //we want to stream our resultset one row at a time, we are not interessted in going back
java.sql.ResultSet.CONCUR_READ_ONLY); //since we do not handle the entire resultset in memory -> tell the statement that we are going to work read only
stmt.setFetchSize(100); //we can change the jdbc "retrieval chunk size". Basicly we limit how much rows are kept in memory. Bigger value = less network calls to DB, but more memory consumption
ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d;", tableTiles, mapkey)); //we do the query, but do not set any limit / offset. Since data is not kept in memory, just streamed from DB this should not be a problem, only the rows from setFetchSize are kept in memory.
while (rs.next()) {
StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var);
final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format"));
if(cb != null)
cb.tileFound(st, encoding);
if(cbBase != null && st.zoom == 0)
cbBase.tileFound(st, encoding);
st.cleanup();
}
rs.close();
stmt.close();
if(cbEnd != null)
cbEnd.searchEnded();
} catch (SQLException x) {

158
DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java
View File

@ -107,68 +107,126 @@ public class ImageIOManager {
}
public static BufferOutputStream imageIOEncode(BufferedImage img, ImageFormat fmt) {
BufferOutputStream bos = new BufferOutputStream();
if(isRequiredJDKVersion(17,-1,-1)){
return imageIOEncodeUnsafe(img, fmt); //we can skip Thread safety for more performance
}
synchronized(imageioLock) {
try {
ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
fmt = validateFormat(fmt);
if(fmt.getEncoding() == ImageEncoding.JPG) {
WritableRaster raster = img.getRaster();
WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(),
img.getHeight(), 0, 0, new int[] {0, 1, 2});
DirectColorModel cm = (DirectColorModel)img.getColorModel();
DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(),
cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask());
// now create the new buffer that is used ot write the image:
BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null);
return imageIOEncodeUnsafe(img, fmt);
}
}
private static BufferOutputStream imageIOEncodeUnsafe(BufferedImage img, ImageFormat fmt) {
BufferOutputStream bos = new BufferOutputStream();
try {
ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
// Find a jpeg writer
ImageWriter writer = null;
Iterator<ImageWriter> iter = ImageIO.getImageWritersByFormatName("jpg");
if (iter.hasNext()) {
writer = iter.next();
}
if(writer == null) {
Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding");
return null;
}
ImageWriteParam iwp = writer.getDefaultWriteParam();
iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
iwp.setCompressionQuality(fmt.getQuality());
fmt = validateFormat(fmt);
ImageOutputStream ios;
ios = ImageIO.createImageOutputStream(bos);
writer.setOutput(ios);
if(fmt.getEncoding() == ImageEncoding.JPG) {
WritableRaster raster = img.getRaster();
WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(),
img.getHeight(), 0, 0, new int[] {0, 1, 2});
DirectColorModel cm = (DirectColorModel)img.getColorModel();
DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(),
cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask());
// now create the new buffer that is used ot write the image:
BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null);
writer.write(null, new IIOImage(rgbBuffer, null, null), iwp);
writer.dispose();
rgbBuffer.flush();
// Find a jpeg writer
ImageWriter writer = null;
Iterator<ImageWriter> iter = ImageIO.getImageWritersByFormatName("jpg");
if (iter.hasNext()) {
writer = iter.next();
}
else if (fmt.getEncoding() == ImageEncoding.WEBP) {
doWEBPEncode(img, fmt, bos);
if(writer == null) {
Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding");
return null;
}
else {
ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */
}
} catch (IOException iox) {
Log.info("Error encoding image - " + iox.getMessage());
return null;
ImageWriteParam iwp = writer.getDefaultWriteParam();
iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
iwp.setCompressionQuality(fmt.getQuality());
ImageOutputStream ios;
ios = ImageIO.createImageOutputStream(bos);
writer.setOutput(ios);
writer.write(null, new IIOImage(rgbBuffer, null, null), iwp);
writer.dispose();
rgbBuffer.flush();
}
else if (fmt.getEncoding() == ImageEncoding.WEBP) {
doWEBPEncode(img, fmt, bos);
}
else {
ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */
}
} catch (IOException iox) {
Log.info("Error encoding image - " + iox.getMessage());
return null;
}
return bos;
}
public static BufferedImage imageIODecode(MapStorageTile.TileRead tr) throws IOException {
if(isRequiredJDKVersion(17,-1,-1)){
return imageIODecodeUnsafe(tr); //we can skip Thread safety for more performance
}
synchronized(imageioLock) {
ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
if (tr.format == ImageEncoding.WEBP) {
return doWEBPDecode(tr.image);
}
return ImageIO.read(tr.image);
return imageIODecodeUnsafe(tr);
}
}
private static BufferedImage imageIODecodeUnsafe(MapStorageTile.TileRead tr) throws IOException {
ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
if (tr.format == ImageEncoding.WEBP) {
return doWEBPDecode(tr.image);
}
return ImageIO.read(tr.image);
}
/**
* Checks if the current JDK is running at least a specific version
* targetMinor and targetBuild can be set to -1, if the java.version only provides a Major release this will then only check for the major release
* @param targetMajor the required minimum major version
* @param targetMinor the required minimum minor version
* @param targetBuild the required minimum build version
* @return true if the current JDK version is the required minimum version
*/
private static boolean isRequiredJDKVersion(int targetMajor, int targetMinor, int targetBuild){
String javaVersion = System.getProperty("java.version");
String[] versionParts = javaVersion.split("\\.");
if(versionParts.length < 3){
if(versionParts.length == 1
&& targetMinor == -1
&& targetBuild == -1
&& parseInt(versionParts[0], -1) >= targetMajor){
return true;//we only have a major version and thats ok
}
return false; //can not evaluate
}
int major = parseInt(versionParts[0], -1);
int minor = parseInt(versionParts[1], -1);
int build = parseInt(versionParts[2], -1);
if(major != -1 && major >= targetMajor &&
minor != -1 && minor >= targetMinor &&
build != -1 && build >= targetBuild
){
return true;
}
return false;
}
/**
* Parses a string to int, with a dynamic fallback value if not parsable
* @param input the String to parse
* @param fallback the Fallback value to use
* @return the parsed integer or the fallback value if unparsable
*/
private static int parseInt(String input, int fallback){
int output = fallback;
try{
output = Integer.parseInt(input);
} catch (NumberFormatException e) {}
return output;
}
}

1
DynmapCoreAPI/build.gradle
View File

@ -6,6 +6,7 @@ eclipse {
name = "Dynmap(DynmapCoreAPI)"
}
}
sourceCompatibility = targetCompatibility = compileJava.sourceCompatibility = compileJava.targetCompatibility = '1.8' // Need this here so eclipse task generates correctly.
description = "DynmapCoreAPI"