From 460e6f98151e482675a83cefb1a0cc7b75c906f7 Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Sun, 25 Oct 2020 18:40:33 -0500 Subject: [PATCH] Add WEBP support, via cwebp/dwebp tools --- .../src/main/java/org/dynmap/DynmapCore.java | 72 ++++- .../src/main/java/org/dynmap/DynmapWorld.java | 2 +- .../src/main/java/org/dynmap/MapType.java | 16 +- .../servlet/MapStorageResourceHandler.java | 7 +- .../java/org/dynmap/utils/ImageIOManager.java | 257 ++++++------------ 5 files changed, 170 insertions(+), 184 deletions(-) diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java index 56e7b4a0..b559af40 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java @@ -33,6 +33,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.dynmap.MapType.ImageEncoding; import org.dynmap.common.DynmapCommandSender; import org.dynmap.common.DynmapListenerManager; import org.dynmap.common.DynmapListenerManager.EventType; @@ -138,6 +139,12 @@ public class DynmapCore implements DynmapCommonAPI { private boolean loginRequired; + // WEBP support + private String cwebpPath; + private String dwebpPath; + private boolean did_cwebpPath_warn = false; + private boolean did_dwebpPath_warn = false; + /* Flag to let code know that we're doing reload - make sure we don't double-register event handlers */ public boolean is_reload = false; public static boolean ignore_chunk_loads = false; /* Flag keep us from processing our own chunk loads */ @@ -200,7 +207,6 @@ public class DynmapCore implements DynmapCommonAPI { public void setMinecraftVersion(String mcver) { this.platformVersion = mcver; } - public void setServer(DynmapServerInterface srv) { server = srv; } @@ -213,6 +219,21 @@ public class DynmapCore implements DynmapCommonAPI { public static final boolean migrateChunks() { return migrate_chunks; } + + public String getCWEBPPath() { + if ((cwebpPath == null) && (!did_cwebpPath_warn)) { + Log.severe("ERROR: trying to use WEBP without cwebp tool installed or cwebpPath set properly"); + did_cwebpPath_warn = true; + } + return cwebpPath; + } + public String getDWEBPPath() { + if ((dwebpPath == null) && (!did_dwebpPath_warn)) { + Log.severe("ERROR: trying to use WEBP without dwebp tool installed or dwebpPath set properly"); + did_dwebpPath_warn = true; + } + return dwebpPath; + } public final String getBiomeName(int biomeid) { String n = null; @@ -416,6 +437,20 @@ public class DynmapCore implements DynmapCommonAPI { return true; } + private String findExecutableOnPath(String fname) { + for (String dirname : System.getenv("PATH").split(File.pathSeparator)) { + File file = new File(dirname, fname); + if (file.isFile() && file.canExecute()) { + return file.getAbsolutePath(); + } + file = new File(dirname, fname + ".exe"); + if (file.isFile() && file.canExecute()) { + return file.getAbsolutePath(); + } + } + return null; + } + public boolean enableCore(EnableCoreCallbacks cb) { /* Update extracted files, if needed */ updateExtractedFiles(); @@ -427,14 +462,47 @@ public class DynmapCore implements DynmapCommonAPI { /* Load control for leaf transparency (spout lighting bug workaround) */ transparentLeaves = configuration.getBoolean("transparent-leaves", true); + + // Inject core instance + ImageIOManager.core = this; + // Check for webp support + cwebpPath = configuration.getString("cwebpPath", null); + dwebpPath = configuration.getString("dwebpPath", null); + if (cwebpPath == null) { + cwebpPath = findExecutableOnPath("cwebp"); + } + if (dwebpPath == null) { + dwebpPath = findExecutableOnPath("dwebp"); + } + if (cwebpPath != null) { + File file = new File(cwebpPath); + if (!file.isFile() || !file.canExecute()) { + cwebpPath = null; + } + } + if (dwebpPath != null) { + File file = new File(dwebpPath); + if (!file.isFile() || !file.canExecute()) { + dwebpPath = null; + } + } + if ((cwebpPath != null) && (dwebpPath != null)) { + Log.info("Found cwebp at " + cwebpPath + " and dwebp at " + dwebpPath + ": webp format enabled"); + } + else { + Log.warning("cwebp or dwebp not found, or cwebpPath or dwebpPath is invalid: webp format disabled"); + cwebpPath = dwebpPath = null; + } /* Get default image format */ def_image_format = configuration.getString("image-format", "png"); MapType.ImageFormat fmt = MapType.ImageFormat.fromID(def_image_format); - if(fmt == null) { + if ((fmt == null) || ((fmt.enc == ImageEncoding.WEBP) && (cwebpPath == null))) { Log.severe("Invalid image-format: " + def_image_format); def_image_format = "png"; + fmt = MapType.ImageFormat.fromID(def_image_format); } + DynmapWorld.doInitialScan(configuration.getBoolean("initial-zoomout-validate", true)); smoothlighting = configuration.getBoolean("smooth-lighting", false); diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapWorld.java b/DynmapCore/src/main/java/org/dynmap/DynmapWorld.java index ed743b00..85ff89e1 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapWorld.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapWorld.java @@ -157,7 +157,7 @@ public abstract class DynmapWorld { if (tr != null) { BufferedImage im = null; try { - im = ImageIOManager.imageIODecode(tr.image); + im = ImageIOManager.imageIODecode(tr); } catch (IOException iox) { // Broken file - zap it tile1.delete(); diff --git a/DynmapCore/src/main/java/org/dynmap/MapType.java b/DynmapCore/src/main/java/org/dynmap/MapType.java index bca1f797..aada3863 100644 --- a/DynmapCore/src/main/java/org/dynmap/MapType.java +++ b/DynmapCore/src/main/java/org/dynmap/MapType.java @@ -30,13 +30,16 @@ public abstract class MapType { } public enum ImageEncoding { - PNG("png"), JPG("jpg"); + PNG("png", "image/png"), JPG("jpg", "image/jpeg"), WEBP("webp", "image/webp"); public final String ext; + public final String mimetype; - ImageEncoding(String ext) { + ImageEncoding(String ext, String mime) { this.ext = ext; + this.mimetype = mime; } public String getFileExt() { return ext; } + public String getContentType() { return mimetype; } public static ImageEncoding fromOrd(int ix) { ImageEncoding[] v = values(); @@ -63,7 +66,14 @@ public abstract class MapType { FORMAT_JPG("jpg", 0.85f, ImageEncoding.JPG), FORMAT_JPG90("jpg-q90", 0.90f, ImageEncoding.JPG), FORMAT_JPG95("jpg-q95", 0.95f, ImageEncoding.JPG), - FORMAT_JPG100("jpg-q100", 1.00f, ImageEncoding.JPG); + FORMAT_JPG100("jpg-q100", 1.00f, ImageEncoding.JPG), + FORMAT_WEBP75("webp-q75", 75, ImageEncoding.WEBP), + FORMAT_WEBP80("webp-q80", 80, ImageEncoding.WEBP), + FORMAT_WEBP85("webp-q85", 85, ImageEncoding.WEBP), + FORMAT_WEBP("webp", 85, ImageEncoding.WEBP), + FORMAT_WEBP90("webp-q90", 90, ImageEncoding.WEBP), + FORMAT_WEBP95("webp-q95", 95, ImageEncoding.WEBP), + FORMAT_WEBP100("webp-q100", 100, ImageEncoding.WEBP); String id; float qual; ImageEncoding enc; diff --git a/DynmapCore/src/main/java/org/dynmap/servlet/MapStorageResourceHandler.java b/DynmapCore/src/main/java/org/dynmap/servlet/MapStorageResourceHandler.java index dccc7da4..824cf1a6 100644 --- a/DynmapCore/src/main/java/org/dynmap/servlet/MapStorageResourceHandler.java +++ b/DynmapCore/src/main/java/org/dynmap/servlet/MapStorageResourceHandler.java @@ -120,12 +120,7 @@ public class MapStorageResourceHandler extends AbstractHandler { // Got tile, package up for response response.setDateHeader("Last-Modified", tr.lastModified); response.setIntHeader("Content-Length", tr.image.length()); - if (tr.format == ImageEncoding.PNG) { - response.setContentType("image/png"); - } - else { - response.setContentType("image/jpeg"); - } + response.setContentType(tr.format.getContentType()); ServletOutputStream out = response.getOutputStream(); out.write(tr.image.buffer(), 0, tr.image.length()); out.flush(); diff --git a/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java b/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java index 50cb75cd..8e7dd86f 100644 --- a/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java +++ b/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java @@ -1,10 +1,8 @@ package org.dynmap.utils; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.RandomAccessFile; +import java.io.FileOutputStream; +import java.io.OutputStream; import java.util.Iterator; -import java.util.LinkedList; import java.awt.image.BufferedImage; import java.awt.image.DirectColorModel; import java.awt.image.WritableRaster; @@ -17,8 +15,14 @@ import javax.imageio.ImageWriter; import javax.imageio.stream.ImageOutputStream; import org.dynmap.Log; +import org.dynmap.MapType.ImageEncoding; import org.dynmap.MapType.ImageFormat; -import org.dynmap.debug.Debug; +import org.dynmap.storage.MapStorageTile; + +import com.google.common.io.Files; + +import org.dynmap.DynmapCore; + /** * Implements soft-locks for prevent concurrency issues with file updates */ @@ -26,14 +30,82 @@ public class ImageIOManager { public static String preUpdateCommand = null; public static String postUpdateCommand = null; private static Object imageioLock = new Object(); + public static DynmapCore core; // Injected during enableCore + private static boolean did_warning = false; + + private static ImageFormat validateFormat(ImageFormat fmt) { + // If WEBP, see if supported + if (fmt.getEncoding() == ImageEncoding.WEBP) { + if (core.getCWEBPPath() == null) { // No encoder? + if (!did_warning) { + Log.warning("Attempt to use WEBP support when not usable: using JPEG"); + did_warning = true; + } + fmt = ImageFormat.FORMAT_JPG; // Switch to JPEN + } + } + return fmt; + } + + private static void doWEBPEncode(BufferedImage img, ImageFormat fmt, OutputStream out) throws IOException { + BufferOutputStream bos = new BufferOutputStream(); + + ImageIO.write(img, "png", bos); // Encode as PNG in buffere output stream + // Write to a tmp file + File tmpfile = File.createTempFile("pngToWebp", "png"); + FileOutputStream fos = new FileOutputStream(tmpfile); + fos.write(bos.buf, 0, bos.len); + fos.close(); + // Run encoder to new new temp file + File tmpfile2 = File.createTempFile("pngToWebp", "webp"); + String args[] = { core.getCWEBPPath(), "-q", Integer.toString((int)fmt.getQuality()), tmpfile.getAbsolutePath(), "-o", tmpfile2.getAbsolutePath() }; + Process pr = Runtime.getRuntime().exec(args); + try { + pr.waitFor(); + } catch (InterruptedException ix) { + throw new IOException("Error waiting for encoder"); + } + // Read output file into output stream + Files.copy(tmpfile2, out); + out.flush(); + // Clean up temp files + tmpfile.delete(); + tmpfile2.delete(); + } + + private static BufferedImage doWEBPDecode(BufferInputStream buf) throws IOException { + // Write to a tmp file + File tmpfile = File.createTempFile("webpToPng", "webp"); + Files.write(buf.buffer(), tmpfile); + // Run encoder to new new temp file + File tmpfile2 = File.createTempFile("webpToPng", "png"); + String args[] = { core.getDWEBPPath(), tmpfile.getAbsolutePath(), "-o", tmpfile2.getAbsolutePath() }; + Process pr = Runtime.getRuntime().exec(args); + try { + pr.waitFor(); + } catch (InterruptedException ix) { + throw new IOException("Error waiting for encoder"); + } + // Read file + BufferedImage obuf = ImageIO.read(tmpfile2); + // Clean up temp files + tmpfile.delete(); + tmpfile2.delete(); + + return obuf; + } + public static BufferOutputStream imageIOEncode(BufferedImage img, ImageFormat fmt) { BufferOutputStream bos = new BufferOutputStream(); synchronized(imageioLock) { try { ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - if(fmt.getFileExt().equals("jpg")) { + + fmt = validateFormat(fmt); + + if(fmt.getEncoding() == ImageEncoding.JPG) { WritableRaster raster = img.getRaster(); WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(), img.getHeight(), 0, 0, new int[] {0, 1, 2}); @@ -66,6 +138,9 @@ public class ImageIOManager { rgbBuffer.flush(); } + else if (fmt.getEncoding() == ImageEncoding.WEBP) { + doWEBPEncode(img, fmt, bos); + } else { ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */ } @@ -77,175 +152,13 @@ public class ImageIOManager { return bos; } - private static final int MAX_WRITE_RETRIES = 6; - - private static LinkedList baoslist = new LinkedList(); - private static Object baos_lock = new Object(); - /** - * Wrapper for IOImage.write - implements retries for busy files - * @param img - buffered image to write - * @param fmt - format to use for file - * @param fname - filename - * @throws IOException if error writing file - */ - public static void imageIOWrite(BufferedImage img, ImageFormat fmt, File fname) throws IOException { - int retrycnt = 0; - boolean done = false; - byte[] rslt; - int rsltlen; - BufferOutputStream baos; - synchronized(baos_lock) { - if(baoslist.isEmpty()) { - baos = new BufferOutputStream(); - } - else { - baos = baoslist.removeFirst(); - baos.reset(); - } - } + public static BufferedImage imageIODecode(MapStorageTile.TileRead tr) throws IOException { synchronized(imageioLock) { ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - if(fmt.getFileExt().equals("jpg")) { - WritableRaster raster = img.getRaster(); - WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(), - img.getHeight(), 0, 0, new int[] {0, 1, 2}); - DirectColorModel cm = (DirectColorModel)img.getColorModel(); - DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(), - cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask()); - // now create the new buffer that is used ot write the image: - BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null); - - // Find a jpeg writer - ImageWriter writer = null; - Iterator iter = ImageIO.getImageWritersByFormatName("jpg"); - if (iter.hasNext()) { - writer = iter.next(); - } - if(writer == null) { - Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding"); - return; - } - ImageWriteParam iwp = writer.getDefaultWriteParam(); - iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); - iwp.setCompressionQuality(fmt.getQuality()); - - ImageOutputStream ios = ImageIO.createImageOutputStream(baos); - writer.setOutput(ios); - - writer.write(null, new IIOImage(rgbBuffer, null, null), iwp); - writer.dispose(); - - rgbBuffer.flush(); + if (tr.format == ImageEncoding.WEBP) { + return doWEBPDecode(tr.image); } - else { - ImageIO.write(img, fmt.getFileExt(), baos); /* Write to byte array stream - prevent bogus I/O errors */ - } - } - // Get buffer and length - rslt = baos.buf; - rsltlen = baos.len; - - File fcur = new File(fname.getPath()); - File fnew = new File(fname.getPath() + ".new"); - File fold = new File(fname.getPath() + ".old"); - while(!done) { - RandomAccessFile f = null; - try { - f = new RandomAccessFile(fnew, "rw"); - f.write(rslt, 0, rsltlen); - done = true; - } catch (IOException fnfx) { - if(retrycnt < MAX_WRITE_RETRIES) { - Debug.debug("Image file " + fname.getPath() + " - unable to write - retry #" + retrycnt); - try { Thread.sleep(50 << retrycnt); } catch (InterruptedException ix) { throw fnfx; } - retrycnt++; - } - else { - Log.info("Image file " + fname.getPath() + " - unable to write - failed"); - throw fnfx; - } - } finally { - if(f != null) { - try { f.close(); } catch (IOException iox) { done = false; } - } - if(done) { - if (preUpdateCommand != null && !preUpdateCommand.isEmpty()) { - try { - new ProcessBuilder(preUpdateCommand, fnew.getAbsolutePath()).start().waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - fcur.renameTo(fold); - fnew.renameTo(fname); - fold.delete(); - if (postUpdateCommand != null && !postUpdateCommand.isEmpty()) { - try { - new ProcessBuilder(postUpdateCommand, fname.getAbsolutePath()).start().waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - } - } - // Put back in pool - synchronized(baos_lock) { - baoslist.addFirst(baos); - } - } - /** - * Wrapper for IOImage.read - implements retries for busy files - * @param fname - file to read - * @return buffered image with contents - * @throws IOException if error reading file - */ - public static BufferedImage imageIORead(File fname) throws IOException { - int retrycnt = 0; - boolean done = false; - BufferedImage img = null; - - while(!done) { - FileInputStream fis = null; - try { - fis = new FileInputStream(fname); - byte[] b = new byte[(int) fname.length()]; - fis.read(b); - fis.close(); - fis = null; - BufferInputStream bais = new BufferInputStream(b); - synchronized(imageioLock) { - ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - img = ImageIO.read(bais); - } - bais.close(); - done = true; /* Done if no I/O error - retries don't fix format errors */ - } catch (IOException iox) { - } finally { - if(fis != null) { - try { fis.close(); } catch (IOException io) {} - fis = null; - } - } - if(!done) { - if(retrycnt < MAX_WRITE_RETRIES) { - Debug.debug("Image file " + fname.getPath() + " - unable to write - retry #" + retrycnt); - try { Thread.sleep(50 << retrycnt); } catch (InterruptedException ix) { } - retrycnt++; - } - else { - Log.info("Image file " + fname.getPath() + " - unable to read - failed"); - throw new IOException("Error reading image file " + fname.getPath()); - } - } - } - return img; - } - - public static BufferedImage imageIODecode(InputStream str) throws IOException { - synchronized(imageioLock) { - ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - return ImageIO.read(str); + return ImageIO.read(tr.image); } } }