diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index 4e3f63f6..d080cbab 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,6 +17,15 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' + implementation 'io.github.linktosriram:s3-lite-core:0.2.0' + implementation 'io.github.linktosriram:s3-lite-api:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-apache:0.2.0' + implementation 'io.github.linktosriram:s3-lite-util:0.2.0' + implementation 'org.apache.httpcomponents:httpclient:4.5.9' + implementation 'javax.xml.bind:jaxb-api:2.3.1' + implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1' } processResources { @@ -51,6 +60,13 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) + include(dependency('io.github.linktosriram:s3-lite-core:')) + include(dependency('io.github.linktosriram:s3-lite-api:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-spi:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-apache:')) + include(dependency('io.github.linktosriram:s3-lite-util:')) + include(dependency('org.apache.httpcomponents:httpclient:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -61,6 +77,8 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') + relocate('io.github.linktosriram.s3lite', 'org.dynmap.s3lite') + destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java index ef25f18c..316111bf 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java @@ -1,6 +1,9 @@ package org.dynmap; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; @@ -54,12 +57,14 @@ import org.dynmap.modsupport.ModSupportImpl; import org.dynmap.renderer.DynmapBlockState; import org.dynmap.servlet.*; import org.dynmap.storage.MapStorage; +import org.dynmap.storage.aws_s3.AWSS3MapStorage; import org.dynmap.storage.filetree.FileTreeMapStorage; import org.dynmap.storage.mysql.MySQLMapStorage; import org.dynmap.storage.mariadb.MariaDBMapStorage; import org.dynmap.storage.sqllte.SQLiteMapStorage; import org.dynmap.storage.postgresql.PostgreSQLMapStorage; import org.dynmap.utils.BlockStep; +import org.dynmap.utils.BufferOutputStream; import org.dynmap.utils.ImageIOManager; import org.dynmap.web.BanIPFilter; import org.dynmap.web.CustomHeaderFilter; @@ -439,6 +444,9 @@ public class DynmapCore implements DynmapCommonAPI { else if (storetype.equals("postgres") || storetype.equals("postgresql")) { defaultStorage = new PostgreSQLMapStorage(); } + else if (storetype.equals("aws_s3")) { + defaultStorage = new AWSS3MapStorage(); + } else { Log.severe("Invalid storage type for map data: " + storetype); return false; @@ -486,7 +494,10 @@ public class DynmapCore implements DynmapCommonAPI { authmgr = new WebAuthManager(this); defaultStorage.setLoginEnabled(this); } - + // If storage serves web files, extract and publsh them + if (defaultStorage.needsStaticWebFiles()) { + updateStaticWebToStorage(); + } /* Load control for leaf transparency (spout lighting bug workaround) */ transparentLeaves = configuration.getBoolean("transparent-leaves", true); @@ -2817,6 +2828,63 @@ public class DynmapCore implements DynmapCommonAPI { } return dir.delete(); } + private void updateStaticWebToStorage() { + if(jarfile == null) return; + // If doing update and web path update is disabled, send warning + if (!this.updatewebpathfiles) { + return; + } + Log.info("Publishing web files to storage"); + /* Open JAR as ZIP */ + ZipFile zf = null; + InputStream ins = null; + byte[] buf = new byte[2048]; + String n = null; + try { + zf = new ZipFile(jarfile); + Enumeration e = zf.entries(); + while (e.hasMoreElements()) { + ZipEntry ze = e.nextElement(); + n = ze.getName(); + if (!n.startsWith("extracted/web/")) { + continue; + } + n = n.substring("extracted/web/".length()); + // If file is going to web path, redirect it to the configured web + if (ze.isDirectory()) { + continue; + } + try { + ins = zf.getInputStream(ze); + BufferOutputStream buffer = new BufferOutputStream(); + int len; + while ((len = ins.read(buf)) >= 0) { + buffer.write(buf, 0, len); + } + defaultStorage.setStaticWebFile(n, buffer); + } catch(IOException io) { + Log.severe("Error updating file in storage - " + n, io); + } finally { + if (ins != null) { + ins.close(); + ins = null; + } + } + } + } catch (IOException iox) { + Log.severe("Error extracting file - " + n); + } finally { + if (ins != null) { + try { ins.close(); } catch (IOException iox) {} + ins = null; + } + if (zf != null) { + try { zf.close(); } catch (IOException iox) {} + zf = null; + } + } + } + private void updateExtractedFiles() { if(jarfile == null) return; File df = this.getDataFolder(); @@ -2918,13 +2986,13 @@ public class DynmapCore implements DynmapCommonAPI { } else { try { - f.getParentFile().mkdirs(); - fos = new FileOutputStream(f); - ins = zf.getInputStream(ze); - int len; - while ((len = ins.read(buf)) >= 0) { - fos.write(buf, 0, len); - } + f.getParentFile().mkdirs(); + fos = new FileOutputStream(f); + ins = zf.getInputStream(ze); + int len; + while ((len = ins.read(buf)) >= 0) { + fos.write(buf, 0, len); + } } catch(IOException io) { Log.severe("Error updating file - " + f.getPath(), io); } finally { diff --git a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java index 0ca7f1f9..0135a5e9 100644 --- a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java +++ b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java @@ -272,21 +272,28 @@ public class JsonFileClientUpdateComponent extends ClientUpdateComponent { byte[] outputBytes = sb.toString().getBytes(cs_utf8); MapManager.scheduleDelayedJob(new Runnable() { public void run() { - File f = new File(baseStandaloneDir, "config.js"); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(f); - fos.write(outputBytes); - } catch (IOException iox) { - Log.severe("Exception while writing " + f.getPath(), iox); - } finally { - if(fos != null) { - try { - fos.close(); - } catch (IOException x) {} - fos = null; - } - } + if (core.getDefaultMapStorage().needsStaticWebFiles()) { + BufferOutputStream os = new BufferOutputStream(); + os.write(outputBytes); + core.getDefaultMapStorage().setStaticWebFile("standalone/config.js", os); + } + else { + File f = new File(baseStandaloneDir, "config.js"); + FileOutputStream fos = null; + try { + fos = new FileOutputStream(f); + fos.write(outputBytes); + } catch (IOException iox) { + Log.severe("Exception while writing " + f.getPath(), iox); + } finally { + if(fos != null) { + try { + fos.close(); + } catch (IOException x) {} + fos = null; + } + } + } } }, 0); } diff --git a/DynmapCore/src/main/java/org/dynmap/MapType.java b/DynmapCore/src/main/java/org/dynmap/MapType.java index 744a0382..46dde413 100644 --- a/DynmapCore/src/main/java/org/dynmap/MapType.java +++ b/DynmapCore/src/main/java/org/dynmap/MapType.java @@ -47,6 +47,15 @@ public abstract class MapType { return v[ix]; return null; } + public static ImageEncoding fromContentType(String ct) { + ImageEncoding[] v = values(); + for (int i = 0; i < v.length; i++) { + if (v[i].mimetype.equalsIgnoreCase(ct)) { + return v[i]; + } + } + return null; + } public static ImageEncoding fromExt(String x) { ImageEncoding[] v = values(); for (int i = 0; i < v.length; i++) { diff --git a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java index 8106b29a..21f7f310 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java @@ -1,5 +1,6 @@ package org.dynmap.storage; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -459,6 +460,20 @@ public abstract class MapStorage { } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return false; + } + /** + * Set static web file content + * @param fileid - file path + * @param buffer - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream buffer) { + return false; + } + public void logSQLException(String opmsg, SQLException x) { Log.severe("SQLException: " + opmsg); Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState()); diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java new file mode 100644 index 00000000..cf068f05 --- /dev/null +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -0,0 +1,686 @@ +package org.dynmap.storage.aws_s3; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.dynmap.DynmapCore; +import org.dynmap.DynmapWorld; +import org.dynmap.Log; +import org.dynmap.MapType; +import org.dynmap.MapType.ImageEncoding; +import org.dynmap.MapType.ImageVariant; +import org.dynmap.PlayerFaces.FaceType; +import org.dynmap.WebAuthManager; +import org.dynmap.storage.MapStorage; +import org.dynmap.storage.MapStorageTile; +import org.dynmap.storage.MapStorageTileEnumCB; +import org.dynmap.storage.MapStorageBaseTileEnumCB; +import org.dynmap.storage.MapStorageTileSearchEndCB; +import org.dynmap.utils.BufferInputStream; +import org.dynmap.utils.BufferOutputStream; + +import io.github.linktosriram.s3lite.api.client.S3Client; +import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException; +import io.github.linktosriram.s3lite.api.exception.S3Exception; +import io.github.linktosriram.s3lite.api.region.Region; +import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest; +import io.github.linktosriram.s3lite.api.request.GetObjectRequest; +import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request; +import io.github.linktosriram.s3lite.api.request.PutObjectRequest; +import io.github.linktosriram.s3lite.api.response.GetObjectResponse; +import io.github.linktosriram.s3lite.api.response.ListObjectsV2Response; +import io.github.linktosriram.s3lite.api.response.ResponseBytes; +import io.github.linktosriram.s3lite.api.response.S3Object; +import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials; +import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder; +import io.github.linktosriram.s3lite.http.apache.ApacheSdkHttpClient; +import io.github.linktosriram.s3lite.http.spi.request.RequestBody; + +public class AWSS3MapStorage extends MapStorage { + public class StorageTile extends MapStorageTile { + private final String baseKey; + private final String uri; + + StorageTile(DynmapWorld world, MapType map, int x, int y, + int zoom, ImageVariant var) { + super(world, map, x, y, zoom, var); + + String baseURI; + if (zoom > 0) { + baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y; + } + else { + baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; + } + uri = baseURI + "." + map.getImageFormat().getFileExt(); + baseKey = AWSS3MapStorage.this.prefix + "tiles/" + world.getName() + "/" + uri; + } + @Override + public boolean exists() { + boolean exists = false; + try { + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) + exists = true; + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } + } + return exists; + } + + @Override + public boolean matchesHashCode(long hash) { + return false; + } + + @Override + public TileRead read() { + try { + GetObjectRequest req = GetObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + ResponseBytes obj = s3.getObjectAsBytes(req); + if (obj != null) { + GetObjectResponse rsp = obj.getResponse(); + TileRead tr = new TileRead(); + byte[] buf = obj.getBytes(); + tr.image = new BufferInputStream(buf); + tr.format = ImageEncoding.fromContentType(rsp.getContentType()); + tr.hashCode = rsp.geteTag().hashCode(); + tr.lastModified = rsp.getLastModified().toEpochMilli(); + + return tr; + } + } catch (NoSuchKeyException nskx) { + return null; // Nominal case if it doesn't exist + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + return null; + } + + @Override + public boolean write(long hash, BufferOutputStream encImage, long timestamp) { + boolean done = false; + try { + if (encImage == null) { // Delete? + DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + s3.deleteObject(req); + } + else { + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + } + done = true; + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + // Signal update for zoom out + if (zoom == 0) { + world.enqueueZoomOutUpdate(this); + } + return done; + } + + @Override + public boolean getWriteLock() { + return true; + } + + @Override + public void releaseWriteLock() { + } + + @Override + public boolean getReadLock(long timeout) { + return true; + } + + @Override + public void releaseReadLock() { + } + + @Override + public void cleanup() { + } + + @Override + public String getURI() { + return baseKey; + } + + @Override + public void enqueueZoomOutUpdate() { + world.enqueueZoomOutUpdate(this); + } + @Override + public MapStorageTile getZoomOutTile() { + int xx, yy; + int step = 1 << zoom; + if(x >= 0) + xx = x - (x % (2*step)); + else + xx = x + (x % (2*step)); + yy = -y; + if(yy >= 0) + yy = yy - (yy % (2*step)); + else + yy = yy + (yy % (2*step)); + yy = -yy; + return new StorageTile(world, map, xx, yy, zoom+1, var); + } + @Override + public boolean equals(Object o) { + if (o instanceof StorageTile) { + StorageTile st = (StorageTile) o; + return baseKey.equals(st.baseKey); + } + return false; + } + @Override + public int hashCode() { + return baseKey.hashCode(); + } + @Override + public String toString() { + return baseKey; + } + } + + private String bucketname; + private String region; + private String access_key_id; + private String secret_access_key; + private S3Client s3; + private String prefix; + + public AWSS3MapStorage() { + } + + @Override + public boolean init(DynmapCore core) { + if (!super.init(core)) { + return false; + } + if (!core.isInternalWebServerDisabled) { + Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt"); + return false; + } + if (core.isLoginSupportEnabled()) { + Log.severe("AWS S3 storage is not supported option with loegin support enabled: set login-enabled: false in configuration.txt"); + return false; + } + // Get our settings + bucketname = core.configuration.getString("storage/bucketname", "dynmap"); + region = core.configuration.getString("storage/region", "us-east-1"); + access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID")); + secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY")); + prefix = core.configuration.getString("storage/prefix", ""); + if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) { + prefix += '/'; + } + // Now creste the access client for the S3 service + Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region); + s3 = new DefaultS3ClientBuilder() + .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key)) + .region(Region.fromString(region)) + .httpClient(ApacheSdkHttpClient.defaultClient()) + .build(); + if (s3 == null) { + Log.severe("Error creating S3 access client"); + return false; + } + // Make sure bucket exists (do list) + ListObjectsV2Request listreq = ListObjectsV2Request.builder() + .bucketName(bucketname) + .maxKeys(1) + .prefix(prefix) + .build(); + try { + ListObjectsV2Response rslt = s3.listObjectsV2(listreq); + if (rslt == null) { + Log.severe("Error: cannot find or access S3 bucket"); + return false; + } + List content = rslt.getContents(); + } catch (S3Exception s3x) { + Log.severe("AWS Exception", s3x); + Log.severe("req=" + listreq); + return false; + } + return true; + } + + @Override + public MapStorageTile getTile(DynmapWorld world, MapType map, int x, int y, + int zoom, ImageVariant var) { + return new StorageTile(world, map, x, y, zoom, var); + } + + @Override + public MapStorageTile getTile(DynmapWorld world, String uri) { + String[] suri = uri.split("/"); + if (suri.length < 2) return null; + String mname = suri[0]; // Map URI - might include variant + MapType mt = null; + ImageVariant imgvar = null; + // Find matching map type and image variant + for (int mti = 0; (mt == null) && (mti < world.maps.size()); mti++) { + MapType type = world.maps.get(mti); + ImageVariant[] var = type.getVariants(); + for (int ivi = 0; (imgvar == null) && (ivi < var.length); ivi++) { + if (mname.equals(type.getPrefix() + var[ivi].variantSuffix)) { + mt = type; + imgvar = var[ivi]; + } + } + } + if (mt == null) { // Not found? + return null; + } + // Now, take the last section and parse out coordinates and zoom + String fname = suri[suri.length-1]; + String[] coord = fname.split("[_\\.]"); + if (coord.length < 3) { // 3 or 4 + return null; + } + int zoom = 0; + int x, y; + try { + if (coord[0].charAt(0) == 'z') { + zoom = coord[0].length(); + x = Integer.parseInt(coord[1]); + y = Integer.parseInt(coord[2]); + } + else { + x = Integer.parseInt(coord[0]); + y = Integer.parseInt(coord[1]); + } + return getTile(world, mt, x, y, zoom, imgvar); + } catch (NumberFormatException nfx) { + return null; + } + } + + + private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, + MapStorageTileSearchEndCB cbEnd) { + String basekey = prefix + "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build(); + boolean done = false; + try { + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + key = key.substring(basekey.length()); // Strip off base + // Parse the extension + String ext = null; + int extoff = key.lastIndexOf('.'); + if (extoff >= 0) { + ext = key.substring(extoff+1); + key = key.substring(0, extoff); + } + // If not valid image extension, ignore + ImageEncoding fmt = ImageEncoding.fromExt(ext); + if (fmt == null) { + continue; + } + // See if zoom tile: figure out zoom level + int zoom = 0; + if (key.startsWith("z")) { + while (key.startsWith("z")) { + key = key.substring(1); + zoom++; + } + if (key.startsWith("_")) { + key = key.substring(1); + } + } + // Split remainder to get coords + String[] coord = key.split("_"); + if (coord.length == 2) { // Must be 2 to be a tile + try { + int x = Integer.parseInt(coord[0]); + int y = Integer.parseInt(coord[1]); + // Invoke callback + MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); + if(cb != null) + cb.tileFound(t, fmt); + if(cbBase != null && t.zoom == 0) + cbBase.tileFound(t, fmt); + t.cleanup(); + } catch (NumberFormatException nfx) { + } + } + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } + } + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } + } + if(cbEnd != null) { + cbEnd.searchEnded(); + } + } + + @Override + public void enumMapTiles(DynmapWorld world, MapType map, MapStorageTileEnumCB cb) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processEnumMapTiles(world, mt, var, cb, null, null); + } + } + } + + @Override + public void enumMapBaseTiles(DynmapWorld world, MapType map, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processEnumMapTiles(world, mt, var, null, cbBase, cbEnd); + } + } + } + + private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { + String basekey = prefix + "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build(); + try { + boolean done = false; + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(key).build(); + s3.deleteObject(delreq); + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } + } + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } + } + } + + @Override + public void purgeMapTiles(DynmapWorld world, MapType map) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processPurgeMapTiles(world, mt, var); + } + } + } + + @Override + public boolean setPlayerFaceImage(String playername, FaceType facetype, + BufferOutputStream encImage) { + boolean done = false; + String baseKey = prefix + "faces/" + facetype.id + "/" + playername + ".png"; + try { + if (encImage == null) { // Delete? + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); + } + else { + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + } + done = true; + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + return done; + } + + @Override + public BufferInputStream getPlayerFaceImage(String playername, + FaceType facetype) { + return null; + } + + @Override + public boolean hasPlayerFaceImage(String playername, FaceType facetype) { + String baseKey = prefix + "faces/" + facetype.id + "/" + playername + ".png"; + boolean exists = false; + try { + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) + exists = true; + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } + } + return exists; + } + + @Override + public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { + boolean done = false; + String baseKey = prefix + "tiles/_markers_/" + markerid + ".png"; + try { + if (encImage == null) { // Delete? + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); + } + else { + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + } + done = true; + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + return done; + } + + @Override + public BufferInputStream getMarkerImage(String markerid) { + return null; + } + + @Override + public boolean setMarkerFile(String world, String content) { + boolean done = false; + String baseKey = prefix + "tiles/_markers_/marker_" + world + ".json"; + try { + if (content == null) { // Delete? + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); + } + else { + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("application/json").build(); + s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); + } + done = true; + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + return done; + } + + @Override + public String getMarkerFile(String world) { + return null; + } + + @Override + // For external web server only + public String getMarkersURI(boolean login_enabled) { + return "tiles/"; + } + + @Override + // For external web server only + public String getTilesURI(boolean login_enabled) { + return "tiles/"; + } + + /** + * URI to use for loading configuration JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getConfigurationJSONURI(boolean login_enabled) { + return "standalone/dynmap_config.json"; + } + /** + * URI to use for loading update JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getUpdateJSONURI(boolean login_enabled) { + return "standalone/dynmap_{world}.json"; + } + + @Override + public void addPaths(StringBuilder sb, DynmapCore core) { + String p = core.getTilesFolder().getAbsolutePath(); + if(!p.endsWith("/")) + p += "/"; + sb.append("$tilespath = \'"); + sb.append(WebAuthManager.esc(p)); + sb.append("\';\n"); + sb.append("$markerspath = \'"); + sb.append(WebAuthManager.esc(p)); + sb.append("\';\n"); + + // Need to call base to add webpath + super.addPaths(sb, core); + } + + + @Override + public BufferInputStream getStandaloneFile(String fileid) { + return null; + } + + + // Cache to avoid rewriting same standalong file repeatedly + private ConcurrentHashMap standalone_cache = new ConcurrentHashMap(); + + @Override + public boolean setStandaloneFile(String fileid, BufferOutputStream content) { + return setStaticWebFile("standalone/" + fileid, content); + } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return true; + } + /** + * Set static web file content + * @param fileid - file path + * @param content - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream content) { + + boolean done = false; + String baseKey = prefix + fileid; + try { + byte[] cacheval = standalone_cache.get(fileid); + + if (content == null) { // Delete? + if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached? + return true; + } + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); + standalone_cache.put(fileid, new byte[0]); // Mark in cache + } + else { + byte[] digest = content.buf; + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(content.buf); + digest = md.digest(); + } catch (NoSuchAlgorithmException nsax) { + + } + // If cached and same, just return + if (Arrays.equals(digest, cacheval)) { + return true; + } + String ct = "text/plain"; + if (fileid.endsWith(".json")) { + ct = "application/json"; + } + else if (fileid.endsWith(".php")) { + ct = "application/x-httpd-php"; + } + else if (fileid.endsWith(".html")) { + ct = "text/html"; + } + else if (fileid.endsWith(".css")) { + ct = "text/css"; + } + else if (fileid.endsWith(".js")) { + ct = "application/x-javascript"; + } + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build(); + s3.putObject(req, RequestBody.fromBytes(content.buf, content.len)); + standalone_cache.put(fileid, digest); + } + done = true; + } catch (S3Exception x) { + Log.severe("AWS Exception", x); + } + return done; + } + +} diff --git a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs index 36274c6d..8f00b17b 100644 --- a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs +++ b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs @@ -1,5 +1,5 @@ # -#Sun Feb 13 13:56:50 CST 2022 +#Thu Feb 17 22:40:49 CST 2022 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 diff --git a/fabric-1.14.4/src/main/resources/configuration.txt b/fabric-1.14.4/src/main/resources/configuration.txt index 49bc0515..a618dd32 100644 --- a/fabric-1.14.4/src/main/resources/configuration.txt +++ b/fabric-1.14.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.15.2/src/main/resources/configuration.txt b/fabric-1.15.2/src/main/resources/configuration.txt index 349a91b2..60fd3bed 100644 --- a/fabric-1.15.2/src/main/resources/configuration.txt +++ b/fabric-1.15.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.16.4/src/main/resources/configuration.txt b/fabric-1.16.4/src/main/resources/configuration.txt index 3814101c..83a3da83 100644 --- a/fabric-1.16.4/src/main/resources/configuration.txt +++ b/fabric-1.16.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.17.1/src/main/resources/configuration.txt b/fabric-1.17.1/src/main/resources/configuration.txt index 3814101c..83a3da83 100644 --- a/fabric-1.17.1/src/main/resources/configuration.txt +++ b/fabric-1.17.1/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.18/src/main/resources/configuration.txt b/fabric-1.18/src/main/resources/configuration.txt index 6facc34b..61de127f 100644 --- a/fabric-1.18/src/main/resources/configuration.txt +++ b/fabric-1.18/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.13.2/src/main/resources/configuration.txt b/forge-1.13.2/src/main/resources/configuration.txt index 29a2f80d..dad28957 100644 --- a/forge-1.13.2/src/main/resources/configuration.txt +++ b/forge-1.13.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.14.4/src/main/resources/configuration.txt b/forge-1.14.4/src/main/resources/configuration.txt index b720ffb3..5a76b049 100644 --- a/forge-1.14.4/src/main/resources/configuration.txt +++ b/forge-1.14.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.15.2/src/main/resources/configuration.txt b/forge-1.15.2/src/main/resources/configuration.txt index b720ffb3..5a76b049 100644 --- a/forge-1.15.2/src/main/resources/configuration.txt +++ b/forge-1.15.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.16.5/src/main/resources/configuration.txt b/forge-1.16.5/src/main/resources/configuration.txt index 05cc179c..09180111 100644 --- a/forge-1.16.5/src/main/resources/configuration.txt +++ b/forge-1.16.5/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.17.1/src/main/resources/configuration.txt b/forge-1.17.1/src/main/resources/configuration.txt index ec572378..93ecb163 100644 --- a/forge-1.17.1/src/main/resources/configuration.txt +++ b/forge-1.17.1/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.18/src/main/resources/configuration.txt b/forge-1.18/src/main/resources/configuration.txt index 05cc179c..09180111 100644 --- a/forge-1.18/src/main/resources/configuration.txt +++ b/forge-1.18/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index 9712bf72..d8b39046 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -40,6 +40,13 @@ storage: #password: dynmap #prefix: "" #flags: "?allowReconnect=true&autoReconnect=true" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" components: - class: org.dynmap.ClientConfigurationComponent