From ea97296684b0263d7549e50adc220ff6b613e87d Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Sun, 20 Feb 2022 20:58:21 -0600 Subject: [PATCH] Get AWS S3 storage based web working --- DynmapCore/build.gradle | 41 +- .../src/main/java/org/dynmap/DynmapCore.java | 80 +++- .../dynmap/JsonFileClientUpdateComponent.java | 37 +- .../java/org/dynmap/storage/MapStorage.java | 15 + .../storage/aws_s3/AWSS3MapStorage.java | 430 +++++++++++------- bukkit-helper/.project | 37 +- 6 files changed, 409 insertions(+), 231 deletions(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index f6b85116..d080cbab 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,17 +17,15 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' - implementation 'software.amazon.awssdk:s3:2.17.132' - implementation 'software.amazon.awssdk:aws-core:2.17.132' - implementation 'software.amazon.awssdk:sdk-core:2.17.132' - implementation 'software.amazon.awssdk:utils:2.17.132' - implementation 'software.amazon.awssdk:http-client-spi:2.17.132' - implementation 'software.amazon.awssdk:profiles:2.17.132' - implementation 'software.amazon.awssdk:regions:2.17.132' - implementation 'software.amazon.awssdk:auth:2.17.132' - implementation 'software.amazon.awssdk:metrics-spi:2.17.132' - implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132' - implementation 'software.amazon.awssdk:protocol-core:2.17.132' + implementation 'io.github.linktosriram:s3-lite-core:0.2.0' + implementation 'io.github.linktosriram:s3-lite-api:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-apache:0.2.0' + implementation 'io.github.linktosriram:s3-lite-util:0.2.0' + implementation 'org.apache.httpcomponents:httpclient:4.5.9' + implementation 'javax.xml.bind:jaxb-api:2.3.1' + implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1' } processResources { @@ -62,17 +60,13 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) - include(dependency('software.amazon.awssdk:s3:')) - include(dependency('software.amazon.awssdk:aws-core:')) - include(dependency('software.amazon.awssdk:sdk-core:')) - include(dependency('software.amazon.awssdk:utils:')) - include(dependency('software.amazon.awssdk:http-client-spi:')) - include(dependency('software.amazon.awssdk:profiles:')) - include(dependency('software.amazon.awssdk:regions:')) - include(dependency('software.amazon.awssdk:auth:')) - include(dependency('software.amazon.awssdk:metrics-spi:')) - include(dependency('software.amazon.awssdk:aws-xml-protocol:')) - include(dependency('software.amazon.awssdk:protocol-core:')) + include(dependency('io.github.linktosriram:s3-lite-core:')) + include(dependency('io.github.linktosriram:s3-lite-api:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-spi:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-apache:')) + include(dependency('io.github.linktosriram:s3-lite-util:')) + include(dependency('org.apache.httpcomponents:httpclient:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -83,7 +77,8 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') - relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk') + relocate('io.github.linktosriram.s3lite', 'org.dynmap.s3lite') + destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java index e3984c44..316111bf 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java @@ -1,6 +1,9 @@ package org.dynmap; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; @@ -61,6 +64,7 @@ import org.dynmap.storage.mariadb.MariaDBMapStorage; import org.dynmap.storage.sqllte.SQLiteMapStorage; import org.dynmap.storage.postgresql.PostgreSQLMapStorage; import org.dynmap.utils.BlockStep; +import org.dynmap.utils.BufferOutputStream; import org.dynmap.utils.ImageIOManager; import org.dynmap.web.BanIPFilter; import org.dynmap.web.CustomHeaderFilter; @@ -490,7 +494,10 @@ public class DynmapCore implements DynmapCommonAPI { authmgr = new WebAuthManager(this); defaultStorage.setLoginEnabled(this); } - + // If storage serves web files, extract and publsh them + if (defaultStorage.needsStaticWebFiles()) { + updateStaticWebToStorage(); + } /* Load control for leaf transparency (spout lighting bug workaround) */ transparentLeaves = configuration.getBoolean("transparent-leaves", true); @@ -2821,6 +2828,63 @@ public class DynmapCore implements DynmapCommonAPI { } return dir.delete(); } + private void updateStaticWebToStorage() { + if(jarfile == null) return; + // If doing update and web path update is disabled, send warning + if (!this.updatewebpathfiles) { + return; + } + Log.info("Publishing web files to storage"); + /* Open JAR as ZIP */ + ZipFile zf = null; + InputStream ins = null; + byte[] buf = new byte[2048]; + String n = null; + try { + zf = new ZipFile(jarfile); + Enumeration e = zf.entries(); + while (e.hasMoreElements()) { + ZipEntry ze = e.nextElement(); + n = ze.getName(); + if (!n.startsWith("extracted/web/")) { + continue; + } + n = n.substring("extracted/web/".length()); + // If file is going to web path, redirect it to the configured web + if (ze.isDirectory()) { + continue; + } + try { + ins = zf.getInputStream(ze); + BufferOutputStream buffer = new BufferOutputStream(); + int len; + while ((len = ins.read(buf)) >= 0) { + buffer.write(buf, 0, len); + } + defaultStorage.setStaticWebFile(n, buffer); + } catch(IOException io) { + Log.severe("Error updating file in storage - " + n, io); + } finally { + if (ins != null) { + ins.close(); + ins = null; + } + } + } + } catch (IOException iox) { + Log.severe("Error extracting file - " + n); + } finally { + if (ins != null) { + try { ins.close(); } catch (IOException iox) {} + ins = null; + } + if (zf != null) { + try { zf.close(); } catch (IOException iox) {} + zf = null; + } + } + } + private void updateExtractedFiles() { if(jarfile == null) return; File df = this.getDataFolder(); @@ -2922,13 +2986,13 @@ public class DynmapCore implements DynmapCommonAPI { } else { try { - f.getParentFile().mkdirs(); - fos = new FileOutputStream(f); - ins = zf.getInputStream(ze); - int len; - while ((len = ins.read(buf)) >= 0) { - fos.write(buf, 0, len); - } + f.getParentFile().mkdirs(); + fos = new FileOutputStream(f); + ins = zf.getInputStream(ze); + int len; + while ((len = ins.read(buf)) >= 0) { + fos.write(buf, 0, len); + } } catch(IOException io) { Log.severe("Error updating file - " + f.getPath(), io); } finally { diff --git a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java index 0ca7f1f9..0135a5e9 100644 --- a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java +++ b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java @@ -272,21 +272,28 @@ public class JsonFileClientUpdateComponent extends ClientUpdateComponent { byte[] outputBytes = sb.toString().getBytes(cs_utf8); MapManager.scheduleDelayedJob(new Runnable() { public void run() { - File f = new File(baseStandaloneDir, "config.js"); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(f); - fos.write(outputBytes); - } catch (IOException iox) { - Log.severe("Exception while writing " + f.getPath(), iox); - } finally { - if(fos != null) { - try { - fos.close(); - } catch (IOException x) {} - fos = null; - } - } + if (core.getDefaultMapStorage().needsStaticWebFiles()) { + BufferOutputStream os = new BufferOutputStream(); + os.write(outputBytes); + core.getDefaultMapStorage().setStaticWebFile("standalone/config.js", os); + } + else { + File f = new File(baseStandaloneDir, "config.js"); + FileOutputStream fos = null; + try { + fos = new FileOutputStream(f); + fos.write(outputBytes); + } catch (IOException iox) { + Log.severe("Exception while writing " + f.getPath(), iox); + } finally { + if(fos != null) { + try { + fos.close(); + } catch (IOException x) {} + fos = null; + } + } + } } }, 0); } diff --git a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java index 8106b29a..21f7f310 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java @@ -1,5 +1,6 @@ package org.dynmap.storage; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -459,6 +460,20 @@ public abstract class MapStorage { } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return false; + } + /** + * Set static web file content + * @param fileid - file path + * @param buffer - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream buffer) { + return false; + } + public void logSQLException(String opmsg, SQLException x) { Log.severe("SQLException: " + opmsg); Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState()); diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 38ffeafb..1ce8a8df 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -1,9 +1,14 @@ package org.dynmap.storage.aws_s3; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import org.dynmap.DynmapCore; import org.dynmap.DynmapWorld; @@ -21,33 +26,32 @@ import org.dynmap.storage.MapStorageTileSearchEndCB; import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferOutputStream; -import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; -import software.amazon.awssdk.services.s3.model.GetBucketAclResponse; -import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; -import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Object; +import io.github.linktosriram.s3lite.api.client.S3Client; +import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException; +import io.github.linktosriram.s3lite.api.exception.S3Exception; +import io.github.linktosriram.s3lite.api.region.Region; +import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest; +import io.github.linktosriram.s3lite.api.request.GetObjectRequest; +import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request; +import io.github.linktosriram.s3lite.api.request.PutObjectRequest; +import io.github.linktosriram.s3lite.api.response.GetObjectResponse; +import io.github.linktosriram.s3lite.api.response.ListObjectsV2Response; +import io.github.linktosriram.s3lite.api.response.ResponseBytes; +import io.github.linktosriram.s3lite.api.response.S3Object; +import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials; +import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder; +import io.github.linktosriram.s3lite.http.apache.ApacheSdkHttpClient; +import io.github.linktosriram.s3lite.http.spi.request.RequestBody; public class AWSS3MapStorage extends MapStorage { public class StorageTile extends MapStorageTile { private final String baseKey; + private final String uri; StorageTile(DynmapWorld world, MapType map, int x, int y, int zoom, ImageVariant var) { super(world, map, x, y, zoom, var); + String baseURI; if (zoom > 0) { baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y; @@ -55,18 +59,21 @@ public class AWSS3MapStorage extends MapStorage { else { baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; } - baseKey = "tiles/" + world.getName() + "/" + baseURI + "." + map.getImageFormat().getFileExt(); + uri = baseURI + "." + map.getImageFormat().getFileExt(); + baseKey = "tiles/" + world.getName() + "/" + uri; } @Override public boolean exists() { boolean exists = false; try { - GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); - GetObjectAclResponse rslt = s3.getObjectAcl(req); - if (rslt != null) + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) exists = true; - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } } return exists; } @@ -78,25 +85,24 @@ public class AWSS3MapStorage extends MapStorage { @Override public TileRead read() { - AWSS3MapStorage.this.getWriteLock(baseKey); try { - GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectRequest req = GetObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); ResponseBytes obj = s3.getObjectAsBytes(req); if (obj != null) { - GetObjectResponse rsp = obj.response(); + GetObjectResponse rsp = obj.getResponse(); TileRead tr = new TileRead(); - byte[] buf = obj.asByteArray(); + byte[] buf = obj.getBytes(); tr.image = new BufferInputStream(buf); - tr.format = ImageEncoding.fromContentType(rsp.contentType()); - tr.hashCode = rsp.eTag().hashCode(); - tr.lastModified = rsp.lastModified().toEpochMilli(); + tr.format = ImageEncoding.fromContentType(rsp.getContentType()); + tr.hashCode = rsp.geteTag().hashCode(); + tr.lastModified = rsp.getLastModified().toEpochMilli(); return tr; } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); - } finally { - AWSS3MapStorage.this.releaseWriteLock(baseKey); + } catch (NoSuchKeyException nskx) { + return null; // Nominal case if it doesn't exist + } catch (S3Exception x) { + Log.severe("AWS Exception", x); } return null; } @@ -104,21 +110,18 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean write(long hash, BufferOutputStream encImage, long timestamp) { boolean done = false; - AWSS3MapStorage.this.getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(req); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - AWSS3MapStorage.this.releaseWriteLock(baseKey); } // Signal update for zoom out if (zoom == 0) { @@ -129,22 +132,20 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean getWriteLock() { - return AWSS3MapStorage.this.getWriteLock(baseKey); + return true; } @Override public void releaseWriteLock() { - AWSS3MapStorage.this.releaseWriteLock(baseKey); } @Override public boolean getReadLock(long timeout) { - return AWSS3MapStorage.this.getReadLock(baseKey, timeout); + return true; } @Override public void releaseReadLock() { - AWSS3MapStorage.this.releaseReadLock(baseKey); } @Override @@ -153,7 +154,7 @@ public class AWSS3MapStorage extends MapStorage { @Override public String getURI() { - return null; + return baseKey; } @Override @@ -196,7 +197,8 @@ public class AWSS3MapStorage extends MapStorage { private String bucketname; private String region; - private String profile_id; + private String access_key_id; + private String secret_access_key; private S3Client s3; public AWSS3MapStorage() { @@ -211,27 +213,45 @@ public class AWSS3MapStorage extends MapStorage { Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt"); return false; } + if (core.isLoginSupportEnabled()) { + Log.severe("AWS S3 storage is not supported option with loegin support enabled: set login-enabled: false in configuration.txt"); + return false; + } // Get our settings bucketname = core.configuration.getString("storage/bucketname", "dynmap"); region = core.configuration.getString("storage/region", "us-east-1"); + access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID")); + secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY")); + // Now creste the access client for the S3 service + Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region); + s3 = new DefaultS3ClientBuilder() + .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key)) + .region(Region.fromString(region)) + .httpClient(ApacheSdkHttpClient.defaultClient()) + .build(); + if (s3 == null) { + Log.severe("Error creating S3 access client"); + return false; + } + // Make sure bucket exists (do list) + ListObjectsV2Request listreq = ListObjectsV2Request.builder() + .bucketName(bucketname) + .maxKeys(1) + .build(); try { - // Now creste the access client for the S3 service - Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); - s3 = S3Client.builder().region(Region.of(region)).build(); - if (s3 == null) { - Log.severe("Error creating S3 access client"); - return false; - } - // Make sure bucket exists and get ACL - GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build(); - GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr); - if (bucketACL == null) { + ListObjectsV2Response rslt = s3.listObjectsV2(listreq); + if (rslt == null) { Log.severe("Error: cannot find or access S3 bucket"); return false; } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); - return false; + List content = rslt.getContents(); + Log.info("content=" + content.size()); + } catch (S3Exception s3x) { + if (!s3x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", s3x); + Log.severe("req=" + listreq); + return false; + } } return true; } @@ -288,57 +308,71 @@ public class AWSS3MapStorage extends MapStorage { } - private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { + private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, + MapStorageTileSearchEndCB cbEnd) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build(); + boolean done = false; try { - ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); - ListObjectsV2Response result = s3.listObjectsV2(req); - List objects = result.contents(); - for (S3Object os : objects) { - String key = os.key(); - key = key.substring(basekey.length()); // Strip off base - // Parse the extension - String ext = null; - int extoff = key.lastIndexOf('.'); - if (extoff >= 0) { - ext = key.substring(extoff+1); - key = key.substring(0, extoff); - } - // If not valid image extension, ignore - ImageEncoding fmt = ImageEncoding.fromExt(ext); - if (fmt == null) { - continue; - } - // See if zoom tile: figure out zoom level - int zoom = 0; - if (key.startsWith("z")) { - while (key.startsWith("z")) { - key = key.substring(1); - zoom++; - } - if (key.startsWith("_")) { - key = key.substring(1); - } - } - // Split remainder to get coords - String[] coord = key.split("_"); - if (coord.length == 2) { // Must be 2 to be a tile - try { - int x = Integer.parseInt(coord[0]); - int y = Integer.parseInt(coord[1]); - // Invoke callback - MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); - if(cb != null) - cb.tileFound(t, fmt); - if(cbBase != null && t.zoom == 0) - cbBase.tileFound(t, fmt); - t.cleanup(); - } catch (NumberFormatException nfx) { - } - } - } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + key = key.substring(basekey.length()); // Strip off base + // Parse the extension + String ext = null; + int extoff = key.lastIndexOf('.'); + if (extoff >= 0) { + ext = key.substring(extoff+1); + key = key.substring(0, extoff); + } + // If not valid image extension, ignore + ImageEncoding fmt = ImageEncoding.fromExt(ext); + if (fmt == null) { + continue; + } + // See if zoom tile: figure out zoom level + int zoom = 0; + if (key.startsWith("z")) { + while (key.startsWith("z")) { + key = key.substring(1); + zoom++; + } + if (key.startsWith("_")) { + key = key.substring(1); + } + } + // Split remainder to get coords + String[] coord = key.split("_"); + if (coord.length == 2) { // Must be 2 to be a tile + try { + int x = Integer.parseInt(coord[0]); + int y = Integer.parseInt(coord[1]); + // Invoke callback + MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); + if(cb != null) + cb.tileFound(t, fmt); + if(cbBase != null && t.zoom == 0) + cbBase.tileFound(t, fmt); + t.cleanup(); + } catch (NumberFormatException nfx) { + } + } + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } + } + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } } if(cbEnd != null) { cbEnd.searchEnded(); @@ -383,28 +417,30 @@ public class AWSS3MapStorage extends MapStorage { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build(); try { - ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); - ListObjectsV2Response result = s3.listObjectsV2(req); - List objects = result.contents(); - ArrayList keys = new ArrayList(); - for (S3Object os : objects) { - String key = os.key(); - keys.add(ObjectIdentifier.builder().key(key).build()); - if (keys.size() >= 100) { - DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); - s3.deleteObjects(delreq); - keys.clear(); - } + boolean done = false; + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(key).build(); + s3.deleteObject(delreq); + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } } - // Any left? - if (keys.size() > 0) { - DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); - s3.deleteObjects(delreq); - keys.clear(); - } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } } } @@ -431,21 +467,18 @@ public class AWSS3MapStorage extends MapStorage { BufferOutputStream encImage) { boolean done = false; String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; - getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -461,12 +494,14 @@ public class AWSS3MapStorage extends MapStorage { String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; boolean exists = false; try { - GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); - GetObjectAclResponse rslt = s3.getObjectAcl(req); - if (rslt != null) + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) exists = true; - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } } return exists; } @@ -475,21 +510,18 @@ public class AWSS3MapStorage extends MapStorage { public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { boolean done = false; String baseKey = "_markers_/" + markerid + ".png"; - getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -503,21 +535,18 @@ public class AWSS3MapStorage extends MapStorage { public boolean setMarkerFile(String world, String content) { boolean done = false; String baseKey = "_markers_/marker_" + world + ".json"; - getWriteLock(baseKey); try { if (content == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build(); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("application/json").build(); s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -530,15 +559,32 @@ public class AWSS3MapStorage extends MapStorage { @Override // For external web server only public String getMarkersURI(boolean login_enabled) { - return login_enabled?"standalone/markers.php?marker=":"tiles/"; + return "tiles/"; } @Override // For external web server only public String getTilesURI(boolean login_enabled) { - return login_enabled?"standalone/tiles.php?tile=":"tiles/"; + return "tiles/"; } + /** + * URI to use for loading configuration JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getConfigurationJSONURI(boolean login_enabled) { + return "standalone/dynmap_config.json"; + } + /** + * URI to use for loading update JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getUpdateJSONURI(boolean login_enabled) { + return "standalone/dynmap_{world}.json"; + } + @Override public void addPaths(StringBuilder sb, DynmapCore core) { String p = core.getTilesFolder().getAbsolutePath(); @@ -561,28 +607,76 @@ public class AWSS3MapStorage extends MapStorage { return null; } + + // Cache to avoid rewriting same standalong file repeatedly + private ConcurrentHashMap standalone_cache = new ConcurrentHashMap(); + @Override public boolean setStandaloneFile(String fileid, BufferOutputStream content) { + return setStaticWebFile("standalone/" + fileid, content); + } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return true; + } + /** + * Set static web file content + * @param fileid - file path + * @param content - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream content) { boolean done = false; - String baseKey = "standalone/" + fileid; - getWriteLock(baseKey); try { + byte[] cacheval = standalone_cache.get(fileid); + if (content == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached? + return true; + } + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(fileid).build(); s3.deleteObject(delreq); + standalone_cache.put(fileid, new byte[0]); // Mark in cache } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build(); - s3.putObject(req, RequestBody.fromBytes(content.buf)); + byte[] digest = content.buf; + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(content.buf); + digest = md.digest(); + } catch (NoSuchAlgorithmException nsax) { + + } + // If cached and same, just return + if (Arrays.equals(digest, cacheval)) { + return true; + } + String ct = "text/plain"; + if (fileid.endsWith(".json")) { + ct = "application/json"; + } + else if (fileid.endsWith(".php")) { + ct = "application/x-httpd-php"; + } + else if (fileid.endsWith(".html")) { + ct = "text/html"; + } + else if (fileid.endsWith(".css")) { + ct = "text/css"; + } + else if (fileid.endsWith(".js")) { + ct = "application/x-javascript"; + } + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(fileid).contentType(ct).build(); + s3.putObject(req, RequestBody.fromBytes(content.buf, content.len)); + standalone_cache.put(fileid, digest); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; - } + } diff --git a/bukkit-helper/.project b/bukkit-helper/.project index e86ee638..b372cd57 100644 --- a/bukkit-helper/.project +++ b/bukkit-helper/.project @@ -2,32 +2,35 @@ Dynmap(Spigot-Common) bukkit-helper - + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature org.eclipse.buildship.core.gradleprojectnature - - - org.eclipse.jdt.core.javabuilder - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - org.eclipse.m2e.core.maven2Builder - - - - 1 + 30 - org.eclipse.core.resources.regexFilterMatcher node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__