From be43f53d818db93f99b0900237e4274e4a040409 Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Thu, 17 Feb 2022 21:59:53 -0600 Subject: [PATCH 1/4] Start S3 support --- DynmapCore/build.gradle | 14 + .../src/main/java/org/dynmap/DynmapCore.java | 4 + .../src/main/java/org/dynmap/MapType.java | 9 + .../storage/aws_s3/AWSS3MapStorage.java | 590 ++++++++++++++++++ bukkit-helper/.project | 37 +- .../.settings/org.eclipse.jdt.core.prefs | 2 +- spigot/src/main/resources/configuration.txt | 8 + 7 files changed, 643 insertions(+), 21 deletions(-) create mode 100644 DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index 4e3f63f6..cdd7e06c 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,6 +17,12 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' + implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000') + implementation 'com.amazonaws:aws-java-sdk-core' + implementation 'com.amazonaws:aws-java-sdk-s3' + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' } processResources { @@ -51,6 +57,12 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) + include(dependency('com.amazonaws:aws-java-sdk-bom:')) + include(dependency('com.amazonaws:aws-java-sdk-core:')) + include(dependency('com.amazonaws:aws-java-sdk-s3:')) + include(dependency('com.fasterxml.jackson.core:jackson-core:')) + include(dependency('com.fasterxml.jackson.core:jackson-databind:')) + include(dependency('com.fasterxml.jackson.core:jackson-annotations:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -61,6 +73,8 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') + relocate('com.amazonaws', 'org.dynmap.com.amazonaws') + relocate('com.fasterxml', 'org.dynmap.com.fasterxml') destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java index ef25f18c..e3984c44 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java @@ -54,6 +54,7 @@ import org.dynmap.modsupport.ModSupportImpl; import org.dynmap.renderer.DynmapBlockState; import org.dynmap.servlet.*; import org.dynmap.storage.MapStorage; +import org.dynmap.storage.aws_s3.AWSS3MapStorage; import org.dynmap.storage.filetree.FileTreeMapStorage; import org.dynmap.storage.mysql.MySQLMapStorage; import org.dynmap.storage.mariadb.MariaDBMapStorage; @@ -439,6 +440,9 @@ public class DynmapCore implements DynmapCommonAPI { else if (storetype.equals("postgres") || storetype.equals("postgresql")) { defaultStorage = new PostgreSQLMapStorage(); } + else if (storetype.equals("aws_s3")) { + defaultStorage = new AWSS3MapStorage(); + } else { Log.severe("Invalid storage type for map data: " + storetype); return false; diff --git a/DynmapCore/src/main/java/org/dynmap/MapType.java b/DynmapCore/src/main/java/org/dynmap/MapType.java index 744a0382..46dde413 100644 --- a/DynmapCore/src/main/java/org/dynmap/MapType.java +++ b/DynmapCore/src/main/java/org/dynmap/MapType.java @@ -47,6 +47,15 @@ public abstract class MapType { return v[ix]; return null; } + public static ImageEncoding fromContentType(String ct) { + ImageEncoding[] v = values(); + for (int i = 0; i < v.length; i++) { + if (v[i].mimetype.equalsIgnoreCase(ct)) { + return v[i]; + } + } + return null; + } public static ImageEncoding fromExt(String x) { ImageEncoding[] v = values(); for (int i = 0; i < v.length; i++) { diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java new file mode 100644 index 00000000..055fa52b --- /dev/null +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -0,0 +1,590 @@ +package org.dynmap.storage.aws_s3; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.dynmap.DynmapCore; +import org.dynmap.DynmapWorld; +import org.dynmap.Log; +import org.dynmap.MapType; +import org.dynmap.MapType.ImageEncoding; +import org.dynmap.MapType.ImageVariant; +import org.dynmap.PlayerFaces.FaceType; +import org.dynmap.WebAuthManager; +import org.dynmap.storage.MapStorage; +import org.dynmap.storage.MapStorageTile; +import org.dynmap.storage.MapStorageTileEnumCB; +import org.dynmap.storage.MapStorageBaseTileEnumCB; +import org.dynmap.storage.MapStorageTileSearchEndCB; +import org.dynmap.utils.BufferInputStream; +import org.dynmap.utils.BufferOutputStream; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +public class AWSS3MapStorage extends MapStorage { + public class StorageTile extends MapStorageTile { + private final String baseKey; + + StorageTile(DynmapWorld world, MapType map, int x, int y, + int zoom, ImageVariant var) { + super(world, map, x, y, zoom, var); + String baseURI; + if (zoom > 0) { + baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y; + } + else { + baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; + } + baseKey = "tiles/" + world.getName() + "/" + baseURI + "." + map.getImageFormat().getFileExt(); + } + @Override + public boolean exists() { + boolean exists = false; + try { + AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + if (rslt != null) + exists = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } + return exists; + } + + @Override + public boolean matchesHashCode(long hash) { + return false; + } + + @Override + public TileRead read() { + AWSS3MapStorage.this.getWriteLock(baseKey); + try { + S3Object obj = s3.getObject(bucketname, baseKey); + if (obj != null) { + ObjectMetadata md = obj.getObjectMetadata(); + TileRead tr = new TileRead(); + byte[] buf = new byte[(int) md.getContentLength()]; + InputStream fis = obj.getObjectContent(); + fis.read(buf, 0, buf.length); // Read whole thing + tr.image = new BufferInputStream(buf); + tr.format = ImageEncoding.fromContentType(md.getContentType()); + tr.hashCode = md.getContentMD5().hashCode(); + tr.lastModified = md.getLastModified().getTime(); + + return tr; + } + } catch (IOException x) { + Log.severe("AWS Exception", x); + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + AWSS3MapStorage.this.releaseWriteLock(baseKey); + } + return null; + } + + @Override + public boolean write(long hash, BufferOutputStream encImage, long timestamp) { + boolean done = false; + AWSS3MapStorage.this.getWriteLock(baseKey); + try { + if (encImage == null) { // Delete? + s3.deleteObject(bucketname, baseKey); + } + else { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType(map.getImageFormat().getEncoding().getContentType()); + s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + } + done = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + AWSS3MapStorage.this.releaseWriteLock(baseKey); + } + // Signal update for zoom out + if (zoom == 0) { + world.enqueueZoomOutUpdate(this); + } + return done; + } + + @Override + public boolean getWriteLock() { + return AWSS3MapStorage.this.getWriteLock(baseKey); + } + + @Override + public void releaseWriteLock() { + AWSS3MapStorage.this.releaseWriteLock(baseKey); + } + + @Override + public boolean getReadLock(long timeout) { + return AWSS3MapStorage.this.getReadLock(baseKey, timeout); + } + + @Override + public void releaseReadLock() { + AWSS3MapStorage.this.releaseReadLock(baseKey); + } + + @Override + public void cleanup() { + } + + @Override + public String getURI() { + return null; + } + + @Override + public void enqueueZoomOutUpdate() { + world.enqueueZoomOutUpdate(this); + } + @Override + public MapStorageTile getZoomOutTile() { + int xx, yy; + int step = 1 << zoom; + if(x >= 0) + xx = x - (x % (2*step)); + else + xx = x + (x % (2*step)); + yy = -y; + if(yy >= 0) + yy = yy - (yy % (2*step)); + else + yy = yy + (yy % (2*step)); + yy = -yy; + return new StorageTile(world, map, xx, yy, zoom+1, var); + } + @Override + public boolean equals(Object o) { + if (o instanceof StorageTile) { + StorageTile st = (StorageTile) o; + return baseKey.equals(st.baseKey); + } + return false; + } + @Override + public int hashCode() { + return baseKey.hashCode(); + } + @Override + public String toString() { + return baseKey; + } + } + + private String bucketname; + private String region; + private String profile_id; + private AmazonS3 s3; + + public AWSS3MapStorage() { + } + + @Override + public boolean init(DynmapCore core) { + if (!super.init(core)) { + return false; + } + if (!core.isInternalWebServerDisabled) { + Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt"); + return false; + } + // Get our settings + bucketname = core.configuration.getString("storage/bucketname", "dynmap"); + region = core.configuration.getString("storage/region", "us-east-1"); + profile_id = core.configuration.getString("storage/aws_profile_id", System.getenv("AWS_PROFILE")); + try { + // Now creste the access client for the S3 service + Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); + s3 = AmazonS3ClientBuilder.standard().withRegion(region) + //.withCredentials(new ProfileCredentialsProvider("profile " + profile_id)) + .build(); + if (s3 == null) { + Log.severe("Error creating S3 access client"); + return false; + } + // Make sure bucket exists and get ACL + AccessControlList bucketACL = s3.getBucketAcl(bucketname); + if (bucketACL == null) { + Log.severe("Error: cannot find or access S3 bucket"); + return false; + } + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + return false; + } + return true; + } + + @Override + public MapStorageTile getTile(DynmapWorld world, MapType map, int x, int y, + int zoom, ImageVariant var) { + return new StorageTile(world, map, x, y, zoom, var); + } + + @Override + public MapStorageTile getTile(DynmapWorld world, String uri) { + String[] suri = uri.split("/"); + if (suri.length < 2) return null; + String mname = suri[0]; // Map URI - might include variant + MapType mt = null; + ImageVariant imgvar = null; + // Find matching map type and image variant + for (int mti = 0; (mt == null) && (mti < world.maps.size()); mti++) { + MapType type = world.maps.get(mti); + ImageVariant[] var = type.getVariants(); + for (int ivi = 0; (imgvar == null) && (ivi < var.length); ivi++) { + if (mname.equals(type.getPrefix() + var[ivi].variantSuffix)) { + mt = type; + imgvar = var[ivi]; + } + } + } + if (mt == null) { // Not found? + return null; + } + // Now, take the last section and parse out coordinates and zoom + String fname = suri[suri.length-1]; + String[] coord = fname.split("[_\\.]"); + if (coord.length < 3) { // 3 or 4 + return null; + } + int zoom = 0; + int x, y; + try { + if (coord[0].charAt(0) == 'z') { + zoom = coord[0].length(); + x = Integer.parseInt(coord[1]); + y = Integer.parseInt(coord[2]); + } + else { + x = Integer.parseInt(coord[0]); + y = Integer.parseInt(coord[1]); + } + return getTile(world, mt, x, y, zoom, imgvar); + } catch (NumberFormatException nfx) { + return null; + } + } + + + private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { + String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + try { + ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); + List objects = result.getObjectSummaries(); + for (S3ObjectSummary os : objects) { + String key = os.getKey(); + key = key.substring(basekey.length()); // Strip off base + // Parse the extension + String ext = null; + int extoff = key.lastIndexOf('.'); + if (extoff >= 0) { + ext = key.substring(extoff+1); + key = key.substring(0, extoff); + } + // If not valid image extension, ignore + ImageEncoding fmt = ImageEncoding.fromExt(ext); + if (fmt == null) { + continue; + } + // See if zoom tile: figure out zoom level + int zoom = 0; + if (key.startsWith("z")) { + while (key.startsWith("z")) { + key = key.substring(1); + zoom++; + } + if (key.startsWith("_")) { + key = key.substring(1); + } + } + // Split remainder to get coords + String[] coord = key.split("_"); + if (coord.length == 2) { // Must be 2 to be a tile + try { + int x = Integer.parseInt(coord[0]); + int y = Integer.parseInt(coord[1]); + // Invoke callback + MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); + if(cb != null) + cb.tileFound(t, fmt); + if(cbBase != null && t.zoom == 0) + cbBase.tileFound(t, fmt); + t.cleanup(); + } catch (NumberFormatException nfx) { + } + } + } + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } + if(cbEnd != null) { + cbEnd.searchEnded(); + } + } + + @Override + public void enumMapTiles(DynmapWorld world, MapType map, MapStorageTileEnumCB cb) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processEnumMapTiles(world, mt, var, cb, null, null); + } + } + } + + @Override + public void enumMapBaseTiles(DynmapWorld world, MapType map, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processEnumMapTiles(world, mt, var, null, cbBase, cbEnd); + } + } + } + + private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { + String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + try { + ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); + List objects = result.getObjectSummaries(); + ArrayList keys = new ArrayList(); + for (S3ObjectSummary os : objects) { + String key = os.getKey(); + keys.add(key); + if (keys.size() >= 100) { + DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); + s3.deleteObjects(dor); + keys.clear(); + } + } + // Any left? + if (keys.size() > 0) { + DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); + s3.deleteObjects(dor); + keys.clear(); + } + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } + } + + @Override + public void purgeMapTiles(DynmapWorld world, MapType map) { + List mtlist; + + if (map != null) { + mtlist = Collections.singletonList(map); + } + else { // Else, add all directories under world directory (for maps) + mtlist = new ArrayList(world.maps); + } + for (MapType mt : mtlist) { + ImageVariant[] vars = mt.getVariants(); + for (ImageVariant var : vars) { + processPurgeMapTiles(world, mt, var); + } + } + } + + @Override + public boolean setPlayerFaceImage(String playername, FaceType facetype, + BufferOutputStream encImage) { + boolean done = false; + String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; + getWriteLock(baseKey); + try { + if (encImage == null) { // Delete? + s3.deleteObject(bucketname, baseKey); + } + else { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType("image/png"); + s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + } + done = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + releaseWriteLock(baseKey); + } + return done; + } + + @Override + public BufferInputStream getPlayerFaceImage(String playername, + FaceType facetype) { + return null; + } + + @Override + public boolean hasPlayerFaceImage(String playername, FaceType facetype) { + String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; + boolean exists = false; + try { + AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + if (rslt != null) + exists = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } + return exists; + } + + @Override + public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { + boolean done = false; + String baseKey = "_markers_/" + markerid + ".png"; + getWriteLock(baseKey); + try { + if (encImage == null) { // Delete? + s3.deleteObject(bucketname, baseKey); + } + else { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType("image/png"); + s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + } + done = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + releaseWriteLock(baseKey); + } + return done; + } + + @Override + public BufferInputStream getMarkerImage(String markerid) { + return null; + } + + @Override + public boolean setMarkerFile(String world, String content) { + boolean done = false; + String baseKey = "_markers_/marker_" + world + ".json"; + getWriteLock(baseKey); + try { + if (content == null) { // Delete? + s3.deleteObject(bucketname, baseKey); + } + else { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType("application/json"); + s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), md); + } + done = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + releaseWriteLock(baseKey); + } + return done; + } + + @Override + public String getMarkerFile(String world) { + return null; + } + + @Override + // For external web server only + public String getMarkersURI(boolean login_enabled) { + return login_enabled?"standalone/markers.php?marker=":"tiles/"; + } + + @Override + // For external web server only + public String getTilesURI(boolean login_enabled) { + return login_enabled?"standalone/tiles.php?tile=":"tiles/"; + } + + @Override + public void addPaths(StringBuilder sb, DynmapCore core) { + String p = core.getTilesFolder().getAbsolutePath(); + if(!p.endsWith("/")) + p += "/"; + sb.append("$tilespath = \'"); + sb.append(WebAuthManager.esc(p)); + sb.append("\';\n"); + sb.append("$markerspath = \'"); + sb.append(WebAuthManager.esc(p)); + sb.append("\';\n"); + + // Need to call base to add webpath + super.addPaths(sb, core); + } + + + @Override + public BufferInputStream getStandaloneFile(String fileid) { + return null; + } + + @Override + public boolean setStandaloneFile(String fileid, BufferOutputStream content) { + + boolean done = false; + String baseKey = "standalone/" + fileid; + getWriteLock(baseKey); + try { + if (content == null) { // Delete? + s3.deleteObject(bucketname, baseKey); + } + else { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType("text/plain"); + s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.buf), md); + } + done = true; + } catch (AmazonServiceException x) { + Log.severe("AWS Exception", x); + } finally { + releaseWriteLock(baseKey); + } + return done; + + } +} diff --git a/bukkit-helper/.project b/bukkit-helper/.project index b372cd57..e86ee638 100644 --- a/bukkit-helper/.project +++ b/bukkit-helper/.project @@ -2,35 +2,32 @@ Dynmap(Spigot-Common) bukkit-helper - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - + org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature org.eclipse.buildship.core.gradleprojectnature + + + org.eclipse.jdt.core.javabuilder + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + org.eclipse.m2e.core.maven2Builder + + + + 1 - 30 + org.eclipse.core.resources.regexFilterMatcher node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ diff --git a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs index 36274c6d..c8f0f61d 100644 --- a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs +++ b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs @@ -1,5 +1,5 @@ # -#Sun Feb 13 13:56:50 CST 2022 +#Thu Feb 17 18:10:31 CST 2022 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index 9712bf72..d70a9f51 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -40,6 +40,14 @@ storage: #password: dynmap #prefix: "" #flags: "?allowReconnect=true&autoReconnect=true" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: dynmap + #region: us-east-1 + #aws_access_key_id: + #aws_secret_access_key: + components: - class: org.dynmap.ClientConfigurationComponent From 45bc02cf3a44cf0e6af60dd2904c9a0bf7d42fef Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Fri, 18 Feb 2022 00:40:55 -0600 Subject: [PATCH 2/4] Switch to sdk v2 --- DynmapCore/build.gradle | 37 ++-- .../storage/aws_s3/AWSS3MapStorage.java | 164 +++++++++--------- .../.settings/org.eclipse.jdt.core.prefs | 2 +- spigot/src/main/resources/configuration.txt | 2 - 4 files changed, 105 insertions(+), 100 deletions(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index cdd7e06c..f6b85116 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,12 +17,17 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' - implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000') - implementation 'com.amazonaws:aws-java-sdk-core' - implementation 'com.amazonaws:aws-java-sdk-s3' - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'software.amazon.awssdk:s3:2.17.132' + implementation 'software.amazon.awssdk:aws-core:2.17.132' + implementation 'software.amazon.awssdk:sdk-core:2.17.132' + implementation 'software.amazon.awssdk:utils:2.17.132' + implementation 'software.amazon.awssdk:http-client-spi:2.17.132' + implementation 'software.amazon.awssdk:profiles:2.17.132' + implementation 'software.amazon.awssdk:regions:2.17.132' + implementation 'software.amazon.awssdk:auth:2.17.132' + implementation 'software.amazon.awssdk:metrics-spi:2.17.132' + implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132' + implementation 'software.amazon.awssdk:protocol-core:2.17.132' } processResources { @@ -57,12 +62,17 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) - include(dependency('com.amazonaws:aws-java-sdk-bom:')) - include(dependency('com.amazonaws:aws-java-sdk-core:')) - include(dependency('com.amazonaws:aws-java-sdk-s3:')) - include(dependency('com.fasterxml.jackson.core:jackson-core:')) - include(dependency('com.fasterxml.jackson.core:jackson-databind:')) - include(dependency('com.fasterxml.jackson.core:jackson-annotations:')) + include(dependency('software.amazon.awssdk:s3:')) + include(dependency('software.amazon.awssdk:aws-core:')) + include(dependency('software.amazon.awssdk:sdk-core:')) + include(dependency('software.amazon.awssdk:utils:')) + include(dependency('software.amazon.awssdk:http-client-spi:')) + include(dependency('software.amazon.awssdk:profiles:')) + include(dependency('software.amazon.awssdk:regions:')) + include(dependency('software.amazon.awssdk:auth:')) + include(dependency('software.amazon.awssdk:metrics-spi:')) + include(dependency('software.amazon.awssdk:aws-xml-protocol:')) + include(dependency('software.amazon.awssdk:protocol-core:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -73,8 +83,7 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') - relocate('com.amazonaws', 'org.dynmap.com.amazonaws') - relocate('com.fasterxml', 'org.dynmap.com.fasterxml') + relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk') destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 055fa52b..38ffeafb 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -1,13 +1,6 @@ package org.dynmap.storage.aws_s3; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -28,19 +21,25 @@ import org.dynmap.storage.MapStorageTileSearchEndCB; import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferOutputStream; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.internal.StaticCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; +import software.amazon.awssdk.services.s3.model.GetBucketAclResponse; +import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; public class AWSS3MapStorage extends MapStorage { public class StorageTile extends MapStorageTile { @@ -62,10 +61,11 @@ public class AWSS3MapStorage extends MapStorage { public boolean exists() { boolean exists = false; try { - AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectAclResponse rslt = s3.getObjectAcl(req); if (rslt != null) exists = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } return exists; @@ -80,23 +80,20 @@ public class AWSS3MapStorage extends MapStorage { public TileRead read() { AWSS3MapStorage.this.getWriteLock(baseKey); try { - S3Object obj = s3.getObject(bucketname, baseKey); + GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + ResponseBytes obj = s3.getObjectAsBytes(req); if (obj != null) { - ObjectMetadata md = obj.getObjectMetadata(); + GetObjectResponse rsp = obj.response(); TileRead tr = new TileRead(); - byte[] buf = new byte[(int) md.getContentLength()]; - InputStream fis = obj.getObjectContent(); - fis.read(buf, 0, buf.length); // Read whole thing + byte[] buf = obj.asByteArray(); tr.image = new BufferInputStream(buf); - tr.format = ImageEncoding.fromContentType(md.getContentType()); - tr.hashCode = md.getContentMD5().hashCode(); - tr.lastModified = md.getLastModified().getTime(); + tr.format = ImageEncoding.fromContentType(rsp.contentType()); + tr.hashCode = rsp.eTag().hashCode(); + tr.lastModified = rsp.lastModified().toEpochMilli(); return tr; } - } catch (IOException x) { - Log.severe("AWS Exception", x); - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { AWSS3MapStorage.this.releaseWriteLock(baseKey); @@ -110,15 +107,15 @@ public class AWSS3MapStorage extends MapStorage { AWSS3MapStorage.this.getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(req); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType(map.getImageFormat().getEncoding().getContentType()); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { AWSS3MapStorage.this.releaseWriteLock(baseKey); @@ -200,7 +197,7 @@ public class AWSS3MapStorage extends MapStorage { private String bucketname; private String region; private String profile_id; - private AmazonS3 s3; + private S3Client s3; public AWSS3MapStorage() { } @@ -217,24 +214,22 @@ public class AWSS3MapStorage extends MapStorage { // Get our settings bucketname = core.configuration.getString("storage/bucketname", "dynmap"); region = core.configuration.getString("storage/region", "us-east-1"); - profile_id = core.configuration.getString("storage/aws_profile_id", System.getenv("AWS_PROFILE")); try { // Now creste the access client for the S3 service Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); - s3 = AmazonS3ClientBuilder.standard().withRegion(region) - //.withCredentials(new ProfileCredentialsProvider("profile " + profile_id)) - .build(); + s3 = S3Client.builder().region(Region.of(region)).build(); if (s3 == null) { Log.severe("Error creating S3 access client"); return false; } // Make sure bucket exists and get ACL - AccessControlList bucketACL = s3.getBucketAcl(bucketname); + GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build(); + GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr); if (bucketACL == null) { Log.severe("Error: cannot find or access S3 bucket"); return false; } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); return false; } @@ -296,10 +291,11 @@ public class AWSS3MapStorage extends MapStorage { private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; try { - ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); - List objects = result.getObjectSummaries(); - for (S3ObjectSummary os : objects) { - String key = os.getKey(); + ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.contents(); + for (S3Object os : objects) { + String key = os.key(); key = key.substring(basekey.length()); // Strip off base // Parse the extension String ext = null; @@ -341,7 +337,7 @@ public class AWSS3MapStorage extends MapStorage { } } } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } if(cbEnd != null) { @@ -388,25 +384,26 @@ public class AWSS3MapStorage extends MapStorage { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; try { - ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); - List objects = result.getObjectSummaries(); - ArrayList keys = new ArrayList(); - for (S3ObjectSummary os : objects) { - String key = os.getKey(); - keys.add(key); + ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.contents(); + ArrayList keys = new ArrayList(); + for (S3Object os : objects) { + String key = os.key(); + keys.add(ObjectIdentifier.builder().key(key).build()); if (keys.size() >= 100) { - DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); - s3.deleteObjects(dor); + DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); + s3.deleteObjects(delreq); keys.clear(); } } // Any left? if (keys.size() > 0) { - DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); - s3.deleteObjects(dor); + DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); + s3.deleteObjects(delreq); keys.clear(); } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } } @@ -437,15 +434,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("image/png"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -464,10 +461,11 @@ public class AWSS3MapStorage extends MapStorage { String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; boolean exists = false; try { - AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectAclResponse rslt = s3.getObjectAcl(req); if (rslt != null) exists = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } return exists; @@ -480,15 +478,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("image/png"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -508,15 +506,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (content == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("application/json"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build(); + s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -571,15 +569,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (content == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("text/plain"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build(); + s3.putObject(req, RequestBody.fromBytes(content.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); diff --git a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs index c8f0f61d..8f00b17b 100644 --- a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs +++ b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs @@ -1,5 +1,5 @@ # -#Thu Feb 17 18:10:31 CST 2022 +#Thu Feb 17 22:40:49 CST 2022 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index d70a9f51..7968a104 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -45,8 +45,6 @@ storage: #type: aws_s3 #bucketname: dynmap #region: us-east-1 - #aws_access_key_id: - #aws_secret_access_key: components: From ea97296684b0263d7549e50adc220ff6b613e87d Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Sun, 20 Feb 2022 20:58:21 -0600 Subject: [PATCH 3/4] Get AWS S3 storage based web working --- DynmapCore/build.gradle | 41 +- .../src/main/java/org/dynmap/DynmapCore.java | 80 +++- .../dynmap/JsonFileClientUpdateComponent.java | 37 +- .../java/org/dynmap/storage/MapStorage.java | 15 + .../storage/aws_s3/AWSS3MapStorage.java | 430 +++++++++++------- bukkit-helper/.project | 37 +- 6 files changed, 409 insertions(+), 231 deletions(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index f6b85116..d080cbab 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,17 +17,15 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' - implementation 'software.amazon.awssdk:s3:2.17.132' - implementation 'software.amazon.awssdk:aws-core:2.17.132' - implementation 'software.amazon.awssdk:sdk-core:2.17.132' - implementation 'software.amazon.awssdk:utils:2.17.132' - implementation 'software.amazon.awssdk:http-client-spi:2.17.132' - implementation 'software.amazon.awssdk:profiles:2.17.132' - implementation 'software.amazon.awssdk:regions:2.17.132' - implementation 'software.amazon.awssdk:auth:2.17.132' - implementation 'software.amazon.awssdk:metrics-spi:2.17.132' - implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132' - implementation 'software.amazon.awssdk:protocol-core:2.17.132' + implementation 'io.github.linktosriram:s3-lite-core:0.2.0' + implementation 'io.github.linktosriram:s3-lite-api:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0' + implementation 'io.github.linktosriram:s3-lite-http-client-apache:0.2.0' + implementation 'io.github.linktosriram:s3-lite-util:0.2.0' + implementation 'org.apache.httpcomponents:httpclient:4.5.9' + implementation 'javax.xml.bind:jaxb-api:2.3.1' + implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1' } processResources { @@ -62,17 +60,13 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) - include(dependency('software.amazon.awssdk:s3:')) - include(dependency('software.amazon.awssdk:aws-core:')) - include(dependency('software.amazon.awssdk:sdk-core:')) - include(dependency('software.amazon.awssdk:utils:')) - include(dependency('software.amazon.awssdk:http-client-spi:')) - include(dependency('software.amazon.awssdk:profiles:')) - include(dependency('software.amazon.awssdk:regions:')) - include(dependency('software.amazon.awssdk:auth:')) - include(dependency('software.amazon.awssdk:metrics-spi:')) - include(dependency('software.amazon.awssdk:aws-xml-protocol:')) - include(dependency('software.amazon.awssdk:protocol-core:')) + include(dependency('io.github.linktosriram:s3-lite-core:')) + include(dependency('io.github.linktosriram:s3-lite-api:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-spi:')) + include(dependency('io.github.linktosriram:s3-lite-http-client-apache:')) + include(dependency('io.github.linktosriram:s3-lite-util:')) + include(dependency('org.apache.httpcomponents:httpclient:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -83,7 +77,8 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') - relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk') + relocate('io.github.linktosriram.s3lite', 'org.dynmap.s3lite') + destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java index e3984c44..316111bf 100644 --- a/DynmapCore/src/main/java/org/dynmap/DynmapCore.java +++ b/DynmapCore/src/main/java/org/dynmap/DynmapCore.java @@ -1,6 +1,9 @@ package org.dynmap; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; @@ -61,6 +64,7 @@ import org.dynmap.storage.mariadb.MariaDBMapStorage; import org.dynmap.storage.sqllte.SQLiteMapStorage; import org.dynmap.storage.postgresql.PostgreSQLMapStorage; import org.dynmap.utils.BlockStep; +import org.dynmap.utils.BufferOutputStream; import org.dynmap.utils.ImageIOManager; import org.dynmap.web.BanIPFilter; import org.dynmap.web.CustomHeaderFilter; @@ -490,7 +494,10 @@ public class DynmapCore implements DynmapCommonAPI { authmgr = new WebAuthManager(this); defaultStorage.setLoginEnabled(this); } - + // If storage serves web files, extract and publsh them + if (defaultStorage.needsStaticWebFiles()) { + updateStaticWebToStorage(); + } /* Load control for leaf transparency (spout lighting bug workaround) */ transparentLeaves = configuration.getBoolean("transparent-leaves", true); @@ -2821,6 +2828,63 @@ public class DynmapCore implements DynmapCommonAPI { } return dir.delete(); } + private void updateStaticWebToStorage() { + if(jarfile == null) return; + // If doing update and web path update is disabled, send warning + if (!this.updatewebpathfiles) { + return; + } + Log.info("Publishing web files to storage"); + /* Open JAR as ZIP */ + ZipFile zf = null; + InputStream ins = null; + byte[] buf = new byte[2048]; + String n = null; + try { + zf = new ZipFile(jarfile); + Enumeration e = zf.entries(); + while (e.hasMoreElements()) { + ZipEntry ze = e.nextElement(); + n = ze.getName(); + if (!n.startsWith("extracted/web/")) { + continue; + } + n = n.substring("extracted/web/".length()); + // If file is going to web path, redirect it to the configured web + if (ze.isDirectory()) { + continue; + } + try { + ins = zf.getInputStream(ze); + BufferOutputStream buffer = new BufferOutputStream(); + int len; + while ((len = ins.read(buf)) >= 0) { + buffer.write(buf, 0, len); + } + defaultStorage.setStaticWebFile(n, buffer); + } catch(IOException io) { + Log.severe("Error updating file in storage - " + n, io); + } finally { + if (ins != null) { + ins.close(); + ins = null; + } + } + } + } catch (IOException iox) { + Log.severe("Error extracting file - " + n); + } finally { + if (ins != null) { + try { ins.close(); } catch (IOException iox) {} + ins = null; + } + if (zf != null) { + try { zf.close(); } catch (IOException iox) {} + zf = null; + } + } + } + private void updateExtractedFiles() { if(jarfile == null) return; File df = this.getDataFolder(); @@ -2922,13 +2986,13 @@ public class DynmapCore implements DynmapCommonAPI { } else { try { - f.getParentFile().mkdirs(); - fos = new FileOutputStream(f); - ins = zf.getInputStream(ze); - int len; - while ((len = ins.read(buf)) >= 0) { - fos.write(buf, 0, len); - } + f.getParentFile().mkdirs(); + fos = new FileOutputStream(f); + ins = zf.getInputStream(ze); + int len; + while ((len = ins.read(buf)) >= 0) { + fos.write(buf, 0, len); + } } catch(IOException io) { Log.severe("Error updating file - " + f.getPath(), io); } finally { diff --git a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java index 0ca7f1f9..0135a5e9 100644 --- a/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java +++ b/DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java @@ -272,21 +272,28 @@ public class JsonFileClientUpdateComponent extends ClientUpdateComponent { byte[] outputBytes = sb.toString().getBytes(cs_utf8); MapManager.scheduleDelayedJob(new Runnable() { public void run() { - File f = new File(baseStandaloneDir, "config.js"); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(f); - fos.write(outputBytes); - } catch (IOException iox) { - Log.severe("Exception while writing " + f.getPath(), iox); - } finally { - if(fos != null) { - try { - fos.close(); - } catch (IOException x) {} - fos = null; - } - } + if (core.getDefaultMapStorage().needsStaticWebFiles()) { + BufferOutputStream os = new BufferOutputStream(); + os.write(outputBytes); + core.getDefaultMapStorage().setStaticWebFile("standalone/config.js", os); + } + else { + File f = new File(baseStandaloneDir, "config.js"); + FileOutputStream fos = null; + try { + fos = new FileOutputStream(f); + fos.write(outputBytes); + } catch (IOException iox) { + Log.severe("Exception while writing " + f.getPath(), iox); + } finally { + if(fos != null) { + try { + fos.close(); + } catch (IOException x) {} + fos = null; + } + } + } } }, 0); } diff --git a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java index 8106b29a..21f7f310 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java @@ -1,5 +1,6 @@ package org.dynmap.storage; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -459,6 +460,20 @@ public abstract class MapStorage { } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return false; + } + /** + * Set static web file content + * @param fileid - file path + * @param buffer - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream buffer) { + return false; + } + public void logSQLException(String opmsg, SQLException x) { Log.severe("SQLException: " + opmsg); Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState()); diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 38ffeafb..1ce8a8df 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -1,9 +1,14 @@ package org.dynmap.storage.aws_s3; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import org.dynmap.DynmapCore; import org.dynmap.DynmapWorld; @@ -21,33 +26,32 @@ import org.dynmap.storage.MapStorageTileSearchEndCB; import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferOutputStream; -import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; -import software.amazon.awssdk.services.s3.model.GetBucketAclResponse; -import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; -import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Object; +import io.github.linktosriram.s3lite.api.client.S3Client; +import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException; +import io.github.linktosriram.s3lite.api.exception.S3Exception; +import io.github.linktosriram.s3lite.api.region.Region; +import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest; +import io.github.linktosriram.s3lite.api.request.GetObjectRequest; +import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request; +import io.github.linktosriram.s3lite.api.request.PutObjectRequest; +import io.github.linktosriram.s3lite.api.response.GetObjectResponse; +import io.github.linktosriram.s3lite.api.response.ListObjectsV2Response; +import io.github.linktosriram.s3lite.api.response.ResponseBytes; +import io.github.linktosriram.s3lite.api.response.S3Object; +import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials; +import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder; +import io.github.linktosriram.s3lite.http.apache.ApacheSdkHttpClient; +import io.github.linktosriram.s3lite.http.spi.request.RequestBody; public class AWSS3MapStorage extends MapStorage { public class StorageTile extends MapStorageTile { private final String baseKey; + private final String uri; StorageTile(DynmapWorld world, MapType map, int x, int y, int zoom, ImageVariant var) { super(world, map, x, y, zoom, var); + String baseURI; if (zoom > 0) { baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y; @@ -55,18 +59,21 @@ public class AWSS3MapStorage extends MapStorage { else { baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; } - baseKey = "tiles/" + world.getName() + "/" + baseURI + "." + map.getImageFormat().getFileExt(); + uri = baseURI + "." + map.getImageFormat().getFileExt(); + baseKey = "tiles/" + world.getName() + "/" + uri; } @Override public boolean exists() { boolean exists = false; try { - GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); - GetObjectAclResponse rslt = s3.getObjectAcl(req); - if (rslt != null) + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) exists = true; - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } } return exists; } @@ -78,25 +85,24 @@ public class AWSS3MapStorage extends MapStorage { @Override public TileRead read() { - AWSS3MapStorage.this.getWriteLock(baseKey); try { - GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectRequest req = GetObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); ResponseBytes obj = s3.getObjectAsBytes(req); if (obj != null) { - GetObjectResponse rsp = obj.response(); + GetObjectResponse rsp = obj.getResponse(); TileRead tr = new TileRead(); - byte[] buf = obj.asByteArray(); + byte[] buf = obj.getBytes(); tr.image = new BufferInputStream(buf); - tr.format = ImageEncoding.fromContentType(rsp.contentType()); - tr.hashCode = rsp.eTag().hashCode(); - tr.lastModified = rsp.lastModified().toEpochMilli(); + tr.format = ImageEncoding.fromContentType(rsp.getContentType()); + tr.hashCode = rsp.geteTag().hashCode(); + tr.lastModified = rsp.getLastModified().toEpochMilli(); return tr; } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); - } finally { - AWSS3MapStorage.this.releaseWriteLock(baseKey); + } catch (NoSuchKeyException nskx) { + return null; // Nominal case if it doesn't exist + } catch (S3Exception x) { + Log.severe("AWS Exception", x); } return null; } @@ -104,21 +110,18 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean write(long hash, BufferOutputStream encImage, long timestamp) { boolean done = false; - AWSS3MapStorage.this.getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(req); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - AWSS3MapStorage.this.releaseWriteLock(baseKey); } // Signal update for zoom out if (zoom == 0) { @@ -129,22 +132,20 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean getWriteLock() { - return AWSS3MapStorage.this.getWriteLock(baseKey); + return true; } @Override public void releaseWriteLock() { - AWSS3MapStorage.this.releaseWriteLock(baseKey); } @Override public boolean getReadLock(long timeout) { - return AWSS3MapStorage.this.getReadLock(baseKey, timeout); + return true; } @Override public void releaseReadLock() { - AWSS3MapStorage.this.releaseReadLock(baseKey); } @Override @@ -153,7 +154,7 @@ public class AWSS3MapStorage extends MapStorage { @Override public String getURI() { - return null; + return baseKey; } @Override @@ -196,7 +197,8 @@ public class AWSS3MapStorage extends MapStorage { private String bucketname; private String region; - private String profile_id; + private String access_key_id; + private String secret_access_key; private S3Client s3; public AWSS3MapStorage() { @@ -211,27 +213,45 @@ public class AWSS3MapStorage extends MapStorage { Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt"); return false; } + if (core.isLoginSupportEnabled()) { + Log.severe("AWS S3 storage is not supported option with loegin support enabled: set login-enabled: false in configuration.txt"); + return false; + } // Get our settings bucketname = core.configuration.getString("storage/bucketname", "dynmap"); region = core.configuration.getString("storage/region", "us-east-1"); + access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID")); + secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY")); + // Now creste the access client for the S3 service + Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region); + s3 = new DefaultS3ClientBuilder() + .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key)) + .region(Region.fromString(region)) + .httpClient(ApacheSdkHttpClient.defaultClient()) + .build(); + if (s3 == null) { + Log.severe("Error creating S3 access client"); + return false; + } + // Make sure bucket exists (do list) + ListObjectsV2Request listreq = ListObjectsV2Request.builder() + .bucketName(bucketname) + .maxKeys(1) + .build(); try { - // Now creste the access client for the S3 service - Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); - s3 = S3Client.builder().region(Region.of(region)).build(); - if (s3 == null) { - Log.severe("Error creating S3 access client"); - return false; - } - // Make sure bucket exists and get ACL - GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build(); - GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr); - if (bucketACL == null) { + ListObjectsV2Response rslt = s3.listObjectsV2(listreq); + if (rslt == null) { Log.severe("Error: cannot find or access S3 bucket"); return false; } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); - return false; + List content = rslt.getContents(); + Log.info("content=" + content.size()); + } catch (S3Exception s3x) { + if (!s3x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", s3x); + Log.severe("req=" + listreq); + return false; + } } return true; } @@ -288,57 +308,71 @@ public class AWSS3MapStorage extends MapStorage { } - private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { + private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, + MapStorageTileSearchEndCB cbEnd) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build(); + boolean done = false; try { - ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); - ListObjectsV2Response result = s3.listObjectsV2(req); - List objects = result.contents(); - for (S3Object os : objects) { - String key = os.key(); - key = key.substring(basekey.length()); // Strip off base - // Parse the extension - String ext = null; - int extoff = key.lastIndexOf('.'); - if (extoff >= 0) { - ext = key.substring(extoff+1); - key = key.substring(0, extoff); - } - // If not valid image extension, ignore - ImageEncoding fmt = ImageEncoding.fromExt(ext); - if (fmt == null) { - continue; - } - // See if zoom tile: figure out zoom level - int zoom = 0; - if (key.startsWith("z")) { - while (key.startsWith("z")) { - key = key.substring(1); - zoom++; - } - if (key.startsWith("_")) { - key = key.substring(1); - } - } - // Split remainder to get coords - String[] coord = key.split("_"); - if (coord.length == 2) { // Must be 2 to be a tile - try { - int x = Integer.parseInt(coord[0]); - int y = Integer.parseInt(coord[1]); - // Invoke callback - MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); - if(cb != null) - cb.tileFound(t, fmt); - if(cbBase != null && t.zoom == 0) - cbBase.tileFound(t, fmt); - t.cleanup(); - } catch (NumberFormatException nfx) { - } - } - } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + key = key.substring(basekey.length()); // Strip off base + // Parse the extension + String ext = null; + int extoff = key.lastIndexOf('.'); + if (extoff >= 0) { + ext = key.substring(extoff+1); + key = key.substring(0, extoff); + } + // If not valid image extension, ignore + ImageEncoding fmt = ImageEncoding.fromExt(ext); + if (fmt == null) { + continue; + } + // See if zoom tile: figure out zoom level + int zoom = 0; + if (key.startsWith("z")) { + while (key.startsWith("z")) { + key = key.substring(1); + zoom++; + } + if (key.startsWith("_")) { + key = key.substring(1); + } + } + // Split remainder to get coords + String[] coord = key.split("_"); + if (coord.length == 2) { // Must be 2 to be a tile + try { + int x = Integer.parseInt(coord[0]); + int y = Integer.parseInt(coord[1]); + // Invoke callback + MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); + if(cb != null) + cb.tileFound(t, fmt); + if(cbBase != null && t.zoom == 0) + cbBase.tileFound(t, fmt); + t.cleanup(); + } catch (NumberFormatException nfx) { + } + } + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } + } + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } } if(cbEnd != null) { cbEnd.searchEnded(); @@ -383,28 +417,30 @@ public class AWSS3MapStorage extends MapStorage { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build(); try { - ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); - ListObjectsV2Response result = s3.listObjectsV2(req); - List objects = result.contents(); - ArrayList keys = new ArrayList(); - for (S3Object os : objects) { - String key = os.key(); - keys.add(ObjectIdentifier.builder().key(key).build()); - if (keys.size() >= 100) { - DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); - s3.deleteObjects(delreq); - keys.clear(); - } + boolean done = false; + while (!done) { + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.getContents(); + for (S3Object os : objects) { + String key = os.getKey(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(key).build(); + s3.deleteObject(delreq); + } + if (result.isTruncated()) { // If more, build continuiation request + req = ListObjectsV2Request.builder().bucketName(bucketname) + .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build(); + } + else { // Else, we're done + done = true; + } } - // Any left? - if (keys.size() > 0) { - DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); - s3.deleteObjects(delreq); - keys.clear(); - } - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + Log.severe("req=" + req); + } } } @@ -431,21 +467,18 @@ public class AWSS3MapStorage extends MapStorage { BufferOutputStream encImage) { boolean done = false; String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; - getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -461,12 +494,14 @@ public class AWSS3MapStorage extends MapStorage { String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; boolean exists = false; try { - GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); - GetObjectAclResponse rslt = s3.getObjectAcl(req); - if (rslt != null) + ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); + ListObjectsV2Response rslt = s3.listObjectsV2(req); + if ((rslt != null) && (rslt.getKeyCount() > 0)) exists = true; - } catch (AwsServiceException x) { - Log.severe("AWS Exception", x); + } catch (S3Exception x) { + if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... + Log.severe("AWS Exception", x); + } } return exists; } @@ -475,21 +510,18 @@ public class AWSS3MapStorage extends MapStorage { public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { boolean done = false; String baseKey = "_markers_/" + markerid + ".png"; - getWriteLock(baseKey); try { if (encImage == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf)); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -503,21 +535,18 @@ public class AWSS3MapStorage extends MapStorage { public boolean setMarkerFile(String world, String content) { boolean done = false; String baseKey = "_markers_/marker_" + world + ".json"; - getWriteLock(baseKey); try { if (content == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build(); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("application/json").build(); s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; } @@ -530,15 +559,32 @@ public class AWSS3MapStorage extends MapStorage { @Override // For external web server only public String getMarkersURI(boolean login_enabled) { - return login_enabled?"standalone/markers.php?marker=":"tiles/"; + return "tiles/"; } @Override // For external web server only public String getTilesURI(boolean login_enabled) { - return login_enabled?"standalone/tiles.php?tile=":"tiles/"; + return "tiles/"; } + /** + * URI to use for loading configuration JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getConfigurationJSONURI(boolean login_enabled) { + return "standalone/dynmap_config.json"; + } + /** + * URI to use for loading update JSON files (for external web server only) + * @param login_enabled - selects based on login security enabled + * @return URI + */ + public String getUpdateJSONURI(boolean login_enabled) { + return "standalone/dynmap_{world}.json"; + } + @Override public void addPaths(StringBuilder sb, DynmapCore core) { String p = core.getTilesFolder().getAbsolutePath(); @@ -561,28 +607,76 @@ public class AWSS3MapStorage extends MapStorage { return null; } + + // Cache to avoid rewriting same standalong file repeatedly + private ConcurrentHashMap standalone_cache = new ConcurrentHashMap(); + @Override public boolean setStandaloneFile(String fileid, BufferOutputStream content) { + return setStaticWebFile("standalone/" + fileid, content); + } + // Test if storage needs static web files + public boolean needsStaticWebFiles() { + return true; + } + /** + * Set static web file content + * @param fileid - file path + * @param content - content for file + * @return true if successful + */ + public boolean setStaticWebFile(String fileid, BufferOutputStream content) { boolean done = false; - String baseKey = "standalone/" + fileid; - getWriteLock(baseKey); try { + byte[] cacheval = standalone_cache.get(fileid); + if (content == null) { // Delete? - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached? + return true; + } + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(fileid).build(); s3.deleteObject(delreq); + standalone_cache.put(fileid, new byte[0]); // Mark in cache } else { - PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build(); - s3.putObject(req, RequestBody.fromBytes(content.buf)); + byte[] digest = content.buf; + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(content.buf); + digest = md.digest(); + } catch (NoSuchAlgorithmException nsax) { + + } + // If cached and same, just return + if (Arrays.equals(digest, cacheval)) { + return true; + } + String ct = "text/plain"; + if (fileid.endsWith(".json")) { + ct = "application/json"; + } + else if (fileid.endsWith(".php")) { + ct = "application/x-httpd-php"; + } + else if (fileid.endsWith(".html")) { + ct = "text/html"; + } + else if (fileid.endsWith(".css")) { + ct = "text/css"; + } + else if (fileid.endsWith(".js")) { + ct = "application/x-javascript"; + } + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(fileid).contentType(ct).build(); + s3.putObject(req, RequestBody.fromBytes(content.buf, content.len)); + standalone_cache.put(fileid, digest); } done = true; - } catch (AwsServiceException x) { + } catch (S3Exception x) { Log.severe("AWS Exception", x); - } finally { - releaseWriteLock(baseKey); } return done; - } + } diff --git a/bukkit-helper/.project b/bukkit-helper/.project index e86ee638..b372cd57 100644 --- a/bukkit-helper/.project +++ b/bukkit-helper/.project @@ -2,32 +2,35 @@ Dynmap(Spigot-Common) bukkit-helper - + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature org.eclipse.buildship.core.gradleprojectnature - - - org.eclipse.jdt.core.javabuilder - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - org.eclipse.m2e.core.maven2Builder - - - - 1 + 30 - org.eclipse.core.resources.regexFilterMatcher node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ From ef045da32fc7ff4bcbfe1e4141fc3039cbe53fdb Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Sun, 20 Feb 2022 21:38:50 -0600 Subject: [PATCH 4/4] Add prefix path support for AWS S3 website --- .../storage/aws_s3/AWSS3MapStorage.java | 34 +++++++++++-------- .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ .../src/main/resources/configuration.txt | 8 +++++ spigot/src/main/resources/configuration.txt | 5 +-- 13 files changed, 110 insertions(+), 17 deletions(-) diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 1ce8a8df..cf068f05 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -60,7 +60,7 @@ public class AWSS3MapStorage extends MapStorage { baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; } uri = baseURI + "." + map.getImageFormat().getFileExt(); - baseKey = "tiles/" + world.getName() + "/" + uri; + baseKey = AWSS3MapStorage.this.prefix + "tiles/" + world.getName() + "/" + uri; } @Override public boolean exists() { @@ -200,6 +200,7 @@ public class AWSS3MapStorage extends MapStorage { private String access_key_id; private String secret_access_key; private S3Client s3; + private String prefix; public AWSS3MapStorage() { } @@ -222,6 +223,10 @@ public class AWSS3MapStorage extends MapStorage { region = core.configuration.getString("storage/region", "us-east-1"); access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID")); secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY")); + prefix = core.configuration.getString("storage/prefix", ""); + if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) { + prefix += '/'; + } // Now creste the access client for the S3 service Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region); s3 = new DefaultS3ClientBuilder() @@ -237,6 +242,7 @@ public class AWSS3MapStorage extends MapStorage { ListObjectsV2Request listreq = ListObjectsV2Request.builder() .bucketName(bucketname) .maxKeys(1) + .prefix(prefix) .build(); try { ListObjectsV2Response rslt = s3.listObjectsV2(listreq); @@ -245,13 +251,10 @@ public class AWSS3MapStorage extends MapStorage { return false; } List content = rslt.getContents(); - Log.info("content=" + content.size()); } catch (S3Exception s3x) { - if (!s3x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match.... - Log.severe("AWS Exception", s3x); - Log.severe("req=" + listreq); - return false; - } + Log.severe("AWS Exception", s3x); + Log.severe("req=" + listreq); + return false; } return true; } @@ -310,7 +313,7 @@ public class AWSS3MapStorage extends MapStorage { private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { - String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + String basekey = prefix + "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build(); boolean done = false; try { @@ -416,7 +419,7 @@ public class AWSS3MapStorage extends MapStorage { } private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { - String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; + String basekey = prefix + "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build(); try { boolean done = false; @@ -466,7 +469,7 @@ public class AWSS3MapStorage extends MapStorage { public boolean setPlayerFaceImage(String playername, FaceType facetype, BufferOutputStream encImage) { boolean done = false; - String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; + String baseKey = prefix + "faces/" + facetype.id + "/" + playername + ".png"; try { if (encImage == null) { // Delete? DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); @@ -491,7 +494,7 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean hasPlayerFaceImage(String playername, FaceType facetype) { - String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; + String baseKey = prefix + "faces/" + facetype.id + "/" + playername + ".png"; boolean exists = false; try { ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build(); @@ -509,7 +512,7 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { boolean done = false; - String baseKey = "_markers_/" + markerid + ".png"; + String baseKey = prefix + "tiles/_markers_/" + markerid + ".png"; try { if (encImage == null) { // Delete? DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); @@ -534,7 +537,7 @@ public class AWSS3MapStorage extends MapStorage { @Override public boolean setMarkerFile(String world, String content) { boolean done = false; - String baseKey = "_markers_/marker_" + world + ".json"; + String baseKey = prefix + "tiles/_markers_/marker_" + world + ".json"; try { if (content == null) { // Delete? DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); @@ -628,6 +631,7 @@ public class AWSS3MapStorage extends MapStorage { public boolean setStaticWebFile(String fileid, BufferOutputStream content) { boolean done = false; + String baseKey = prefix + fileid; try { byte[] cacheval = standalone_cache.get(fileid); @@ -635,7 +639,7 @@ public class AWSS3MapStorage extends MapStorage { if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached? return true; } - DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(fileid).build(); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build(); s3.deleteObject(delreq); standalone_cache.put(fileid, new byte[0]); // Mark in cache } @@ -668,7 +672,7 @@ public class AWSS3MapStorage extends MapStorage { else if (fileid.endsWith(".js")) { ct = "application/x-javascript"; } - PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(fileid).contentType(ct).build(); + PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build(); s3.putObject(req, RequestBody.fromBytes(content.buf, content.len)); standalone_cache.put(fileid, digest); } diff --git a/fabric-1.14.4/src/main/resources/configuration.txt b/fabric-1.14.4/src/main/resources/configuration.txt index 49bc0515..a618dd32 100644 --- a/fabric-1.14.4/src/main/resources/configuration.txt +++ b/fabric-1.14.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.15.2/src/main/resources/configuration.txt b/fabric-1.15.2/src/main/resources/configuration.txt index 349a91b2..60fd3bed 100644 --- a/fabric-1.15.2/src/main/resources/configuration.txt +++ b/fabric-1.15.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.16.4/src/main/resources/configuration.txt b/fabric-1.16.4/src/main/resources/configuration.txt index 3814101c..83a3da83 100644 --- a/fabric-1.16.4/src/main/resources/configuration.txt +++ b/fabric-1.16.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.17.1/src/main/resources/configuration.txt b/fabric-1.17.1/src/main/resources/configuration.txt index 3814101c..83a3da83 100644 --- a/fabric-1.17.1/src/main/resources/configuration.txt +++ b/fabric-1.17.1/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.18/src/main/resources/configuration.txt b/fabric-1.18/src/main/resources/configuration.txt index 6facc34b..61de127f 100644 --- a/fabric-1.18/src/main/resources/configuration.txt +++ b/fabric-1.18/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.13.2/src/main/resources/configuration.txt b/forge-1.13.2/src/main/resources/configuration.txt index 29a2f80d..dad28957 100644 --- a/forge-1.13.2/src/main/resources/configuration.txt +++ b/forge-1.13.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.14.4/src/main/resources/configuration.txt b/forge-1.14.4/src/main/resources/configuration.txt index b720ffb3..5a76b049 100644 --- a/forge-1.14.4/src/main/resources/configuration.txt +++ b/forge-1.14.4/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.15.2/src/main/resources/configuration.txt b/forge-1.15.2/src/main/resources/configuration.txt index b720ffb3..5a76b049 100644 --- a/forge-1.15.2/src/main/resources/configuration.txt +++ b/forge-1.15.2/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.16.5/src/main/resources/configuration.txt b/forge-1.16.5/src/main/resources/configuration.txt index 05cc179c..09180111 100644 --- a/forge-1.16.5/src/main/resources/configuration.txt +++ b/forge-1.16.5/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.17.1/src/main/resources/configuration.txt b/forge-1.17.1/src/main/resources/configuration.txt index ec572378..93ecb163 100644 --- a/forge-1.17.1/src/main/resources/configuration.txt +++ b/forge-1.17.1/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.18/src/main/resources/configuration.txt b/forge-1.18/src/main/resources/configuration.txt index 05cc179c..09180111 100644 --- a/forge-1.18/src/main/resources/configuration.txt +++ b/forge-1.18/src/main/resources/configuration.txt @@ -39,6 +39,14 @@ storage: #userid: dynmap #password: dynmap #prefix: "" + # + # AWS S3 backet web site + #type: aws_s3 + #bucketname: "dynmap-bucket-name" + #region: us-east-1 + #aws_access_key_id: "" + #aws_secret_access_key: "" + #prefix: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index 7968a104..d8b39046 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -43,9 +43,10 @@ storage: # # AWS S3 backet web site #type: aws_s3 - #bucketname: dynmap + #bucketname: "dynmap-bucket-name" #region: us-east-1 - + #aws_access_key_id: "" + #aws_secret_access_key: "" components: - class: org.dynmap.ClientConfigurationComponent