Upstream/dynmap
1
0
Fork 0
mirror of https://github.com/webbukkit/dynmap.git synced 2024-11-27 20:58:40 +01:00
Code Issues Projects Releases Wiki Activity

Get AWS S3 storage based web working

Browse Source
This commit is contained in:
Mike Primm 2022-02-20 20:58:21 -06:00
parent 45bc02cf3a
commit ea97296684
6 changed files with 409 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
DynmapCore/build.gradle
View File

@ -17,17 +17,15 @@ dependencies {
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
implementation 'org.postgresql:postgresql:42.2.18' implementation 'org.postgresql:postgresql:42.2.18'
implementation 'software.amazon.awssdk:s3:2.17.132' implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
implementation 'software.amazon.awssdk:aws-core:2.17.132' implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
implementation 'software.amazon.awssdk:sdk-core:2.17.132' implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
implementation 'software.amazon.awssdk:utils:2.17.132' implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
implementation 'software.amazon.awssdk:http-client-spi:2.17.132' implementation 'io.github.linktosriram:s3-lite-http-client-apache:0.2.0'
implementation 'software.amazon.awssdk:profiles:2.17.132' implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
implementation 'software.amazon.awssdk:regions:2.17.132' implementation 'org.apache.httpcomponents:httpclient:4.5.9'
implementation 'software.amazon.awssdk:auth:2.17.132' implementation 'javax.xml.bind:jaxb-api:2.3.1'
implementation 'software.amazon.awssdk:metrics-spi:2.17.132' implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1'
implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132'
implementation 'software.amazon.awssdk:protocol-core:2.17.132'
} }
processResources { processResources {
@ -62,17 +60,13 @@ shadowJar {
include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty::'))
include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
include(dependency('org.postgresql:postgresql:')) include(dependency('org.postgresql:postgresql:'))
include(dependency('software.amazon.awssdk:s3:')) include(dependency('io.github.linktosriram:s3-lite-core:'))
include(dependency('software.amazon.awssdk:aws-core:')) include(dependency('io.github.linktosriram:s3-lite-api:'))
include(dependency('software.amazon.awssdk:sdk-core:')) include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
include(dependency('software.amazon.awssdk:utils:')) include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
include(dependency('software.amazon.awssdk:http-client-spi:')) include(dependency('io.github.linktosriram:s3-lite-http-client-apache:'))
include(dependency('software.amazon.awssdk:profiles:')) include(dependency('io.github.linktosriram:s3-lite-util:'))
include(dependency('software.amazon.awssdk:regions:')) include(dependency('org.apache.httpcomponents:httpclient:'))
include(dependency('software.amazon.awssdk:auth:'))
include(dependency('software.amazon.awssdk:metrics-spi:'))
include(dependency('software.amazon.awssdk:aws-xml-protocol:'))
include(dependency('software.amazon.awssdk:protocol-core:'))
include(dependency(':DynmapCoreAPI')) include(dependency(':DynmapCoreAPI'))
exclude("META-INF/maven/**") exclude("META-INF/maven/**")
exclude("META-INF/services/**") exclude("META-INF/services/**")
@ -83,7 +77,8 @@ shadowJar {
relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('org.owasp.html', 'org.dynmap.org.owasp.html')
relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('javax.servlet', 'org.dynmap.javax.servlet' )
relocate('org.postgresql', 'org.dynmap.org.postgresql') relocate('org.postgresql', 'org.dynmap.org.postgresql')
relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk') relocate('io.github.linktosriram.s3lite', 'org.dynmap.s3lite')
destinationDir = file '../target' destinationDir = file '../target'
classifier = '' classifier = ''
} }

80
DynmapCore/src/main/java/org/dynmap/DynmapCore.java
View File

@ -1,6 +1,9 @@
package org.dynmap; package org.dynmap;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader; import java.io.FileReader;
@ -61,6 +64,7 @@ import org.dynmap.storage.mariadb.MariaDBMapStorage;
import org.dynmap.storage.sqllte.SQLiteMapStorage; import org.dynmap.storage.sqllte.SQLiteMapStorage;
import org.dynmap.storage.postgresql.PostgreSQLMapStorage; import org.dynmap.storage.postgresql.PostgreSQLMapStorage;
import org.dynmap.utils.BlockStep; import org.dynmap.utils.BlockStep;
import org.dynmap.utils.BufferOutputStream;
import org.dynmap.utils.ImageIOManager; import org.dynmap.utils.ImageIOManager;
import org.dynmap.web.BanIPFilter; import org.dynmap.web.BanIPFilter;
import org.dynmap.web.CustomHeaderFilter; import org.dynmap.web.CustomHeaderFilter;
@ -490,7 +494,10 @@ public class DynmapCore implements DynmapCommonAPI {
authmgr = new WebAuthManager(this); authmgr = new WebAuthManager(this);
defaultStorage.setLoginEnabled(this); defaultStorage.setLoginEnabled(this);
} }
// If storage serves web files, extract and publsh them
if (defaultStorage.needsStaticWebFiles()) {
updateStaticWebToStorage();
}
/* Load control for leaf transparency (spout lighting bug workaround) */ /* Load control for leaf transparency (spout lighting bug workaround) */
transparentLeaves = configuration.getBoolean("transparent-leaves", true); transparentLeaves = configuration.getBoolean("transparent-leaves", true);
@ -2821,6 +2828,63 @@ public class DynmapCore implements DynmapCommonAPI {
} }
return dir.delete(); return dir.delete();
} }
private void updateStaticWebToStorage() {
if(jarfile == null) return;
// If doing update and web path update is disabled, send warning
if (!this.updatewebpathfiles) {
return;
}
Log.info("Publishing web files to storage");
/* Open JAR as ZIP */
ZipFile zf = null;
InputStream ins = null;
byte[] buf = new byte[2048];
String n = null;
try {
zf = new ZipFile(jarfile);
Enumeration<? extends ZipEntry> e = zf.entries();
while (e.hasMoreElements()) {
ZipEntry ze = e.nextElement();
n = ze.getName();
if (!n.startsWith("extracted/web/")) {
continue;
}
n = n.substring("extracted/web/".length());
// If file is going to web path, redirect it to the configured web
if (ze.isDirectory()) {
continue;
}
try {
ins = zf.getInputStream(ze);
BufferOutputStream buffer = new BufferOutputStream();
int len;
while ((len = ins.read(buf)) >= 0) {
buffer.write(buf, 0, len);
}
defaultStorage.setStaticWebFile(n, buffer);
} catch(IOException io) {
Log.severe("Error updating file in storage - " + n, io);
} finally {
if (ins != null) {
ins.close();
ins = null;
}
}
}
} catch (IOException iox) {
Log.severe("Error extracting file - " + n);
} finally {
if (ins != null) {
try { ins.close(); } catch (IOException iox) {}
ins = null;
}
if (zf != null) {
try { zf.close(); } catch (IOException iox) {}
zf = null;
}
}
}
private void updateExtractedFiles() { private void updateExtractedFiles() {
if(jarfile == null) return; if(jarfile == null) return;
File df = this.getDataFolder(); File df = this.getDataFolder();
@ -2922,13 +2986,13 @@ public class DynmapCore implements DynmapCommonAPI {
} }
else { else {
try { try {
f.getParentFile().mkdirs(); f.getParentFile().mkdirs();
fos = new FileOutputStream(f); fos = new FileOutputStream(f);
ins = zf.getInputStream(ze); ins = zf.getInputStream(ze);
int len; int len;
while ((len = ins.read(buf)) >= 0) { while ((len = ins.read(buf)) >= 0) {
fos.write(buf, 0, len); fos.write(buf, 0, len);
} }
} catch(IOException io) { } catch(IOException io) {
Log.severe("Error updating file - " + f.getPath(), io); Log.severe("Error updating file - " + f.getPath(), io);
} finally { } finally {

37
DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java
View File

@ -272,21 +272,28 @@ public class JsonFileClientUpdateComponent extends ClientUpdateComponent {
byte[] outputBytes = sb.toString().getBytes(cs_utf8); byte[] outputBytes = sb.toString().getBytes(cs_utf8);
MapManager.scheduleDelayedJob(new Runnable() { MapManager.scheduleDelayedJob(new Runnable() {
public void run() { public void run() {
File f = new File(baseStandaloneDir, "config.js"); if (core.getDefaultMapStorage().needsStaticWebFiles()) {
FileOutputStream fos = null; BufferOutputStream os = new BufferOutputStream();
try { os.write(outputBytes);
fos = new FileOutputStream(f); core.getDefaultMapStorage().setStaticWebFile("standalone/config.js", os);
fos.write(outputBytes); }
} catch (IOException iox) { else {
Log.severe("Exception while writing " + f.getPath(), iox); File f = new File(baseStandaloneDir, "config.js");
} finally { FileOutputStream fos = null;
if(fos != null) { try {
try { fos = new FileOutputStream(f);
fos.close(); fos.write(outputBytes);
} catch (IOException x) {} } catch (IOException iox) {
fos = null; Log.severe("Exception while writing " + f.getPath(), iox);
} } finally {
} if(fos != null) {
try {
fos.close();
} catch (IOException x) {}
fos = null;
}
}
}
} }
}, 0); }, 0);
} }

15
DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java
View File

@ -1,5 +1,6 @@
package org.dynmap.storage; package org.dynmap.storage;
import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
@ -459,6 +460,20 @@ public abstract class MapStorage {
} }
// Test if storage needs static web files
public boolean needsStaticWebFiles() {
return false;
}
/**
* Set static web file content
* @param fileid - file path
* @param buffer - content for file
* @return true if successful
*/
public boolean setStaticWebFile(String fileid, BufferOutputStream buffer) {
return false;
}
public void logSQLException(String opmsg, SQLException x) { public void logSQLException(String opmsg, SQLException x) {
Log.severe("SQLException: " + opmsg); Log.severe("SQLException: " + opmsg);
Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState()); Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState());

430
DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
View File

@ -1,9 +1,14 @@
package org.dynmap.storage.aws_s3; package org.dynmap.storage.aws_s3;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import org.dynmap.DynmapCore; import org.dynmap.DynmapCore;
import org.dynmap.DynmapWorld; import org.dynmap.DynmapWorld;
@ -21,33 +26,32 @@ import org.dynmap.storage.MapStorageTileSearchEndCB;
import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferInputStream;
import org.dynmap.utils.BufferOutputStream; import org.dynmap.utils.BufferOutputStream;
import software.amazon.awssdk.awscore.exception.AwsServiceException; import io.github.linktosriram.s3lite.api.client.S3Client;
import software.amazon.awssdk.core.ResponseBytes; import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
import software.amazon.awssdk.core.sync.RequestBody; import io.github.linktosriram.s3lite.api.exception.S3Exception;
import software.amazon.awssdk.regions.Region; import io.github.linktosriram.s3lite.api.region.Region;
import software.amazon.awssdk.services.s3.S3Client; import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.Delete; import io.github.linktosriram.s3lite.api.request.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import io.github.linktosriram.s3lite.api.request.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; import io.github.linktosriram.s3lite.api.response.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.GetBucketAclResponse; import io.github.linktosriram.s3lite.api.response.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; import io.github.linktosriram.s3lite.api.response.ResponseBytes;
import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; import io.github.linktosriram.s3lite.api.response.S3Object;
import software.amazon.awssdk.services.s3.model.GetObjectRequest; import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials;
import software.amazon.awssdk.services.s3.model.GetObjectResponse; import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import io.github.linktosriram.s3lite.http.apache.ApacheSdkHttpClient;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import io.github.linktosriram.s3lite.http.spi.request.RequestBody;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AWSS3MapStorage extends MapStorage { public class AWSS3MapStorage extends MapStorage {
public class StorageTile extends MapStorageTile { public class StorageTile extends MapStorageTile {
private final String baseKey; private final String baseKey;
private final String uri;
StorageTile(DynmapWorld world, MapType map, int x, int y, StorageTile(DynmapWorld world, MapType map, int x, int y,
int zoom, ImageVariant var) { int zoom, ImageVariant var) {
super(world, map, x, y, zoom, var); super(world, map, x, y, zoom, var);
String baseURI; String baseURI;
if (zoom > 0) { if (zoom > 0) {
baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y; baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y;
@ -55,18 +59,21 @@ public class AWSS3MapStorage extends MapStorage {
else { else {
baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y; baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y;
} }
baseKey = "tiles/" + world.getName() + "/" + baseURI + "." + map.getImageFormat().getFileExt(); uri = baseURI + "." + map.getImageFormat().getFileExt();
baseKey = "tiles/" + world.getName() + "/" + uri;
} }
@Override @Override
public boolean exists() { public boolean exists() {
boolean exists = false; boolean exists = false;
try { try {
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req); ListObjectsV2Response rslt = s3.listObjectsV2(req);
if (rslt != null) if ((rslt != null) && (rslt.getKeyCount() > 0))
exists = true; exists = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
}
} }
return exists; return exists;
} }
@ -78,25 +85,24 @@ public class AWSS3MapStorage extends MapStorage {
@Override @Override
public TileRead read() { public TileRead read() {
AWSS3MapStorage.this.getWriteLock(baseKey);
try { try {
GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build(); GetObjectRequest req = GetObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
ResponseBytes<GetObjectResponse> obj = s3.getObjectAsBytes(req); ResponseBytes<GetObjectResponse> obj = s3.getObjectAsBytes(req);
if (obj != null) { if (obj != null) {
GetObjectResponse rsp = obj.response(); GetObjectResponse rsp = obj.getResponse();
TileRead tr = new TileRead(); TileRead tr = new TileRead();
byte[] buf = obj.asByteArray(); byte[] buf = obj.getBytes();
tr.image = new BufferInputStream(buf); tr.image = new BufferInputStream(buf);
tr.format = ImageEncoding.fromContentType(rsp.contentType()); tr.format = ImageEncoding.fromContentType(rsp.getContentType());
tr.hashCode = rsp.eTag().hashCode(); tr.hashCode = rsp.geteTag().hashCode();
tr.lastModified = rsp.lastModified().toEpochMilli(); tr.lastModified = rsp.getLastModified().toEpochMilli();
return tr; return tr;
} }
} catch (AwsServiceException x) { } catch (NoSuchKeyException nskx) {
Log.severe("AWS Exception", x); return null; // Nominal case if it doesn't exist
} finally { } catch (S3Exception x) {
AWSS3MapStorage.this.releaseWriteLock(baseKey); Log.severe("AWS Exception", x);
} }
return null; return null;
} }
@ -104,21 +110,18 @@ public class AWSS3MapStorage extends MapStorage {
@Override @Override
public boolean write(long hash, BufferOutputStream encImage, long timestamp) { public boolean write(long hash, BufferOutputStream encImage, long timestamp) {
boolean done = false; boolean done = false;
AWSS3MapStorage.this.getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(req); s3.deleteObject(req);
} }
else { else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf)); s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
} }
done = true; done = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
} }
// Signal update for zoom out // Signal update for zoom out
if (zoom == 0) { if (zoom == 0) {
@ -129,22 +132,20 @@ public class AWSS3MapStorage extends MapStorage {
@Override @Override
public boolean getWriteLock() { public boolean getWriteLock() {
return AWSS3MapStorage.this.getWriteLock(baseKey); return true;
} }
@Override @Override
public void releaseWriteLock() { public void releaseWriteLock() {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
} }
@Override @Override
public boolean getReadLock(long timeout) { public boolean getReadLock(long timeout) {
return AWSS3MapStorage.this.getReadLock(baseKey, timeout); return true;
} }
@Override @Override
public void releaseReadLock() { public void releaseReadLock() {
AWSS3MapStorage.this.releaseReadLock(baseKey);
} }
@Override @Override
@ -153,7 +154,7 @@ public class AWSS3MapStorage extends MapStorage {
@Override @Override
public String getURI() { public String getURI() {
return null; return baseKey;
} }
@Override @Override
@ -196,7 +197,8 @@ public class AWSS3MapStorage extends MapStorage {
private String bucketname; private String bucketname;
private String region; private String region;
private String profile_id; private String access_key_id;
private String secret_access_key;
private S3Client s3; private S3Client s3;
public AWSS3MapStorage() { public AWSS3MapStorage() {
@ -211,27 +213,45 @@ public class AWSS3MapStorage extends MapStorage {
Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt"); Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt");
return false; return false;
} }
if (core.isLoginSupportEnabled()) {
Log.severe("AWS S3 storage is not supported option with loegin support enabled: set login-enabled: false in configuration.txt");
return false;
}
// Get our settings // Get our settings
bucketname = core.configuration.getString("storage/bucketname", "dynmap"); bucketname = core.configuration.getString("storage/bucketname", "dynmap");
region = core.configuration.getString("storage/region", "us-east-1"); region = core.configuration.getString("storage/region", "us-east-1");
access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
// Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region);
s3 = new DefaultS3ClientBuilder()
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
.region(Region.fromString(region))
.httpClient(ApacheSdkHttpClient.defaultClient())
.build();
if (s3 == null) {
Log.severe("Error creating S3 access client");
return false;
}
// Make sure bucket exists (do list)
ListObjectsV2Request listreq = ListObjectsV2Request.builder()
.bucketName(bucketname)
.maxKeys(1)
.build();
try { try {
// Now creste the access client for the S3 service ListObjectsV2Response rslt = s3.listObjectsV2(listreq);
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); if (rslt == null) {
s3 = S3Client.builder().region(Region.of(region)).build();
if (s3 == null) {
Log.severe("Error creating S3 access client");
return false;
}
// Make sure bucket exists and get ACL
GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build();
GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr);
if (bucketACL == null) {
Log.severe("Error: cannot find or access S3 bucket"); Log.severe("Error: cannot find or access S3 bucket");
return false; return false;
} }
} catch (AwsServiceException x) { List<S3Object> content = rslt.getContents();
Log.severe("AWS Exception", x); Log.info("content=" + content.size());
return false; } catch (S3Exception s3x) {
if (!s3x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", s3x);
Log.severe("req=" + listreq);
return false;
}
} }
return true; return true;
} }
@ -288,57 +308,71 @@ public class AWSS3MapStorage extends MapStorage {
} }
private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase,
MapStorageTileSearchEndCB cbEnd) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build();
boolean done = false;
try { try {
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); while (!done) {
ListObjectsV2Response result = s3.listObjectsV2(req); ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.contents(); List<S3Object> objects = result.getContents();
for (S3Object os : objects) { for (S3Object os : objects) {
String key = os.key(); String key = os.getKey();
key = key.substring(basekey.length()); // Strip off base key = key.substring(basekey.length()); // Strip off base
// Parse the extension // Parse the extension
String ext = null; String ext = null;
int extoff = key.lastIndexOf('.'); int extoff = key.lastIndexOf('.');
if (extoff >= 0) { if (extoff >= 0) {
ext = key.substring(extoff+1); ext = key.substring(extoff+1);
key = key.substring(0, extoff); key = key.substring(0, extoff);
} }
// If not valid image extension, ignore // If not valid image extension, ignore
ImageEncoding fmt = ImageEncoding.fromExt(ext); ImageEncoding fmt = ImageEncoding.fromExt(ext);
if (fmt == null) { if (fmt == null) {
continue; continue;
} }
// See if zoom tile: figure out zoom level // See if zoom tile: figure out zoom level
int zoom = 0; int zoom = 0;
if (key.startsWith("z")) { if (key.startsWith("z")) {
while (key.startsWith("z")) { while (key.startsWith("z")) {
key = key.substring(1); key = key.substring(1);
zoom++; zoom++;
} }
if (key.startsWith("_")) { if (key.startsWith("_")) {
key = key.substring(1); key = key.substring(1);
} }
} }
// Split remainder to get coords // Split remainder to get coords
String[] coord = key.split("_"); String[] coord = key.split("_");
if (coord.length == 2) { // Must be 2 to be a tile if (coord.length == 2) { // Must be 2 to be a tile
try { try {
int x = Integer.parseInt(coord[0]); int x = Integer.parseInt(coord[0]);
int y = Integer.parseInt(coord[1]); int y = Integer.parseInt(coord[1]);
// Invoke callback // Invoke callback
MapStorageTile t = new StorageTile(world, map, x, y, zoom, var); MapStorageTile t = new StorageTile(world, map, x, y, zoom, var);
if(cb != null) if(cb != null)
cb.tileFound(t, fmt); cb.tileFound(t, fmt);
if(cbBase != null && t.zoom == 0) if(cbBase != null && t.zoom == 0)
cbBase.tileFound(t, fmt); cbBase.tileFound(t, fmt);
t.cleanup(); t.cleanup();
} catch (NumberFormatException nfx) { } catch (NumberFormatException nfx) {
} }
} }
} }
} catch (AwsServiceException x) { if (result.isTruncated()) { // If more, build continuiation request
Log.severe("AWS Exception", x); req = ListObjectsV2Request.builder().bucketName(bucketname)
.prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build();
}
else { // Else, we're done
done = true;
}
}
} catch (S3Exception x) {
if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
Log.severe("req=" + req);
}
} }
if(cbEnd != null) { if(cbEnd != null) {
cbEnd.searchEnded(); cbEnd.searchEnded();
@ -383,28 +417,30 @@ public class AWSS3MapStorage extends MapStorage {
private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build();
try { try {
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); boolean done = false;
ListObjectsV2Response result = s3.listObjectsV2(req); while (!done) {
List<S3Object> objects = result.contents(); ListObjectsV2Response result = s3.listObjectsV2(req);
ArrayList<ObjectIdentifier> keys = new ArrayList<ObjectIdentifier>(); List<S3Object> objects = result.getContents();
for (S3Object os : objects) { for (S3Object os : objects) {
String key = os.key(); String key = os.getKey();
keys.add(ObjectIdentifier.builder().key(key).build()); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(key).build();
if (keys.size() >= 100) { s3.deleteObject(delreq);
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); }
s3.deleteObjects(delreq); if (result.isTruncated()) { // If more, build continuiation request
keys.clear(); req = ListObjectsV2Request.builder().bucketName(bucketname)
} .prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build();
}
else { // Else, we're done
done = true;
}
} }
// Any left? } catch (S3Exception x) {
if (keys.size() > 0) { if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); Log.severe("AWS Exception", x);
s3.deleteObjects(delreq); Log.severe("req=" + req);
keys.clear(); }
}
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} }
} }
@ -431,21 +467,18 @@ public class AWSS3MapStorage extends MapStorage {
BufferOutputStream encImage) { BufferOutputStream encImage) {
boolean done = false; boolean done = false;
String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq); s3.deleteObject(delreq);
} }
else { else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf)); s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
} }
done = true; done = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
} }
return done; return done;
} }
@ -461,12 +494,14 @@ public class AWSS3MapStorage extends MapStorage {
String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
boolean exists = false; boolean exists = false;
try { try {
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req); ListObjectsV2Response rslt = s3.listObjectsV2(req);
if (rslt != null) if ((rslt != null) && (rslt.getKeyCount() > 0))
exists = true; exists = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
}
} }
return exists; return exists;
} }
@ -475,21 +510,18 @@ public class AWSS3MapStorage extends MapStorage {
public boolean setMarkerImage(String markerid, BufferOutputStream encImage) { public boolean setMarkerImage(String markerid, BufferOutputStream encImage) {
boolean done = false; boolean done = false;
String baseKey = "_markers_/" + markerid + ".png"; String baseKey = "_markers_/" + markerid + ".png";
getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq); s3.deleteObject(delreq);
} }
else { else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf)); s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
} }
done = true; done = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
} }
return done; return done;
} }
@ -503,21 +535,18 @@ public class AWSS3MapStorage extends MapStorage {
public boolean setMarkerFile(String world, String content) { public boolean setMarkerFile(String world, String content) {
boolean done = false; boolean done = false;
String baseKey = "_markers_/marker_" + world + ".json"; String baseKey = "_markers_/marker_" + world + ".json";
getWriteLock(baseKey);
try { try {
if (content == null) { // Delete? if (content == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq); s3.deleteObject(delreq);
} }
else { else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build(); PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("application/json").build();
s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8)));
} }
done = true; done = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
} }
return done; return done;
} }
@ -530,15 +559,32 @@ public class AWSS3MapStorage extends MapStorage {
@Override @Override
// For external web server only // For external web server only
public String getMarkersURI(boolean login_enabled) { public String getMarkersURI(boolean login_enabled) {
return login_enabled?"standalone/markers.php?marker=":"tiles/"; return "tiles/";
} }
@Override @Override
// For external web server only // For external web server only
public String getTilesURI(boolean login_enabled) { public String getTilesURI(boolean login_enabled) {
return login_enabled?"standalone/tiles.php?tile=":"tiles/"; return "tiles/";
} }
/**
* URI to use for loading configuration JSON files (for external web server only)
* @param login_enabled - selects based on login security enabled
* @return URI
*/
public String getConfigurationJSONURI(boolean login_enabled) {
return "standalone/dynmap_config.json";
}
/**
* URI to use for loading update JSON files (for external web server only)
* @param login_enabled - selects based on login security enabled
* @return URI
*/
public String getUpdateJSONURI(boolean login_enabled) {
return "standalone/dynmap_{world}.json";
}
@Override @Override
public void addPaths(StringBuilder sb, DynmapCore core) { public void addPaths(StringBuilder sb, DynmapCore core) {
String p = core.getTilesFolder().getAbsolutePath(); String p = core.getTilesFolder().getAbsolutePath();
@ -561,28 +607,76 @@ public class AWSS3MapStorage extends MapStorage {
return null; return null;
} }
// Cache to avoid rewriting same standalong file repeatedly
private ConcurrentHashMap<String, byte[]> standalone_cache = new ConcurrentHashMap<String, byte[]>();
@Override @Override
public boolean setStandaloneFile(String fileid, BufferOutputStream content) { public boolean setStandaloneFile(String fileid, BufferOutputStream content) {
return setStaticWebFile("standalone/" + fileid, content);
}
// Test if storage needs static web files
public boolean needsStaticWebFiles() {
return true;
}
/**
* Set static web file content
* @param fileid - file path
* @param content - content for file
* @return true if successful
*/
public boolean setStaticWebFile(String fileid, BufferOutputStream content) {
boolean done = false; boolean done = false;
String baseKey = "standalone/" + fileid;
getWriteLock(baseKey);
try { try {
byte[] cacheval = standalone_cache.get(fileid);
if (content == null) { // Delete? if (content == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
return true;
}
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(fileid).build();
s3.deleteObject(delreq); s3.deleteObject(delreq);
standalone_cache.put(fileid, new byte[0]); // Mark in cache
} }
else { else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build(); byte[] digest = content.buf;
s3.putObject(req, RequestBody.fromBytes(content.buf)); try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(content.buf);
digest = md.digest();
} catch (NoSuchAlgorithmException nsax) {
}
// If cached and same, just return
if (Arrays.equals(digest, cacheval)) {
return true;
}
String ct = "text/plain";
if (fileid.endsWith(".json")) {
ct = "application/json";
}
else if (fileid.endsWith(".php")) {
ct = "application/x-httpd-php";
}
else if (fileid.endsWith(".html")) {
ct = "text/html";
}
else if (fileid.endsWith(".css")) {
ct = "text/css";
}
else if (fileid.endsWith(".js")) {
ct = "application/x-javascript";
}
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(fileid).contentType(ct).build();
s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
standalone_cache.put(fileid, digest);
} }
done = true; done = true;
} catch (AwsServiceException x) { } catch (S3Exception x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
} }
return done; return done;
} }
} }

37
bukkit-helper/.project
View File

@ -2,32 +2,35 @@
<projectDescription> <projectDescription>
<name>Dynmap(Spigot-Common)</name> <name>Dynmap(Spigot-Common)</name>
<comment>bukkit-helper</comment> <comment>bukkit-helper</comment>
<projects/> <projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures> <natures>
<nature>org.eclipse.jdt.core.javanature</nature> <nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature> <nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature> <nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures> </natures>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments/>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments/>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments/>
</buildCommand>
</buildSpec>
<linkedResources/>
<filteredResources> <filteredResources>
<filter> <filter>
<id>1</id> <id>1</id>
<name></name>
<type>30</type> <type>30</type>
<name/>
<matcher> <matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id> <id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments> <arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>