Upstream/dynmap
1
0
Fork 0
mirror of https://github.com/webbukkit/dynmap.git synced 2024-11-24 03:05:28 +01:00
Code Issues Projects Releases Wiki Activity

Get AWS S3 storage based web working

Browse Source
This commit is contained in:
Mike Primm 2022-02-20 20:58:21 -06:00
parent 45bc02cf3a
commit ea97296684
6 changed files with 409 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
DynmapCore/build.gradle
View File

@ -17,17 +17,15 @@ dependencies {
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
implementation 'org.postgresql:postgresql:42.2.18'
implementation 'software.amazon.awssdk:s3:2.17.132'
implementation 'software.amazon.awssdk:aws-core:2.17.132'
implementation 'software.amazon.awssdk:sdk-core:2.17.132'
implementation 'software.amazon.awssdk:utils:2.17.132'
implementation 'software.amazon.awssdk:http-client-spi:2.17.132'
implementation 'software.amazon.awssdk:profiles:2.17.132'
implementation 'software.amazon.awssdk:regions:2.17.132'
implementation 'software.amazon.awssdk:auth:2.17.132'
implementation 'software.amazon.awssdk:metrics-spi:2.17.132'
implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132'
implementation 'software.amazon.awssdk:protocol-core:2.17.132'
implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
implementation 'io.github.linktosriram:s3-lite-http-client-apache:0.2.0'
implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
implementation 'org.apache.httpcomponents:httpclient:4.5.9'
implementation 'javax.xml.bind:jaxb-api:2.3.1'
implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1'
}
processResources {
@ -62,17 +60,13 @@ shadowJar {
include(dependency('org.eclipse.jetty::'))
include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
include(dependency('org.postgresql:postgresql:'))
include(dependency('software.amazon.awssdk:s3:'))
include(dependency('software.amazon.awssdk:aws-core:'))
include(dependency('software.amazon.awssdk:sdk-core:'))
include(dependency('software.amazon.awssdk:utils:'))
include(dependency('software.amazon.awssdk:http-client-spi:'))
include(dependency('software.amazon.awssdk:profiles:'))
include(dependency('software.amazon.awssdk:regions:'))
include(dependency('software.amazon.awssdk:auth:'))
include(dependency('software.amazon.awssdk:metrics-spi:'))
include(dependency('software.amazon.awssdk:aws-xml-protocol:'))
include(dependency('software.amazon.awssdk:protocol-core:'))
include(dependency('io.github.linktosriram:s3-lite-core:'))
include(dependency('io.github.linktosriram:s3-lite-api:'))
include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
include(dependency('io.github.linktosriram:s3-lite-http-client-apache:'))
include(dependency('io.github.linktosriram:s3-lite-util:'))
include(dependency('org.apache.httpcomponents:httpclient:'))
include(dependency(':DynmapCoreAPI'))
exclude("META-INF/maven/**")
exclude("META-INF/services/**")
@ -83,7 +77,8 @@ shadowJar {
relocate('org.owasp.html', 'org.dynmap.org.owasp.html')
relocate('javax.servlet', 'org.dynmap.javax.servlet' )
relocate('org.postgresql', 'org.dynmap.org.postgresql')
relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk')
relocate('io.github.linktosriram.s3lite', 'org.dynmap.s3lite')
destinationDir = file '../target'
classifier = ''
}

80
DynmapCore/src/main/java/org/dynmap/DynmapCore.java
View File

@ -1,6 +1,9 @@
package org.dynmap;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
@ -61,6 +64,7 @@ import org.dynmap.storage.mariadb.MariaDBMapStorage;
import org.dynmap.storage.sqllte.SQLiteMapStorage;
import org.dynmap.storage.postgresql.PostgreSQLMapStorage;
import org.dynmap.utils.BlockStep;
import org.dynmap.utils.BufferOutputStream;
import org.dynmap.utils.ImageIOManager;
import org.dynmap.web.BanIPFilter;
import org.dynmap.web.CustomHeaderFilter;
@ -490,7 +494,10 @@ public class DynmapCore implements DynmapCommonAPI {
authmgr = new WebAuthManager(this);
defaultStorage.setLoginEnabled(this);
}
// If storage serves web files, extract and publsh them
if (defaultStorage.needsStaticWebFiles()) {
updateStaticWebToStorage();
}
/* Load control for leaf transparency (spout lighting bug workaround) */
transparentLeaves = configuration.getBoolean("transparent-leaves", true);
@ -2821,6 +2828,63 @@ public class DynmapCore implements DynmapCommonAPI {
}
return dir.delete();
}
private void updateStaticWebToStorage() {
if(jarfile == null) return;
// If doing update and web path update is disabled, send warning
if (!this.updatewebpathfiles) {
return;
}
Log.info("Publishing web files to storage");
/* Open JAR as ZIP */
ZipFile zf = null;
InputStream ins = null;
byte[] buf = new byte[2048];
String n = null;
try {
zf = new ZipFile(jarfile);
Enumeration<? extends ZipEntry> e = zf.entries();
while (e.hasMoreElements()) {
ZipEntry ze = e.nextElement();
n = ze.getName();
if (!n.startsWith("extracted/web/")) {
continue;
}
n = n.substring("extracted/web/".length());
// If file is going to web path, redirect it to the configured web
if (ze.isDirectory()) {
continue;
}
try {
ins = zf.getInputStream(ze);
BufferOutputStream buffer = new BufferOutputStream();
int len;
while ((len = ins.read(buf)) >= 0) {
buffer.write(buf, 0, len);
}
defaultStorage.setStaticWebFile(n, buffer);
} catch(IOException io) {
Log.severe("Error updating file in storage - " + n, io);
} finally {
if (ins != null) {
ins.close();
ins = null;
}
}
}
} catch (IOException iox) {
Log.severe("Error extracting file - " + n);
} finally {
if (ins != null) {
try { ins.close(); } catch (IOException iox) {}
ins = null;
}
if (zf != null) {
try { zf.close(); } catch (IOException iox) {}
zf = null;
}
}
}
private void updateExtractedFiles() {
if(jarfile == null) return;
File df = this.getDataFolder();
@ -2922,13 +2986,13 @@ public class DynmapCore implements DynmapCommonAPI {
}
else {
try {
f.getParentFile().mkdirs();
fos = new FileOutputStream(f);
ins = zf.getInputStream(ze);
int len;
while ((len = ins.read(buf)) >= 0) {
fos.write(buf, 0, len);
}
f.getParentFile().mkdirs();
fos = new FileOutputStream(f);
ins = zf.getInputStream(ze);
int len;
while ((len = ins.read(buf)) >= 0) {
fos.write(buf, 0, len);
}
} catch(IOException io) {
Log.severe("Error updating file - " + f.getPath(), io);
} finally {

37
DynmapCore/src/main/java/org/dynmap/JsonFileClientUpdateComponent.java
View File

@ -272,21 +272,28 @@ public class JsonFileClientUpdateComponent extends ClientUpdateComponent {
byte[] outputBytes = sb.toString().getBytes(cs_utf8);
MapManager.scheduleDelayedJob(new Runnable() {
public void run() {
File f = new File(baseStandaloneDir, "config.js");
FileOutputStream fos = null;
try {
fos = new FileOutputStream(f);
fos.write(outputBytes);
} catch (IOException iox) {
Log.severe("Exception while writing " + f.getPath(), iox);
} finally {
if(fos != null) {
try {
fos.close();
} catch (IOException x) {}
fos = null;
}
}
if (core.getDefaultMapStorage().needsStaticWebFiles()) {
BufferOutputStream os = new BufferOutputStream();
os.write(outputBytes);
core.getDefaultMapStorage().setStaticWebFile("standalone/config.js", os);
}
else {
File f = new File(baseStandaloneDir, "config.js");
FileOutputStream fos = null;
try {
fos = new FileOutputStream(f);
fos.write(outputBytes);
} catch (IOException iox) {
Log.severe("Exception while writing " + f.getPath(), iox);
} finally {
if(fos != null) {
try {
fos.close();
} catch (IOException x) {}
fos = null;
}
}
}
}
}, 0);
}

15
DynmapCore/src/main/java/org/dynmap/storage/MapStorage.java
View File

@ -1,5 +1,6 @@
package org.dynmap.storage;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@ -459,6 +460,20 @@ public abstract class MapStorage {
}
// Test if storage needs static web files
public boolean needsStaticWebFiles() {
return false;
}
/**
* Set static web file content
* @param fileid - file path
* @param buffer - content for file
* @return true if successful
*/
public boolean setStaticWebFile(String fileid, BufferOutputStream buffer) {
return false;
}
public void logSQLException(String opmsg, SQLException x) {
Log.severe("SQLException: " + opmsg);
Log.severe(" ErrorCode: " + x.getErrorCode() + ", SQLState=" + x.getSQLState());

430
DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
View File

@ -1,9 +1,14 @@
package org.dynmap.storage.aws_s3;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import org.dynmap.DynmapCore;
import org.dynmap.DynmapWorld;
@ -21,33 +26,32 @@ import org.dynmap.storage.MapStorageTileSearchEndCB;
import org.dynmap.utils.BufferInputStream;
import org.dynmap.utils.BufferOutputStream;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectAclRequest;
import software.amazon.awssdk.services.s3.model.GetObjectAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
import io.github.linktosriram.s3lite.api.client.S3Client;
import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
import io.github.linktosriram.s3lite.api.exception.S3Exception;
import io.github.linktosriram.s3lite.api.region.Region;
import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest;
import io.github.linktosriram.s3lite.api.request.GetObjectRequest;
import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request;
import io.github.linktosriram.s3lite.api.request.PutObjectRequest;
import io.github.linktosriram.s3lite.api.response.GetObjectResponse;
import io.github.linktosriram.s3lite.api.response.ListObjectsV2Response;
import io.github.linktosriram.s3lite.api.response.ResponseBytes;
import io.github.linktosriram.s3lite.api.response.S3Object;
import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials;
import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder;
import io.github.linktosriram.s3lite.http.apache.ApacheSdkHttpClient;
import io.github.linktosriram.s3lite.http.spi.request.RequestBody;
public class AWSS3MapStorage extends MapStorage {
public class StorageTile extends MapStorageTile {
private final String baseKey;
private final String uri;
StorageTile(DynmapWorld world, MapType map, int x, int y,
int zoom, ImageVariant var) {
super(world, map, x, y, zoom, var);
String baseURI;
if (zoom > 0) {
baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + "zzzzzzzzzzzzzzzz".substring(0, zoom) + "_" + x + "_" + y;
@ -55,18 +59,21 @@ public class AWSS3MapStorage extends MapStorage {
else {
baseURI = map.getPrefix() + var.variantSuffix + "/"+ (x >> 5) + "_" + (y >> 5) + "/" + x + "_" + y;
}
baseKey = "tiles/" + world.getName() + "/" + baseURI + "." + map.getImageFormat().getFileExt();
uri = baseURI + "." + map.getImageFormat().getFileExt();
baseKey = "tiles/" + world.getName() + "/" + uri;
}
@Override
public boolean exists() {
boolean exists = false;
try {
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null)
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build();
ListObjectsV2Response rslt = s3.listObjectsV2(req);
if ((rslt != null) && (rslt.getKeyCount() > 0))
exists = true;
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} catch (S3Exception x) {
if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
}
}
return exists;
}
@ -78,25 +85,24 @@ public class AWSS3MapStorage extends MapStorage {
@Override
public TileRead read() {
AWSS3MapStorage.this.getWriteLock(baseKey);
try {
GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectRequest req = GetObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
ResponseBytes<GetObjectResponse> obj = s3.getObjectAsBytes(req);
if (obj != null) {
GetObjectResponse rsp = obj.response();
GetObjectResponse rsp = obj.getResponse();
TileRead tr = new TileRead();
byte[] buf = obj.asByteArray();
byte[] buf = obj.getBytes();
tr.image = new BufferInputStream(buf);
tr.format = ImageEncoding.fromContentType(rsp.contentType());
tr.hashCode = rsp.eTag().hashCode();
tr.lastModified = rsp.lastModified().toEpochMilli();
tr.format = ImageEncoding.fromContentType(rsp.getContentType());
tr.hashCode = rsp.geteTag().hashCode();
tr.lastModified = rsp.getLastModified().toEpochMilli();
return tr;
}
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
} catch (NoSuchKeyException nskx) {
return null; // Nominal case if it doesn't exist
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
}
return null;
}
@ -104,21 +110,18 @@ public class AWSS3MapStorage extends MapStorage {
@Override
public boolean write(long hash, BufferOutputStream encImage, long timestamp) {
boolean done = false;
AWSS3MapStorage.this.getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(req);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (AwsServiceException x) {
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
}
// Signal update for zoom out
if (zoom == 0) {
@ -129,22 +132,20 @@ public class AWSS3MapStorage extends MapStorage {
@Override
public boolean getWriteLock() {
return AWSS3MapStorage.this.getWriteLock(baseKey);
return true;
}
@Override
public void releaseWriteLock() {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
}
@Override
public boolean getReadLock(long timeout) {
return AWSS3MapStorage.this.getReadLock(baseKey, timeout);
return true;
}
@Override
public void releaseReadLock() {
AWSS3MapStorage.this.releaseReadLock(baseKey);
}
@Override
@ -153,7 +154,7 @@ public class AWSS3MapStorage extends MapStorage {
@Override
public String getURI() {
return null;
return baseKey;
}
@Override
@ -196,7 +197,8 @@ public class AWSS3MapStorage extends MapStorage {
private String bucketname;
private String region;
private String profile_id;
private String access_key_id;
private String secret_access_key;
private S3Client s3;
public AWSS3MapStorage() {
@ -211,27 +213,45 @@ public class AWSS3MapStorage extends MapStorage {
Log.severe("AWS S3 storage is not supported option with internal web server: set disable-webserver: true in configuration.txt");
return false;
}
if (core.isLoginSupportEnabled()) {
Log.severe("AWS S3 storage is not supported option with loegin support enabled: set login-enabled: false in configuration.txt");
return false;
}
// Get our settings
bucketname = core.configuration.getString("storage/bucketname", "dynmap");
region = core.configuration.getString("storage/region", "us-east-1");
access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
// Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region);
s3 = new DefaultS3ClientBuilder()
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
.region(Region.fromString(region))
.httpClient(ApacheSdkHttpClient.defaultClient())
.build();
if (s3 == null) {
Log.severe("Error creating S3 access client");
return false;
}
// Make sure bucket exists (do list)
ListObjectsV2Request listreq = ListObjectsV2Request.builder()
.bucketName(bucketname)
.maxKeys(1)
.build();
try {
// Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id);
s3 = S3Client.builder().region(Region.of(region)).build();
if (s3 == null) {
Log.severe("Error creating S3 access client");
return false;
}
// Make sure bucket exists and get ACL
GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build();
GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr);
if (bucketACL == null) {
ListObjectsV2Response rslt = s3.listObjectsV2(listreq);
if (rslt == null) {
Log.severe("Error: cannot find or access S3 bucket");
return false;
}
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
return false;
List<S3Object> content = rslt.getContents();
Log.info("content=" + content.size());
} catch (S3Exception s3x) {
if (!s3x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", s3x);
Log.severe("req=" + listreq);
return false;
}
}
return true;
}
@ -288,57 +308,71 @@ public class AWSS3MapStorage extends MapStorage {
}
private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) {
private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase,
MapStorageTileSearchEndCB cbEnd) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).maxKeys(1000).build();
boolean done = false;
try {
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.contents();
for (S3Object os : objects) {
String key = os.key();
key = key.substring(basekey.length()); // Strip off base
// Parse the extension
String ext = null;
int extoff = key.lastIndexOf('.');
if (extoff >= 0) {
ext = key.substring(extoff+1);
key = key.substring(0, extoff);
}
// If not valid image extension, ignore
ImageEncoding fmt = ImageEncoding.fromExt(ext);
if (fmt == null) {
continue;
}
// See if zoom tile: figure out zoom level
int zoom = 0;
if (key.startsWith("z")) {
while (key.startsWith("z")) {
key = key.substring(1);
zoom++;
}
if (key.startsWith("_")) {
key = key.substring(1);
}
}
// Split remainder to get coords
String[] coord = key.split("_");
if (coord.length == 2) { // Must be 2 to be a tile
try {
int x = Integer.parseInt(coord[0]);
int y = Integer.parseInt(coord[1]);
// Invoke callback
MapStorageTile t = new StorageTile(world, map, x, y, zoom, var);
if(cb != null)
cb.tileFound(t, fmt);
if(cbBase != null && t.zoom == 0)
cbBase.tileFound(t, fmt);
t.cleanup();
} catch (NumberFormatException nfx) {
}
}
}
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
while (!done) {
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.getContents();
for (S3Object os : objects) {
String key = os.getKey();
key = key.substring(basekey.length()); // Strip off base
// Parse the extension
String ext = null;
int extoff = key.lastIndexOf('.');
if (extoff >= 0) {
ext = key.substring(extoff+1);
key = key.substring(0, extoff);
}
// If not valid image extension, ignore
ImageEncoding fmt = ImageEncoding.fromExt(ext);
if (fmt == null) {
continue;
}
// See if zoom tile: figure out zoom level
int zoom = 0;
if (key.startsWith("z")) {
while (key.startsWith("z")) {
key = key.substring(1);
zoom++;
}
if (key.startsWith("_")) {
key = key.substring(1);
}
}
// Split remainder to get coords
String[] coord = key.split("_");
if (coord.length == 2) { // Must be 2 to be a tile
try {
int x = Integer.parseInt(coord[0]);
int y = Integer.parseInt(coord[1]);
// Invoke callback
MapStorageTile t = new StorageTile(world, map, x, y, zoom, var);
if(cb != null)
cb.tileFound(t, fmt);
if(cbBase != null && t.zoom == 0)
cbBase.tileFound(t, fmt);
t.cleanup();
} catch (NumberFormatException nfx) {
}
}
}
if (result.isTruncated()) { // If more, build continuiation request
req = ListObjectsV2Request.builder().bucketName(bucketname)
.prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build();
}
else { // Else, we're done
done = true;
}
}
} catch (S3Exception x) {
if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
Log.severe("req=" + req);
}
}
if(cbEnd != null) {
cbEnd.searchEnded();
@ -383,28 +417,30 @@ public class AWSS3MapStorage extends MapStorage {
private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(basekey).delimiter("").maxKeys(1000).encodingType("url").requestPayer("requester").build();
try {
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.contents();
ArrayList<ObjectIdentifier> keys = new ArrayList<ObjectIdentifier>();
for (S3Object os : objects) {
String key = os.key();
keys.add(ObjectIdentifier.builder().key(key).build());
if (keys.size() >= 100) {
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(delreq);
keys.clear();
}
boolean done = false;
while (!done) {
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.getContents();
for (S3Object os : objects) {
String key = os.getKey();
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(key).build();
s3.deleteObject(delreq);
}
if (result.isTruncated()) { // If more, build continuiation request
req = ListObjectsV2Request.builder().bucketName(bucketname)
.prefix(basekey).delimiter("").maxKeys(1000).continuationToken(result.getContinuationToken()).encodingType("url").requestPayer("requester").build();
}
else { // Else, we're done
done = true;
}
}
// Any left?
if (keys.size() > 0) {
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(delreq);
keys.clear();
}
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} catch (S3Exception x) {
if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
Log.severe("req=" + req);
}
}
}
@ -431,21 +467,18 @@ public class AWSS3MapStorage extends MapStorage {
BufferOutputStream encImage) {
boolean done = false;
String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (AwsServiceException x) {
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
}
return done;
}
@ -461,12 +494,14 @@ public class AWSS3MapStorage extends MapStorage {
String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
boolean exists = false;
try {
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null)
ListObjectsV2Request req = ListObjectsV2Request.builder().bucketName(bucketname).prefix(baseKey).maxKeys(1).build();
ListObjectsV2Response rslt = s3.listObjectsV2(req);
if ((rslt != null) && (rslt.getKeyCount() > 0))
exists = true;
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} catch (S3Exception x) {
if (!x.getCode().equals("SignatureDoesNotMatch")) { // S3 behavior when no object match....
Log.severe("AWS Exception", x);
}
}
return exists;
}
@ -475,21 +510,18 @@ public class AWSS3MapStorage extends MapStorage {
public boolean setMarkerImage(String markerid, BufferOutputStream encImage) {
boolean done = false;
String baseKey = "_markers_/" + markerid + ".png";
getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (AwsServiceException x) {
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
}
return done;
}
@ -503,21 +535,18 @@ public class AWSS3MapStorage extends MapStorage {
public boolean setMarkerFile(String world, String content) {
boolean done = false;
String baseKey = "_markers_/marker_" + world + ".json";
getWriteLock(baseKey);
try {
if (content == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build();
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("application/json").build();
s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8)));
}
done = true;
} catch (AwsServiceException x) {
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
}
return done;
}
@ -530,15 +559,32 @@ public class AWSS3MapStorage extends MapStorage {
@Override
// For external web server only
public String getMarkersURI(boolean login_enabled) {
return login_enabled?"standalone/markers.php?marker=":"tiles/";
return "tiles/";
}
@Override
// For external web server only
public String getTilesURI(boolean login_enabled) {
return login_enabled?"standalone/tiles.php?tile=":"tiles/";
return "tiles/";
}
/**
* URI to use for loading configuration JSON files (for external web server only)
* @param login_enabled - selects based on login security enabled
* @return URI
*/
public String getConfigurationJSONURI(boolean login_enabled) {
return "standalone/dynmap_config.json";
}
/**
* URI to use for loading update JSON files (for external web server only)
* @param login_enabled - selects based on login security enabled
* @return URI
*/
public String getUpdateJSONURI(boolean login_enabled) {
return "standalone/dynmap_{world}.json";
}
@Override
public void addPaths(StringBuilder sb, DynmapCore core) {
String p = core.getTilesFolder().getAbsolutePath();
@ -561,28 +607,76 @@ public class AWSS3MapStorage extends MapStorage {
return null;
}
// Cache to avoid rewriting same standalong file repeatedly
private ConcurrentHashMap<String, byte[]> standalone_cache = new ConcurrentHashMap<String, byte[]>();
@Override
public boolean setStandaloneFile(String fileid, BufferOutputStream content) {
return setStaticWebFile("standalone/" + fileid, content);
}
// Test if storage needs static web files
public boolean needsStaticWebFiles() {
return true;
}
/**
* Set static web file content
* @param fileid - file path
* @param content - content for file
* @return true if successful
*/
public boolean setStaticWebFile(String fileid, BufferOutputStream content) {
boolean done = false;
String baseKey = "standalone/" + fileid;
getWriteLock(baseKey);
try {
byte[] cacheval = standalone_cache.get(fileid);
if (content == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
return true;
}
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(fileid).build();
s3.deleteObject(delreq);
standalone_cache.put(fileid, new byte[0]); // Mark in cache
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build();
s3.putObject(req, RequestBody.fromBytes(content.buf));
byte[] digest = content.buf;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(content.buf);
digest = md.digest();
} catch (NoSuchAlgorithmException nsax) {
}
// If cached and same, just return
if (Arrays.equals(digest, cacheval)) {
return true;
}
String ct = "text/plain";
if (fileid.endsWith(".json")) {
ct = "application/json";
}
else if (fileid.endsWith(".php")) {
ct = "application/x-httpd-php";
}
else if (fileid.endsWith(".html")) {
ct = "text/html";
}
else if (fileid.endsWith(".css")) {
ct = "text/css";
}
else if (fileid.endsWith(".js")) {
ct = "application/x-javascript";
}
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(fileid).contentType(ct).build();
s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
standalone_cache.put(fileid, digest);
}
done = true;
} catch (AwsServiceException x) {
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
}
return done;
}
}

37
bukkit-helper/.project
View File

@ -2,32 +2,35 @@
<projectDescription>
<name>Dynmap(Spigot-Common)</name>
<comment>bukkit-helper</comment>
<projects/>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments/>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments/>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments/>
</buildCommand>
</buildSpec>
<linkedResources/>
<filteredResources>
<filter>
<id>1</id>
<name></name>
<type>30</type>
<name/>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>