Upstream
/
dynmap
mirror of https://github.com/webbukkit/dynmap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Switch to sdk v2

This commit is contained in:
Mike Primm 2022-02-18 00:40:55 -06:00
parent be43f53d81
commit 45bc02cf3a
4 changed files with 105 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
DynmapCore/build.gradle
View File

@ -17,12 +17,17 @@ dependencies {
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
implementation 'org.postgresql:postgresql:42.2.18'
implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000')
implementation 'com.amazonaws:aws-java-sdk-core'
implementation 'com.amazonaws:aws-java-sdk-s3'
implementation 'com.fasterxml.jackson.core:jackson-core'
implementation 'com.fasterxml.jackson.core:jackson-databind'
implementation 'com.fasterxml.jackson.core:jackson-annotations'
implementation 'software.amazon.awssdk:s3:2.17.132'
implementation 'software.amazon.awssdk:aws-core:2.17.132'
implementation 'software.amazon.awssdk:sdk-core:2.17.132'
implementation 'software.amazon.awssdk:utils:2.17.132'
implementation 'software.amazon.awssdk:http-client-spi:2.17.132'
implementation 'software.amazon.awssdk:profiles:2.17.132'
implementation 'software.amazon.awssdk:regions:2.17.132'
implementation 'software.amazon.awssdk:auth:2.17.132'
implementation 'software.amazon.awssdk:metrics-spi:2.17.132'
implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132'
implementation 'software.amazon.awssdk:protocol-core:2.17.132'
}
processResources {
@ -57,12 +62,17 @@ shadowJar {
include(dependency('org.eclipse.jetty::'))
include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
include(dependency('org.postgresql:postgresql:'))
include(dependency('com.amazonaws:aws-java-sdk-bom:'))
include(dependency('com.amazonaws:aws-java-sdk-core:'))
include(dependency('com.amazonaws:aws-java-sdk-s3:'))
include(dependency('com.fasterxml.jackson.core:jackson-core:'))
include(dependency('com.fasterxml.jackson.core:jackson-databind:'))
include(dependency('com.fasterxml.jackson.core:jackson-annotations:'))
include(dependency('software.amazon.awssdk:s3:'))
include(dependency('software.amazon.awssdk:aws-core:'))
include(dependency('software.amazon.awssdk:sdk-core:'))
include(dependency('software.amazon.awssdk:utils:'))
include(dependency('software.amazon.awssdk:http-client-spi:'))
include(dependency('software.amazon.awssdk:profiles:'))
include(dependency('software.amazon.awssdk:regions:'))
include(dependency('software.amazon.awssdk:auth:'))
include(dependency('software.amazon.awssdk:metrics-spi:'))
include(dependency('software.amazon.awssdk:aws-xml-protocol:'))
include(dependency('software.amazon.awssdk:protocol-core:'))
include(dependency(':DynmapCoreAPI'))
exclude("META-INF/maven/**")
exclude("META-INF/services/**")
@ -73,8 +83,7 @@ shadowJar {
relocate('org.owasp.html', 'org.dynmap.org.owasp.html')
relocate('javax.servlet', 'org.dynmap.javax.servlet' )
relocate('org.postgresql', 'org.dynmap.org.postgresql')
relocate('com.amazonaws', 'org.dynmap.com.amazonaws')
relocate('com.fasterxml', 'org.dynmap.com.fasterxml')
relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk')
destinationDir = file '../target'
classifier = ''
}

164
DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
View File

@ -1,13 +1,6 @@
package org.dynmap.storage.aws_s3;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -28,19 +21,25 @@ import org.dynmap.storage.MapStorageTileSearchEndCB;
import org.dynmap.utils.BufferInputStream;
import org.dynmap.utils.BufferOutputStream;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectAclRequest;
import software.amazon.awssdk.services.s3.model.GetObjectAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AWSS3MapStorage extends MapStorage {
public class StorageTile extends MapStorageTile {
@ -62,10 +61,11 @@ public class AWSS3MapStorage extends MapStorage {
public boolean exists() {
boolean exists = false;
try {
AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey);
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null)
exists = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
}
return exists;
@ -80,23 +80,20 @@ public class AWSS3MapStorage extends MapStorage {
public TileRead read() {
AWSS3MapStorage.this.getWriteLock(baseKey);
try {
S3Object obj = s3.getObject(bucketname, baseKey);
GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build();
ResponseBytes<GetObjectResponse> obj = s3.getObjectAsBytes(req);
if (obj != null) {
ObjectMetadata md = obj.getObjectMetadata();
GetObjectResponse rsp = obj.response();
TileRead tr = new TileRead();
byte[] buf = new byte[(int) md.getContentLength()];
InputStream fis = obj.getObjectContent();
fis.read(buf, 0, buf.length); // Read whole thing
byte[] buf = obj.asByteArray();
tr.image = new BufferInputStream(buf);
tr.format = ImageEncoding.fromContentType(md.getContentType());
tr.hashCode = md.getContentMD5().hashCode();
tr.lastModified = md.getLastModified().getTime();
tr.format = ImageEncoding.fromContentType(rsp.contentType());
tr.hashCode = rsp.eTag().hashCode();
tr.lastModified = rsp.lastModified().toEpochMilli();
return tr;
}
} catch (IOException x) {
Log.severe("AWS Exception", x);
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
@ -110,15 +107,15 @@ public class AWSS3MapStorage extends MapStorage {
AWSS3MapStorage.this.getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey);
DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(req);
}
else {
ObjectMetadata md = new ObjectMetadata();
md.setContentType(map.getImageFormat().getEncoding().getContentType());
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey);
@ -200,7 +197,7 @@ public class AWSS3MapStorage extends MapStorage {
private String bucketname;
private String region;
private String profile_id;
private AmazonS3 s3;
private S3Client s3;
public AWSS3MapStorage() {
}
@ -217,24 +214,22 @@ public class AWSS3MapStorage extends MapStorage {
// Get our settings
bucketname = core.configuration.getString("storage/bucketname", "dynmap");
region = core.configuration.getString("storage/region", "us-east-1");
profile_id = core.configuration.getString("storage/aws_profile_id", System.getenv("AWS_PROFILE"));
try {
// Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id);
s3 = AmazonS3ClientBuilder.standard().withRegion(region)
//.withCredentials(new ProfileCredentialsProvider("profile " + profile_id))
.build();
s3 = S3Client.builder().region(Region.of(region)).build();
if (s3 == null) {
Log.severe("Error creating S3 access client");
return false;
}
// Make sure bucket exists and get ACL
AccessControlList bucketACL = s3.getBucketAcl(bucketname);
GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build();
GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr);
if (bucketACL == null) {
Log.severe("Error: cannot find or access S3 bucket");
return false;
}
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
return false;
}
@ -296,10 +291,11 @@ public class AWSS3MapStorage extends MapStorage {
private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
try {
ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey);
List<S3ObjectSummary> objects = result.getObjectSummaries();
for (S3ObjectSummary os : objects) {
String key = os.getKey();
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.contents();
for (S3Object os : objects) {
String key = os.key();
key = key.substring(basekey.length()); // Strip off base
// Parse the extension
String ext = null;
@ -341,7 +337,7 @@ public class AWSS3MapStorage extends MapStorage {
}
}
}
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
}
if(cbEnd != null) {
@ -388,25 +384,26 @@ public class AWSS3MapStorage extends MapStorage {
private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
try {
ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey);
List<S3ObjectSummary> objects = result.getObjectSummaries();
ArrayList<String> keys = new ArrayList<String>();
for (S3ObjectSummary os : objects) {
String key = os.getKey();
keys.add(key);
ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
ListObjectsV2Response result = s3.listObjectsV2(req);
List<S3Object> objects = result.contents();
ArrayList<ObjectIdentifier> keys = new ArrayList<ObjectIdentifier>();
for (S3Object os : objects) {
String key = os.key();
keys.add(ObjectIdentifier.builder().key(key).build());
if (keys.size() >= 100) {
DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0]));
s3.deleteObjects(dor);
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(delreq);
keys.clear();
}
}
// Any left?
if (keys.size() > 0) {
DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0]));
s3.deleteObjects(dor);
DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(delreq);
keys.clear();
}
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
}
}
@ -437,15 +434,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey);
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
ObjectMetadata md = new ObjectMetadata();
md.setContentType("image/png");
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
@ -464,10 +461,11 @@ public class AWSS3MapStorage extends MapStorage {
String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
boolean exists = false;
try {
AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey);
GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null)
exists = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
}
return exists;
@ -480,15 +478,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey);
try {
if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey);
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
ObjectMetadata md = new ObjectMetadata();
md.setContentType("image/png");
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
@ -508,15 +506,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey);
try {
if (content == null) { // Delete?
s3.deleteObject(bucketname, baseKey);
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
ObjectMetadata md = new ObjectMetadata();
md.setContentType("application/json");
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), md);
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build();
s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8)));
}
done = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);
@ -571,15 +569,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey);
try {
if (content == null) { // Delete?
s3.deleteObject(bucketname, baseKey);
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
ObjectMetadata md = new ObjectMetadata();
md.setContentType("text/plain");
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.buf), md);
PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build();
s3.putObject(req, RequestBody.fromBytes(content.buf));
}
done = true;
} catch (AmazonServiceException x) {
} catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} finally {
releaseWriteLock(baseKey);

2
bukkit-helper/.settings/org.eclipse.jdt.core.prefs
View File

@ -1,5 +1,5 @@
#
#Thu Feb 17 18:10:31 CST 2022
#Thu Feb 17 22:40:49 CST 2022
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8

2
spigot/src/main/resources/configuration.txt
View File

@ -45,8 +45,6 @@ storage:
#type: aws_s3
#bucketname: dynmap
#region: us-east-1
#aws_access_key_id: <access-key>
#aws_secret_access_key: <secret-access-key>
components: