From a669d75de00da45f0e3c8098cea22a73f17d45ed Mon Sep 17 00:00:00 2001 From: ChimneySwift <28179563+ChimneySwift@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:19:16 +1000 Subject: [PATCH 1/4] Update s3-lite client to 0.0.2-SNAPSHOT version --- DynmapCore/build.gradle | 20 +++++++++---------- .../storage/aws_s3/AWSS3MapStorage.java | 8 ++++---- build.gradle | 1 + 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index 5d731e32..3e4347c3 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -19,11 +19,11 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' - implementation 'io.github.linktosriram:s3-lite-core:0.2.0' - implementation 'io.github.linktosriram:s3-lite-api:0.2.0' - implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0' - implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0' - implementation 'io.github.linktosriram:s3-lite-util:0.2.0' + implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT' + implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT' + implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT' + implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT' + implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT' } processResources { @@ -58,11 +58,11 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) - include(dependency('io.github.linktosriram:s3-lite-core:')) - include(dependency('io.github.linktosriram:s3-lite-api:')) - include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:')) - include(dependency('io.github.linktosriram:s3-lite-http-client-spi:')) - include(dependency('io.github.linktosriram:s3-lite-util:')) + include(dependency('io.github.linktosriram.s3lite:core:')) + include(dependency('io.github.linktosriram.s3lite:api:')) + include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:')) + include(dependency('io.github.linktosriram.s3lite:http-client-spi:')) + include(dependency('io.github.linktosriram.s3lite:util:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index e954279f..0dfe8660 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -139,7 +139,7 @@ public class AWSS3MapStorage extends MapStorage { else { PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()) .addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; } catch (S3Exception x) { @@ -518,7 +518,7 @@ public class AWSS3MapStorage extends MapStorage { } else { PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; } catch (S3Exception x) { @@ -571,7 +571,7 @@ public class AWSS3MapStorage extends MapStorage { } else { PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build(); - s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len)); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; } catch (S3Exception x) { @@ -734,7 +734,7 @@ public class AWSS3MapStorage extends MapStorage { ct = "application/x-javascript"; } PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build(); - s3.putObject(req, RequestBody.fromBytes(content.buf, content.len)); + s3.putObject(req, RequestBody.fromBytes(content.buf)); standalone_cache.put(fileid, digest); } done = true; diff --git a/build.gradle b/build.gradle index 1d5cf8f9..de7cb9a3 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,7 @@ allprojects { mavenLocal() maven { url 'https://libraries.minecraft.net/' } maven { url "https://oss.sonatype.org/content/repositories/releases" } + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } maven { url "https://repo.mikeprimm.com" } maven { url "https://repo.maven.apache.org/maven2" } maven { url "https://hub.spigotmc.org/nexus/content/repositories/snapshots/" } From 914fc5a10c73ae848d8cb343297f805fe85dfe5a Mon Sep 17 00:00:00 2001 From: ChimneySwift <28179563+ChimneySwift@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:56:35 +1000 Subject: [PATCH 2/4] Allow for custom S3 endpoints --- .../dynmap/storage/aws_s3/AWSS3MapStorage.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 0dfe8660..577ed2fc 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -1,6 +1,7 @@ package org.dynmap.storage.aws_s3; import java.io.IOException; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -221,7 +222,7 @@ public class AWSS3MapStorage extends MapStorage { } private String bucketname; - private String region; + private Region region; private String access_key_id; private String secret_access_key; private String prefix; @@ -248,10 +249,20 @@ public class AWSS3MapStorage extends MapStorage { } // Get our settings bucketname = core.configuration.getString("storage/bucketname", "dynmap"); - region = core.configuration.getString("storage/region", "us-east-1"); access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID")); secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY")); prefix = core.configuration.getString("storage/prefix", ""); + + // Either use a custom region, or one of the default AWS regions + String region_name = core.configuration.getString("storage/region", "us-east-1"); + String region_endpoint = core.configuration.getString("storage/override_endpoint", ""); + + if (region_endpoint.length() > 0) { + region = Region.of(region_name, URI.create(region_endpoint)); + } else { + region = Region.fromString(region_name); + } + if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) { prefix += '/'; } @@ -763,7 +774,7 @@ public class AWSS3MapStorage extends MapStorage { if (cpoolCount < POOLSIZE) { // Still more we can have c = new DefaultS3ClientBuilder() .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key)) - .region(Region.fromString(region)) + .region(region) .httpClient(URLConnectionSdkHttpClient.create()) .build(); if (c == null) { From d16fdc82756ae47a2b23aa9768ea55547e12cb9e Mon Sep 17 00:00:00 2001 From: ChimneySwift <28179563+ChimneySwift@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:07:28 +1000 Subject: [PATCH 3/4] Add override_endpoint setting to configuration files --- fabric-1.14.4/src/main/resources/configuration.txt | 1 + fabric-1.15.2/src/main/resources/configuration.txt | 1 + fabric-1.16.4/src/main/resources/configuration.txt | 1 + fabric-1.17.1/src/main/resources/configuration.txt | 1 + fabric-1.18.2/src/main/resources/configuration.txt | 1 + fabric-1.19.1/src/main/resources/configuration.txt | 1 + fabric-1.19.3/src/main/resources/configuration.txt | 1 + fabric-1.19.4/src/main/resources/configuration.txt | 1 + fabric-1.19/src/main/resources/configuration.txt | 1 + fabric-1.20.2/src/main/resources/configuration.txt | 1 + fabric-1.20/src/main/resources/configuration.txt | 1 + forge-1.14.4/src/main/resources/configuration.txt | 1 + forge-1.15.2/src/main/resources/configuration.txt | 1 + forge-1.16.5/src/main/resources/configuration.txt | 1 + forge-1.17.1/src/main/resources/configuration.txt | 1 + forge-1.18.2/src/main/resources/configuration.txt | 1 + forge-1.19.2/src/main/resources/configuration.txt | 1 + forge-1.19.3/src/main/resources/configuration.txt | 1 + forge-1.19/src/main/resources/configuration.txt | 1 + forge-1.20.2/src/main/resources/configuration.txt | 1 + forge-1.20/src/main/resources/configuration.txt | 1 + spigot/src/main/resources/configuration.txt | 3 ++- 22 files changed, 23 insertions(+), 1 deletion(-) diff --git a/fabric-1.14.4/src/main/resources/configuration.txt b/fabric-1.14.4/src/main/resources/configuration.txt index 20eade27..215aa4aa 100644 --- a/fabric-1.14.4/src/main/resources/configuration.txt +++ b/fabric-1.14.4/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.15.2/src/main/resources/configuration.txt b/fabric-1.15.2/src/main/resources/configuration.txt index 20eade27..215aa4aa 100644 --- a/fabric-1.15.2/src/main/resources/configuration.txt +++ b/fabric-1.15.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.16.4/src/main/resources/configuration.txt b/fabric-1.16.4/src/main/resources/configuration.txt index 794b1291..d22579b4 100644 --- a/fabric-1.16.4/src/main/resources/configuration.txt +++ b/fabric-1.16.4/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.17.1/src/main/resources/configuration.txt b/fabric-1.17.1/src/main/resources/configuration.txt index 794b1291..d22579b4 100644 --- a/fabric-1.17.1/src/main/resources/configuration.txt +++ b/fabric-1.17.1/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.18.2/src/main/resources/configuration.txt b/fabric-1.18.2/src/main/resources/configuration.txt index 0d47a67f..45be04a6 100644 --- a/fabric-1.18.2/src/main/resources/configuration.txt +++ b/fabric-1.18.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.19.1/src/main/resources/configuration.txt b/fabric-1.19.1/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.19.1/src/main/resources/configuration.txt +++ b/fabric-1.19.1/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.19.3/src/main/resources/configuration.txt b/fabric-1.19.3/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.19.3/src/main/resources/configuration.txt +++ b/fabric-1.19.3/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.19.4/src/main/resources/configuration.txt b/fabric-1.19.4/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.19.4/src/main/resources/configuration.txt +++ b/fabric-1.19.4/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.19/src/main/resources/configuration.txt b/fabric-1.19/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.19/src/main/resources/configuration.txt +++ b/fabric-1.19/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.20.2/src/main/resources/configuration.txt b/fabric-1.20.2/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.20.2/src/main/resources/configuration.txt +++ b/fabric-1.20.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/fabric-1.20/src/main/resources/configuration.txt b/fabric-1.20/src/main/resources/configuration.txt index 51067ee9..19d48831 100644 --- a/fabric-1.20/src/main/resources/configuration.txt +++ b/fabric-1.20/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.14.4/src/main/resources/configuration.txt b/forge-1.14.4/src/main/resources/configuration.txt index 3b1d0c66..86bfc113 100644 --- a/forge-1.14.4/src/main/resources/configuration.txt +++ b/forge-1.14.4/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.15.2/src/main/resources/configuration.txt b/forge-1.15.2/src/main/resources/configuration.txt index 3b1d0c66..86bfc113 100644 --- a/forge-1.15.2/src/main/resources/configuration.txt +++ b/forge-1.15.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.16.5/src/main/resources/configuration.txt b/forge-1.16.5/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.16.5/src/main/resources/configuration.txt +++ b/forge-1.16.5/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.17.1/src/main/resources/configuration.txt b/forge-1.17.1/src/main/resources/configuration.txt index 06aed6d7..fe4ae531 100644 --- a/forge-1.17.1/src/main/resources/configuration.txt +++ b/forge-1.17.1/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.18.2/src/main/resources/configuration.txt b/forge-1.18.2/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.18.2/src/main/resources/configuration.txt +++ b/forge-1.18.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.19.2/src/main/resources/configuration.txt b/forge-1.19.2/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.19.2/src/main/resources/configuration.txt +++ b/forge-1.19.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.19.3/src/main/resources/configuration.txt b/forge-1.19.3/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.19.3/src/main/resources/configuration.txt +++ b/forge-1.19.3/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.19/src/main/resources/configuration.txt b/forge-1.19/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.19/src/main/resources/configuration.txt +++ b/forge-1.19/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.20.2/src/main/resources/configuration.txt b/forge-1.20.2/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.20.2/src/main/resources/configuration.txt +++ b/forge-1.20.2/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/forge-1.20/src/main/resources/configuration.txt b/forge-1.20/src/main/resources/configuration.txt index 300412ab..2f2259ed 100644 --- a/forge-1.20/src/main/resources/configuration.txt +++ b/forge-1.20/src/main/resources/configuration.txt @@ -47,6 +47,7 @@ storage: #aws_access_key_id: "" #aws_secret_access_key: "" #prefix: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index 73529b49..c0a2739d 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -46,7 +46,8 @@ storage: #bucketname: "dynmap-bucket-name" #region: us-east-1 #aws_access_key_id: "" - #aws_secret_access_key: "" + #aws_secret_access_key: "" + #override_endpoint: "" components: - class: org.dynmap.ClientConfigurationComponent From 1ba6dd4683d6f79eacc58121cb87e1dc2cf6f5d9 Mon Sep 17 00:00:00 2001 From: ChimneySwift <28179563+ChimneySwift@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:21:35 +1000 Subject: [PATCH 4/4] Bundle xml.bind as this is used by s3-lite --- DynmapCore/build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index 3e4347c3..adf499da 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT' implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT' implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT' + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1' + implementation 'com.sun.xml.bind:jaxb-impl:3.0.0' } processResources { @@ -62,7 +64,9 @@ shadowJar { include(dependency('io.github.linktosriram.s3lite:api:')) include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:')) include(dependency('io.github.linktosriram.s3lite:http-client-spi:')) - include(dependency('io.github.linktosriram.s3lite:util:')) + include(dependency('io.github.linktosriram.s3lite:util:')) + include(dependency('jakarta.xml.bind::')) + include(dependency('com.sun.xml.bind::')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**")