Merge pull request #4030 from ChimneySwift/generic-s3

Allow generic S3 endpoints for alternative services

DynmapCore/build.gradle

@@ -19,11 +19,13 @@ dependencies {
     implementation 'org.yaml:snakeyaml:1.23'	// DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
     implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
     implementation 'org.postgresql:postgresql:42.2.18'
-    implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT'
-    implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT'
-    implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT'
-    implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT'
-    implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT'
+    implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1'
+    implementation 'com.sun.xml.bind:jaxb-impl:3.0.0'
 }
 
 processResources {
@@ -58,11 +60,13 @@ shadowJar {
         include(dependency('org.eclipse.jetty::'))
         include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
         include(dependency('org.postgresql:postgresql:'))
-        include(dependency('io.github.linktosriram:s3-lite-core:'))
+        include(dependency('io.github.linktosriram.s3lite:core:'))
-        include(dependency('io.github.linktosriram:s3-lite-api:'))
+        include(dependency('io.github.linktosriram.s3lite:api:'))
-        include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
+        include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:'))
-        include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
+        include(dependency('io.github.linktosriram.s3lite:http-client-spi:'))
-	    include(dependency('io.github.linktosriram:s3-lite-util:'))
+        include(dependency('io.github.linktosriram.s3lite:util:'))
+        include(dependency('jakarta.xml.bind::'))
+        include(dependency('com.sun.xml.bind::'))
         include(dependency(':DynmapCoreAPI'))
         exclude("META-INF/maven/**")
         exclude("META-INF/services/**")

DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java

@@ -1,6 +1,7 @@
 package org.dynmap.storage.aws_s3;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -139,7 +140,7 @@ public class AWSS3MapStorage extends MapStorage {
         		else {
         			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
         					.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
-        			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+        			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
         		}
     			done = true;
             } catch (S3Exception x) {
@@ -221,7 +222,7 @@ public class AWSS3MapStorage extends MapStorage {
     }
     
     private String bucketname;
-    private String region;
+    private Region region;
     private String access_key_id;
     private String secret_access_key;
     private String prefix;
@@ -248,10 +249,20 @@ public class AWSS3MapStorage extends MapStorage {
         }
         // Get our settings
         bucketname = core.configuration.getString("storage/bucketname", "dynmap");
-        region = core.configuration.getString("storage/region", "us-east-1");
         access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
         secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
         prefix = core.configuration.getString("storage/prefix", "");
+
+        // Either use a custom region, or one of the default AWS regions
+        String region_name = core.configuration.getString("storage/region", "us-east-1");
+        String region_endpoint = core.configuration.getString("storage/override_endpoint", "");
+
+        if (region_endpoint.length() > 0) {
+            region = Region.of(region_name, URI.create(region_endpoint));
+        } else {
+            region = Region.fromString(region_name);
+        }
+
         if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) {
         	prefix += '/';
         }
@@ -518,7 +529,7 @@ public class AWSS3MapStorage extends MapStorage {
     		}
     		else {
     			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
-    			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+    			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
     		}
 			done = true;
         } catch (S3Exception x) {
@@ -571,7 +582,7 @@ public class AWSS3MapStorage extends MapStorage {
     		}
     		else {
        			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
-    			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+    			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
     		}
 			done = true;
         } catch (S3Exception x) {
@@ -734,7 +745,7 @@ public class AWSS3MapStorage extends MapStorage {
     				ct = "application/x-javascript";
     			}
        			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
-    			s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
+    			s3.putObject(req, RequestBody.fromBytes(content.buf));
         		standalone_cache.put(fileid, digest);
     		}
 			done = true;
@@ -763,7 +774,7 @@ public class AWSS3MapStorage extends MapStorage {
                     if (cpoolCount < POOLSIZE) {  // Still more we can have
                         c = new DefaultS3ClientBuilder()
                         	    .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
-                        	    .region(Region.fromString(region))
+                        	    .region(region)
                         	    .httpClient(URLConnectionSdkHttpClient.create())
                         	    .build();
                         if (c == null) {

build.gradle

@@ -30,6 +30,7 @@ allprojects  {
      mavenLocal()
      maven { url 'https://libraries.minecraft.net/' }
      maven { url "https://oss.sonatype.org/content/repositories/releases" }
+     maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
      maven { url "https://repo.mikeprimm.com" }
      maven { url "https://repo.maven.apache.org/maven2" }
      maven { url "https://hub.spigotmc.org/nexus/content/repositories/snapshots/" }

fabric-1.14.4/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.15.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.16.4/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.17.1/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.18.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.19.1/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.19.3/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.19.4/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.19/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.20.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

fabric-1.20/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.14.4/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.15.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.16.5/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.17.1/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.18.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.19.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.19.3/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.19/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.20.2/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

forge-1.20/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"  
   #prefix: ""
+  #override_endpoint: ""
 
 components:
   - class: org.dynmap.ClientConfigurationComponent

spigot/src/main/resources/configuration.txt

@@ -47,6 +47,7 @@ storage:
   #region: us-east-1
   #aws_access_key_id: "<aws-access-key-id>"
   #aws_secret_access_key: "<aws-secret-access-key>"
+  #override_endpoint: ""
   
 components:
   - class: org.dynmap.ClientConfigurationComponent