Merge pull request #4030 from ChimneySwift/generic-s3

Allow generic S3 endpoints for alternative services
2024-11-14 06:16:46 +01:00 · 2023-12-12 22:02:03 -06:00 · 2023-12-12 22:02:03 -06:00 · b0e56d3e5a
commit b0e56d3e5a
parent 20700c21b8 1ba6dd4683
25 changed files with 56 additions and 18 deletions
--- a/DynmapCore/build.gradle
+++ b/DynmapCore/build.gradle
@ -19,11 +19,13 @@ dependencies {
    implementation 'org.yaml:snakeyaml:1.23'	// DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
    implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
    implementation 'org.postgresql:postgresql:42.2.18'
-    implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
-    implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
-    implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
-    implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
-    implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
+    implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT'
+    implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT'
+    implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT'
+    implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT'
+    implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT'
+    implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1'
+    implementation 'com.sun.xml.bind:jaxb-impl:3.0.0'
 }

 processResources {
@ -58,11 +60,13 @@ shadowJar {
        include(dependency('org.eclipse.jetty::'))
        include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
        include(dependency('org.postgresql:postgresql:'))
-        include(dependency('io.github.linktosriram:s3-lite-core:'))
-        include(dependency('io.github.linktosriram:s3-lite-api:'))
-        include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
-        include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
-	    include(dependency('io.github.linktosriram:s3-lite-util:'))
+        include(dependency('io.github.linktosriram.s3lite:core:'))
+        include(dependency('io.github.linktosriram.s3lite:api:'))
+        include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:'))
+        include(dependency('io.github.linktosriram.s3lite:http-client-spi:'))
+        include(dependency('io.github.linktosriram.s3lite:util:'))
+        include(dependency('jakarta.xml.bind::'))
+        include(dependency('com.sun.xml.bind::'))
        include(dependency(':DynmapCoreAPI'))
        exclude("META-INF/maven/**")
        exclude("META-INF/services/**")
--- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
+++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
@ -1,6 +1,7 @@
 package org.dynmap.storage.aws_s3;

 import java.io.IOException;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@ -139,7 +140,7 @@ public class AWSS3MapStorage extends MapStorage {
        		else {
        			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
        					.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
-        			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+        			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
        		}
    			done = true;
            } catch (S3Exception x) {
@ -221,7 +222,7 @@ public class AWSS3MapStorage extends MapStorage {
    }
    
    private String bucketname;
-    private String region;
+    private Region region;
    private String access_key_id;
    private String secret_access_key;
    private String prefix;
@ -248,10 +249,20 @@ public class AWSS3MapStorage extends MapStorage {
        }
        // Get our settings
        bucketname = core.configuration.getString("storage/bucketname", "dynmap");
-        region = core.configuration.getString("storage/region", "us-east-1");
        access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
        secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
        prefix = core.configuration.getString("storage/prefix", "");
+
+        // Either use a custom region, or one of the default AWS regions
+        String region_name = core.configuration.getString("storage/region", "us-east-1");
+        String region_endpoint = core.configuration.getString("storage/override_endpoint", "");
+
+        if (region_endpoint.length() > 0) {
+            region = Region.of(region_name, URI.create(region_endpoint));
+        } else {
+            region = Region.fromString(region_name);
+        }
+
        if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) {
        	prefix += '/';
        }
@ -518,7 +529,7 @@ public class AWSS3MapStorage extends MapStorage {
    		}
    		else {
    			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
-    			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+    			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
    		}
 			done = true;
        } catch (S3Exception x) {
@ -571,7 +582,7 @@ public class AWSS3MapStorage extends MapStorage {
    		}
    		else {
       			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
-    			s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+    			s3.putObject(req, RequestBody.fromBytes(encImage.buf));
    		}
 			done = true;
        } catch (S3Exception x) {
@ -734,7 +745,7 @@ public class AWSS3MapStorage extends MapStorage {
    				ct = "application/x-javascript";
    			}
       			PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
-    			s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
+    			s3.putObject(req, RequestBody.fromBytes(content.buf));
        		standalone_cache.put(fileid, digest);
    		}
 			done = true;
@ -763,7 +774,7 @@ public class AWSS3MapStorage extends MapStorage {
                    if (cpoolCount < POOLSIZE) {  // Still more we can have
                        c = new DefaultS3ClientBuilder()
                        	    .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
-                        	    .region(Region.fromString(region))
+                        	    .region(region)
                        	    .httpClient(URLConnectionSdkHttpClient.create())
                        	    .build();
                        if (c == null) {
--- a/build.gradle
+++ b/build.gradle
@ -30,6 +30,7 @@ allprojects  {
     mavenLocal()
     maven { url 'https://libraries.minecraft.net/' }
     maven { url "https://oss.sonatype.org/content/repositories/releases" }
+     maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
     maven { url "https://repo.mikeprimm.com" }
     maven { url "https://repo.maven.apache.org/maven2" }
     maven { url "https://hub.spigotmc.org/nexus/content/repositories/snapshots/" }
--- a/fabric-1.14.4/src/main/resources/configuration.txt
+++ b/fabric-1.14.4/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.15.2/src/main/resources/configuration.txt
+++ b/fabric-1.15.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.16.4/src/main/resources/configuration.txt
+++ b/fabric-1.16.4/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.17.1/src/main/resources/configuration.txt
+++ b/fabric-1.17.1/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.18.2/src/main/resources/configuration.txt
+++ b/fabric-1.18.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.19.1/src/main/resources/configuration.txt
+++ b/fabric-1.19.1/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.19.3/src/main/resources/configuration.txt
+++ b/fabric-1.19.3/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.19.4/src/main/resources/configuration.txt
+++ b/fabric-1.19.4/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.19/src/main/resources/configuration.txt
+++ b/fabric-1.19/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.20.2/src/main/resources/configuration.txt
+++ b/fabric-1.20.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/fabric-1.20/src/main/resources/configuration.txt
+++ b/fabric-1.20/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.14.4/src/main/resources/configuration.txt
+++ b/forge-1.14.4/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.15.2/src/main/resources/configuration.txt
+++ b/forge-1.15.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.16.5/src/main/resources/configuration.txt
+++ b/forge-1.16.5/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.17.1/src/main/resources/configuration.txt
+++ b/forge-1.17.1/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.18.2/src/main/resources/configuration.txt
+++ b/forge-1.18.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.19.2/src/main/resources/configuration.txt
+++ b/forge-1.19.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.19.3/src/main/resources/configuration.txt
+++ b/forge-1.19.3/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.19/src/main/resources/configuration.txt
+++ b/forge-1.19/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.20.2/src/main/resources/configuration.txt
+++ b/forge-1.20.2/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/forge-1.20/src/main/resources/configuration.txt
+++ b/forge-1.20/src/main/resources/configuration.txt
@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>"  
  #prefix: ""
+  #override_endpoint: ""

 components:
  - class: org.dynmap.ClientConfigurationComponent
--- a/spigot/src/main/resources/configuration.txt
+++ b/spigot/src/main/resources/configuration.txt
@ -46,7 +46,8 @@ storage:
  #bucketname: "dynmap-bucket-name"
  #region: us-east-1
  #aws_access_key_id: "<aws-access-key-id>"
-  #aws_secret_access_key: "<aws-secret-access-key>"  
+  #aws_secret_access_key: "<aws-secret-access-key>"
+  #override_endpoint: ""
  
 components:
  - class: org.dynmap.ClientConfigurationComponent