From 68aeaa65e58713f5d3c7d86502c4646758cde26e Mon Sep 17 00:00:00 2001 From: Skye Date: Fri, 23 Feb 2024 16:03:34 +0900 Subject: [PATCH 1/4] S3 support --- .../common/config/storage/S3Config.java | 50 +++ .../common/config/storage/StorageType.java | 4 +- .../bluemap/config/storages/s3.conf | 32 ++ BlueMapCore/build.gradle.kts | 1 + .../bluemap/core/storage/s3/S3Storage.java | 342 ++++++++++++++++++ .../core/storage/s3/S3StorageSettings.java | 14 + 6 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java create mode 100644 BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf create mode 100644 BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java create mode 100644 BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java new file mode 100644 index 00000000..ad4b4340 --- /dev/null +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java @@ -0,0 +1,50 @@ +package de.bluecolored.bluemap.common.config.storage; + +import de.bluecolored.bluemap.api.debug.DebugDump; +import de.bluecolored.bluemap.core.storage.Compression; +import de.bluecolored.bluemap.core.storage.s3.S3StorageSettings; +import org.spongepowered.configurate.objectmapping.ConfigSerializable; + +import java.util.Optional; + +@SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) +@DebugDump +@ConfigSerializable +public class S3Config extends StorageConfig implements S3StorageSettings { + private String endpoint = null; + private String region = ""; + private String accessKey = ""; + private String secretKey = ""; + private String bucket = ""; + private Compression compression = Compression.GZIP; + + @Override + public Optional getEndpoint() { + return Optional.ofNullable(endpoint); + } + + @Override + public String getRegion() { + return region; + } + + @Override + public String getAccessKey() { + return accessKey; + } + + @Override + public String getSecretKey() { + return secretKey; + } + + @Override + public String getBucket() { + return bucket; + } + + @Override + public Compression getCompression() { + return compression; + } +} diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java index 20a24e94..aae515d7 100644 --- a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java @@ -26,12 +26,14 @@ package de.bluecolored.bluemap.common.config.storage; import de.bluecolored.bluemap.core.storage.Storage; import de.bluecolored.bluemap.core.storage.file.FileStorage; +import de.bluecolored.bluemap.core.storage.s3.S3Storage; import de.bluecolored.bluemap.core.storage.sql.SQLStorage; public enum StorageType { FILE (FileConfig.class, FileStorage::new), - SQL (SQLConfig.class, SQLStorage::create); + SQL (SQLConfig.class, SQLStorage::create), + S3 (S3Config.class, S3Storage::new); private final Class configType; private final StorageFactory storageFactory; diff --git a/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf b/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf new file mode 100644 index 00000000..b882b9d8 --- /dev/null +++ b/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf @@ -0,0 +1,32 @@ +## ## +## BlueMap ## +## Storage-Config ## +## ## + +# The storage-type of this storage. +# Depending on this setting, different config-entries are allowed/expected in this config file. +# Don't change this value! (If you want a different storage-type, check out the other example-configs) +storage-type: S3 + +# This is the S3 endpoint that bluemap will use. +# If you're using a third-party S3 compatible provider, +# uncomment this and set it to your provider's S3 endpoint URL. +#endpoint: "https://s3.us-west-004.backblazeb2.com" + +# This is the S3 region that bluemap will use. +# Set this to the AWS region of your S3 bucket. +region: "us-east-1" + +# The S3 credentials that bluemap will use. +access-key: "changeme" +secret-key: "changeme" + +# The S3 bucket name. +bucket: "bluemap" + +# The compression-type that bluemap will use to compress generated map-data. +# Available compression-types are: +# - GZIP +# - NONE +# The default is: GZIP +compression: GZIP diff --git a/BlueMapCore/build.gradle.kts b/BlueMapCore/build.gradle.kts index da0d6399..73131794 100644 --- a/BlueMapCore/build.gradle.kts +++ b/BlueMapCore/build.gradle.kts @@ -70,6 +70,7 @@ dependencies { api ("org.apache.commons:commons-dbcp2:2.9.0") api ("io.airlift:aircompressor:0.24") api ("org.lz4:lz4-java:1.8.0") + api ("software.amazon.awssdk:s3:2.24.9") api ("de.bluecolored.bluemap.api:BlueMapAPI") diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java new file mode 100644 index 00000000..76e5ba6d --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java @@ -0,0 +1,342 @@ +package de.bluecolored.bluemap.core.storage.s3; + +import com.flowpowered.math.vector.Vector2i; +import de.bluecolored.bluemap.core.storage.*; +import de.bluecolored.bluemap.core.util.OnCloseOutputStream; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.*; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class S3Storage extends Storage { + private boolean closed = false; + private final S3Client client; + private final Compression hiresCompression; + private final String bucket; + public S3Storage(S3StorageSettings settings) { + AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), ""); + var builder = S3Client + .builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) + .region(Region.of(settings.getRegion())); + if (settings.getEndpoint().isPresent()) { + builder = builder.endpointOverride(URI.create(settings.getEndpoint().get())); + } + this.client = builder.build(); + this.hiresCompression = settings.getCompression(); + this.bucket = settings.getBucket(); + } + @Override + public void initialize() throws IOException {} + + @Override + public OutputStream writeMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + + OutputStream pipeOut = makeUploadStream(path, getFileMime(lod)); + return new BufferedOutputStream(compression.compress(pipeOut)); + } + + @Override + public Optional readMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + try { + InputStream is = client.getObject( + GetObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new CompressedInputStream(is, compression)); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public Optional readMapTileInfo(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + try { + var info = client.headObject( + HeadObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new TileInfo() { + @Override + public CompressedInputStream readMapTile() throws IOException { + return S3Storage.this.readMapTile(mapId, lod, tile) + .orElseThrow(() -> new IOException("Tile no longer present!")); + } + + @Override + public Compression getCompression() { + return compression; + } + + @Override + public long getSize() { + return info.contentLength(); + } + + @Override + public long getLastModified() { + return info.lastModified().getEpochSecond(); + } + }); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public void deleteMapTile(String mapId, int lod, Vector2i tile) throws IOException { + String path = getFilePath(mapId, lod, tile); + try { + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public OutputStream writeMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + + OutputStream pipeOut = makeUploadStream(path, null); + return new BufferedOutputStream(pipeOut); + } + + private OutputStream makeUploadStream(String path, String FileMime) { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + return new OnCloseOutputStream(new BufferedOutputStream(byteOut), () -> + client.putObject( + PutObjectRequest + .builder() + .bucket(bucket) + .key(path) + .contentType(FileMime) + .build(), + RequestBody.fromBytes(byteOut.toByteArray()) + ) + ); + } + + @Override + public Optional readMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + InputStream is = client.getObject( + GetObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(is); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public Optional readMetaInfo(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + var info = client.headObject( + HeadObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new MetaInfo() { + @Override + public InputStream readMeta() throws IOException { + return S3Storage.this.readMeta(mapId, name) + .orElseThrow(() -> new IOException("Meta no longer present!")); + } + + @Override + public long getSize() { + return info.contentLength(); + } + }); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public void deleteMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public void purgeMap(String mapId, Function onProgress) throws IOException { + String directory = getFilePath(mapId); + try { + var files = client.listObjectsV2Paginator( + ListObjectsV2Request + .builder() + .bucket(bucket) + .prefix(directory + "/") + .build() + ); + var filesList = files.contents().stream().collect(Collectors.toList()); + for (int i = 0; i < filesList.size(); i++) { + var file = filesList.get(i); + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(file.key()) + .build() + ); + + if (!onProgress.apply( + new ProgressInfo((double) (i + 1) / filesList.size()) + )) return; + } + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public Collection collectMapIds() throws IOException { + try { + var files = client.listObjectsV2Paginator( + ListObjectsV2Request + .builder() + .bucket(bucket) + .delimiter("/") + .build() + ); + List ids = new ArrayList<>(); + for (var file: files.contents()) { + String id = file.key().split("/")[0]; + if (!ids.contains(id)) { + ids.add(id); + } + } + return ids; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public void close() throws IOException { + client.close(); + closed = true; + } + + public String getFileMime(int lod){ + if (lod == 0) { + if (hiresCompression.getFileSuffix().equals(".gz")) { + return "application/gzip"; + } else if (hiresCompression == Compression.NONE) { + return "application/json"; + } else { + return "application/octet-stream"; + } + } else { + return "image/png"; + } + } + + public String getFilePath(String mapId, int lod, Vector2i tile){ + String path = "x" + tile.getX() + "z" + tile.getY(); + char[] cs = path.toCharArray(); + List folders = new ArrayList<>(); + StringBuilder folder = new StringBuilder(); + for (char c : cs){ + folder.append(c); + if (c >= '0' && c <= '9'){ + folders.add(folder.toString()); + folder.delete(0, folder.length()); + } + } + String fileName = folders.remove(folders.size() - 1); + + StringBuilder p = new StringBuilder(getFilePath(mapId) + "/tiles/" + lod); + for (String s : folders){ + p.append("/").append(s); + } + + if (lod == 0) { + return p + "/" + (fileName + ".json" + hiresCompression.getFileSuffix()); + } else { + return p + "/" + (fileName + ".png"); + } + } + + public String getFilePath(String mapId) { + return mapId; + } + + public String getMetaFilePath(String mapId, String name) { + return getFilePath(mapId) + "/" + escapeMetaName(name); + } +} diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java new file mode 100644 index 00000000..d62bff0b --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java @@ -0,0 +1,14 @@ +package de.bluecolored.bluemap.core.storage.s3; + +import de.bluecolored.bluemap.core.storage.Compression; + +import java.util.Optional; + +public interface S3StorageSettings { + Optional getEndpoint(); + String getRegion(); + String getAccessKey(); + String getSecretKey(); + String getBucket(); + Compression getCompression(); +} \ No newline at end of file From 09473e7b721694e2fce348523d59a453165b1ea4 Mon Sep 17 00:00:00 2001 From: Skye Date: Fri, 23 Feb 2024 18:01:58 +0900 Subject: [PATCH 2/4] S3 storage: that should be commonPrefixes, whoops --- .../de/bluecolored/bluemap/core/storage/s3/S3Storage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java index 76e5ba6d..e1f4cbf8 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java @@ -269,8 +269,8 @@ public class S3Storage extends Storage { .build() ); List ids = new ArrayList<>(); - for (var file: files.contents()) { - String id = file.key().split("/")[0]; + for (var file: files.commonPrefixes()) { + String id = file.prefix().split("/")[0]; if (!ids.contains(id)) { ids.add(id); } From 43adac0dbff976a8778f0fba1e5b6e07db617e01 Mon Sep 17 00:00:00 2001 From: Skye Date: Fri, 23 Feb 2024 18:30:38 +0900 Subject: [PATCH 3/4] S3 storage: fix issues from code review --- .../common/config/storage/S3Config.java | 11 ++-- .../bluemap/core/storage/s3/S3Storage.java | 62 ++++++++----------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java index ad4b4340..cefdfa4f 100644 --- a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java @@ -8,15 +8,14 @@ import org.spongepowered.configurate.objectmapping.ConfigSerializable; import java.util.Optional; @SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) -@DebugDump @ConfigSerializable public class S3Config extends StorageConfig implements S3StorageSettings { - private String endpoint = null; - private String region = ""; - private String accessKey = ""; + @DebugDump private String endpoint = null; + @DebugDump private String region = ""; + @DebugDump private String accessKey = ""; private String secretKey = ""; - private String bucket = ""; - private Compression compression = Compression.GZIP; + @DebugDump private String bucket = ""; + @DebugDump private Compression compression = Compression.GZIP; @Override public Optional getEndpoint() { diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java index e1f4cbf8..c4bc294b 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java @@ -5,6 +5,7 @@ import de.bluecolored.bluemap.core.storage.*; import de.bluecolored.bluemap.core.util.OnCloseOutputStream; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -62,10 +63,9 @@ public class S3Storage extends Storage { .build() ); return Optional.of(new CompressedInputStream(is, compression)); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return Optional.empty(); - } + } catch (NoSuchKeyException e) { + return Optional.empty(); + } catch (SdkException e) { throw new IOException(e); } } @@ -104,10 +104,9 @@ public class S3Storage extends Storage { return info.lastModified().getEpochSecond(); } }); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return Optional.empty(); - } + } catch (NoSuchKeyException e) { + return Optional.empty(); + } catch (SdkException e) { throw new IOException(e); } } @@ -123,10 +122,8 @@ public class S3Storage extends Storage { .key(path) .build() ); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return; - } + } catch (NoSuchKeyException ignored) { + } catch (SdkException e) { throw new IOException(e); } } @@ -166,10 +163,9 @@ public class S3Storage extends Storage { .build() ); return Optional.of(is); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return Optional.empty(); - } + } catch (NoSuchKeyException e) { + return Optional.empty(); + } catch (SdkException e) { throw new IOException(e); } } @@ -197,10 +193,9 @@ public class S3Storage extends Storage { return info.contentLength(); } }); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return Optional.empty(); - } + } catch (NoSuchKeyException e) { + return Optional.empty(); + } catch (SdkException e) { throw new IOException(e); } } @@ -216,10 +211,8 @@ public class S3Storage extends Storage { .key(path) .build() ); - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return; - } + } catch (NoSuchKeyException ignored) { + } catch (SdkException e) { throw new IOException(e); } } @@ -238,22 +231,21 @@ public class S3Storage extends Storage { var filesList = files.contents().stream().collect(Collectors.toList()); for (int i = 0; i < filesList.size(); i++) { var file = filesList.get(i); - client.deleteObject( - DeleteObjectRequest - .builder() - .bucket(bucket) - .key(file.key()) - .build() - ); + try { + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(file.key()) + .build() + ); + } catch (NoSuchKeyException ignored) {} if (!onProgress.apply( new ProgressInfo((double) (i + 1) / filesList.size()) )) return; } - } catch (Exception e) { - if (e instanceof NoSuchKeyException) { - return; - } + } catch (SdkException e) { throw new IOException(e); } } From dee263d96ae7a365cfdb657c73830e4a3d8f4e09 Mon Sep 17 00:00:00 2001 From: Skye Date: Sat, 24 Feb 2024 12:57:45 +0900 Subject: [PATCH 4/4] S3 storage: use s3-lite --- BlueMapCore/build.gradle.kts | 3 +- .../bluemap/core/storage/s3/S3Storage.java | 109 ++++++++++-------- 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/BlueMapCore/build.gradle.kts b/BlueMapCore/build.gradle.kts index 73131794..479a6687 100644 --- a/BlueMapCore/build.gradle.kts +++ b/BlueMapCore/build.gradle.kts @@ -70,7 +70,8 @@ dependencies { api ("org.apache.commons:commons-dbcp2:2.9.0") api ("io.airlift:aircompressor:0.24") api ("org.lz4:lz4-java:1.8.0") - api ("software.amazon.awssdk:s3:2.24.9") + api ("com.github.vgskye.s3-lite:core:4a9e099bf8") + api ("com.github.vgskye.s3-lite:http-client-url-connection:4a9e099bf8") api ("de.bluecolored.bluemap.api:BlueMapAPI") diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java index c4bc294b..a211a52d 100644 --- a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java @@ -3,22 +3,27 @@ package de.bluecolored.bluemap.core.storage.s3; import com.flowpowered.math.vector.Vector2i; import de.bluecolored.bluemap.core.storage.*; import de.bluecolored.bluemap.core.util.OnCloseOutputStream; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.*; +import io.github.linktosriram.s3lite.api.client.S3Client; +import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException; +import io.github.linktosriram.s3lite.api.exception.S3Exception; +import io.github.linktosriram.s3lite.api.region.Region; +import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest; +import io.github.linktosriram.s3lite.api.request.GetObjectRequest; +import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request; +import io.github.linktosriram.s3lite.api.request.PutObjectRequest; +import io.github.linktosriram.s3lite.api.response.CommonPrefix; +import io.github.linktosriram.s3lite.api.response.ListObjectsV2ResponsePager; +import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials; +import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder; +import io.github.linktosriram.s3lite.http.spi.request.RequestBody; +import io.github.linktosriram.s3lite.http.urlconnection.URLConnectionSdkHttpClient; import java.io.*; import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; public class S3Storage extends Storage { private boolean closed = false; @@ -26,15 +31,14 @@ public class S3Storage extends Storage { private final Compression hiresCompression; private final String bucket; public S3Storage(S3StorageSettings settings) { - AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), ""); - var builder = S3Client - .builder() - .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) - .region(Region.of(settings.getRegion())); - if (settings.getEndpoint().isPresent()) { - builder = builder.endpointOverride(URI.create(settings.getEndpoint().get())); - } - this.client = builder.build(); + String endpoint = settings.getEndpoint().orElse( + String.format("https://s3.%s.amazonaws.com", settings.getRegion()) + ); + this.client = new DefaultS3ClientBuilder() + .credentialsProvider(() -> AwsBasicCredentials.create(settings.getAccessKey(), settings.getSecretKey())) + .region(Region.of(settings.getRegion(), URI.create(endpoint))) + .httpClient(URLConnectionSdkHttpClient.create()) // Or use URLConnectionSdkHttpClient + .build(); this.hiresCompression = settings.getCompression(); this.bucket = settings.getBucket(); } @@ -58,14 +62,14 @@ public class S3Storage extends Storage { InputStream is = client.getObject( GetObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); return Optional.of(new CompressedInputStream(is, compression)); } catch (NoSuchKeyException e) { return Optional.empty(); - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -76,9 +80,9 @@ public class S3Storage extends Storage { String path = getFilePath(mapId, lod, tile); try { var info = client.headObject( - HeadObjectRequest + GetObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); @@ -96,17 +100,17 @@ public class S3Storage extends Storage { @Override public long getSize() { - return info.contentLength(); + return info.getContentLength(); } @Override public long getLastModified() { - return info.lastModified().getEpochSecond(); + return info.getLastModified().getEpochSecond(); } }); } catch (NoSuchKeyException e) { return Optional.empty(); - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -118,12 +122,12 @@ public class S3Storage extends Storage { client.deleteObject( DeleteObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); } catch (NoSuchKeyException ignored) { - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -142,9 +146,10 @@ public class S3Storage extends Storage { client.putObject( PutObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .contentType(FileMime) + .contentLength((long) byteOut.toByteArray().length) .build(), RequestBody.fromBytes(byteOut.toByteArray()) ) @@ -158,14 +163,14 @@ public class S3Storage extends Storage { InputStream is = client.getObject( GetObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); return Optional.of(is); } catch (NoSuchKeyException e) { return Optional.empty(); - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -175,9 +180,9 @@ public class S3Storage extends Storage { String path = getMetaFilePath(mapId, name); try { var info = client.headObject( - HeadObjectRequest + GetObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); @@ -190,12 +195,12 @@ public class S3Storage extends Storage { @Override public long getSize() { - return info.contentLength(); + return info.getContentLength(); } }); } catch (NoSuchKeyException e) { return Optional.empty(); - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -207,12 +212,12 @@ public class S3Storage extends Storage { client.deleteObject( DeleteObjectRequest .builder() - .bucket(bucket) + .bucketName(bucket) .key(path) .build() ); } catch (NoSuchKeyException ignored) { - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -221,22 +226,26 @@ public class S3Storage extends Storage { public void purgeMap(String mapId, Function onProgress) throws IOException { String directory = getFilePath(mapId); try { - var files = client.listObjectsV2Paginator( + var files = new ListObjectsV2ResponsePager(client, ListObjectsV2Request .builder() - .bucket(bucket) + .bucketName(bucket) .prefix(directory + "/") - .build() ); - var filesList = files.contents().stream().collect(Collectors.toList()); + var filesList = StreamSupport + .stream( + Spliterators.spliteratorUnknownSize( + files.getContents(), 0), + false + ).collect(Collectors.toList()); for (int i = 0; i < filesList.size(); i++) { var file = filesList.get(i); try { client.deleteObject( DeleteObjectRequest .builder() - .bucket(bucket) - .key(file.key()) + .bucketName(bucket) + .key(file.getKey()) .build() ); } catch (NoSuchKeyException ignored) {} @@ -245,7 +254,7 @@ public class S3Storage extends Storage { new ProgressInfo((double) (i + 1) / filesList.size()) )) return; } - } catch (SdkException e) { + } catch (S3Exception e) { throw new IOException(e); } } @@ -253,16 +262,16 @@ public class S3Storage extends Storage { @Override public Collection collectMapIds() throws IOException { try { - var files = client.listObjectsV2Paginator( + var files = new ListObjectsV2ResponsePager(client, ListObjectsV2Request .builder() - .bucket(bucket) + .bucketName(bucket) .delimiter("/") - .build() ); List ids = new ArrayList<>(); - for (var file: files.commonPrefixes()) { - String id = file.prefix().split("/")[0]; + for (Iterator it = files.getCommonPrefixes(); it.hasNext(); ) { + var file = it.next(); + String id = file.getPrefix().split("/")[0]; if (!ids.contains(id)) { ids.add(id); }