Upstream
/
BlueMap
mirror of https://github.com/BlueMap-Minecraft/BlueMap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

S3 storage: use s3-lite

This commit is contained in:
Skye 2024-02-24 12:57:45 +09:00
parent 43adac0dbf
commit dee263d96a
No known key found for this signature in database
GPG Key ID: 0104BC05F41B77B8
2 changed files with 61 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
BlueMapCore/build.gradle.kts
View File

@ -70,7 +70,8 @@ dependencies {
api ("org.apache.commons:commons-dbcp2:2.9.0") api ("org.apache.commons:commons-dbcp2:2.9.0")
api ("io.airlift:aircompressor:0.24") api ("io.airlift:aircompressor:0.24")
api ("org.lz4:lz4-java:1.8.0") api ("org.lz4:lz4-java:1.8.0")
api ("software.amazon.awssdk:s3:2.24.9") api ("com.github.vgskye.s3-lite:core:4a9e099bf8")
api ("com.github.vgskye.s3-lite:http-client-url-connection:4a9e099bf8")
api ("de.bluecolored.bluemap.api:BlueMapAPI") api ("de.bluecolored.bluemap.api:BlueMapAPI")

109
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java
View File

@ -3,22 +3,27 @@ package de.bluecolored.bluemap.core.storage.s3;
import com.flowpowered.math.vector.Vector2i; import com.flowpowered.math.vector.Vector2i;
import de.bluecolored.bluemap.core.storage.*; import de.bluecolored.bluemap.core.storage.*;
import de.bluecolored.bluemap.core.util.OnCloseOutputStream; import de.bluecolored.bluemap.core.util.OnCloseOutputStream;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import io.github.linktosriram.s3lite.api.client.S3Client;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
import software.amazon.awssdk.core.exception.SdkException; import io.github.linktosriram.s3lite.api.exception.S3Exception;
import software.amazon.awssdk.core.sync.RequestBody; import io.github.linktosriram.s3lite.api.region.Region;
import software.amazon.awssdk.regions.Region; import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.S3Client; import io.github.linktosriram.s3lite.api.request.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.*; import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request;
import io.github.linktosriram.s3lite.api.request.PutObjectRequest;
import io.github.linktosriram.s3lite.api.response.CommonPrefix;
import io.github.linktosriram.s3lite.api.response.ListObjectsV2ResponsePager;
import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials;
import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder;
import io.github.linktosriram.s3lite.http.spi.request.RequestBody;
import io.github.linktosriram.s3lite.http.urlconnection.URLConnectionSdkHttpClient;
import java.io.*; import java.io.*;
import java.net.URI; import java.net.URI;
import java.util.ArrayList; import java.util.*;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
public class S3Storage extends Storage { public class S3Storage extends Storage {
private boolean closed = false; private boolean closed = false;
@ -26,15 +31,14 @@ public class S3Storage extends Storage {
private final Compression hiresCompression; private final Compression hiresCompression;
private final String bucket; private final String bucket;
public S3Storage(S3StorageSettings settings) { public S3Storage(S3StorageSettings settings) {
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), ""); String endpoint = settings.getEndpoint().orElse(
var builder = S3Client String.format("https://s3.%s.amazonaws.com", settings.getRegion())
.builder() );
.credentialsProvider(StaticCredentialsProvider.create(awsCreds)) this.client = new DefaultS3ClientBuilder()
.region(Region.of(settings.getRegion())); .credentialsProvider(() -> AwsBasicCredentials.create(settings.getAccessKey(), settings.getSecretKey()))
if (settings.getEndpoint().isPresent()) { .region(Region.of(settings.getRegion(), URI.create(endpoint)))
builder = builder.endpointOverride(URI.create(settings.getEndpoint().get())); .httpClient(URLConnectionSdkHttpClient.create()) // Or use URLConnectionSdkHttpClient
} .build();
this.client = builder.build();
this.hiresCompression = settings.getCompression(); this.hiresCompression = settings.getCompression();
this.bucket = settings.getBucket(); this.bucket = settings.getBucket();
} }
@ -58,14 +62,14 @@ public class S3Storage extends Storage {
InputStream is = client.getObject( InputStream is = client.getObject(
GetObjectRequest GetObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
return Optional.of(new CompressedInputStream(is, compression)); return Optional.of(new CompressedInputStream(is, compression));
} catch (NoSuchKeyException e) { } catch (NoSuchKeyException e) {
return Optional.empty(); return Optional.empty();
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -76,9 +80,9 @@ public class S3Storage extends Storage {
String path = getFilePath(mapId, lod, tile); String path = getFilePath(mapId, lod, tile);
try { try {
var info = client.headObject( var info = client.headObject(
HeadObjectRequest GetObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
@ -96,17 +100,17 @@ public class S3Storage extends Storage {
@Override @Override
public long getSize() { public long getSize() {
return info.contentLength(); return info.getContentLength();
} }
@Override @Override
public long getLastModified() { public long getLastModified() {
return info.lastModified().getEpochSecond(); return info.getLastModified().getEpochSecond();
} }
}); });
} catch (NoSuchKeyException e) { } catch (NoSuchKeyException e) {
return Optional.empty(); return Optional.empty();
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -118,12 +122,12 @@ public class S3Storage extends Storage {
client.deleteObject( client.deleteObject(
DeleteObjectRequest DeleteObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
} catch (NoSuchKeyException ignored) { } catch (NoSuchKeyException ignored) {
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -142,9 +146,10 @@ public class S3Storage extends Storage {
client.putObject( client.putObject(
PutObjectRequest PutObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.contentType(FileMime) .contentType(FileMime)
.contentLength((long) byteOut.toByteArray().length)
.build(), .build(),
RequestBody.fromBytes(byteOut.toByteArray()) RequestBody.fromBytes(byteOut.toByteArray())
) )
@ -158,14 +163,14 @@ public class S3Storage extends Storage {
InputStream is = client.getObject( InputStream is = client.getObject(
GetObjectRequest GetObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
return Optional.of(is); return Optional.of(is);
} catch (NoSuchKeyException e) { } catch (NoSuchKeyException e) {
return Optional.empty(); return Optional.empty();
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -175,9 +180,9 @@ public class S3Storage extends Storage {
String path = getMetaFilePath(mapId, name); String path = getMetaFilePath(mapId, name);
try { try {
var info = client.headObject( var info = client.headObject(
HeadObjectRequest GetObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
@ -190,12 +195,12 @@ public class S3Storage extends Storage {
@Override @Override
public long getSize() { public long getSize() {
return info.contentLength(); return info.getContentLength();
} }
}); });
} catch (NoSuchKeyException e) { } catch (NoSuchKeyException e) {
return Optional.empty(); return Optional.empty();
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -207,12 +212,12 @@ public class S3Storage extends Storage {
client.deleteObject( client.deleteObject(
DeleteObjectRequest DeleteObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(path) .key(path)
.build() .build()
); );
} catch (NoSuchKeyException ignored) { } catch (NoSuchKeyException ignored) {
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -221,22 +226,26 @@ public class S3Storage extends Storage {
public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) throws IOException { public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) throws IOException {
String directory = getFilePath(mapId); String directory = getFilePath(mapId);
try { try {
var files = client.listObjectsV2Paginator( var files = new ListObjectsV2ResponsePager(client,
ListObjectsV2Request ListObjectsV2Request
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.prefix(directory + "/") .prefix(directory + "/")
.build()
); );
var filesList = files.contents().stream().collect(Collectors.toList()); var filesList = StreamSupport
.stream(
Spliterators.spliteratorUnknownSize(
files.getContents(), 0),
false
).collect(Collectors.toList());
for (int i = 0; i < filesList.size(); i++) { for (int i = 0; i < filesList.size(); i++) {
var file = filesList.get(i); var file = filesList.get(i);
try { try {
client.deleteObject( client.deleteObject(
DeleteObjectRequest DeleteObjectRequest
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.key(file.key()) .key(file.getKey())
.build() .build()
); );
} catch (NoSuchKeyException ignored) {} } catch (NoSuchKeyException ignored) {}
@ -245,7 +254,7 @@ public class S3Storage extends Storage {
new ProgressInfo((double) (i + 1) / filesList.size()) new ProgressInfo((double) (i + 1) / filesList.size())
)) return; )) return;
} }
} catch (SdkException e) { } catch (S3Exception e) {
throw new IOException(e); throw new IOException(e);
} }
} }
@ -253,16 +262,16 @@ public class S3Storage extends Storage {
@Override @Override
public Collection<String> collectMapIds() throws IOException { public Collection<String> collectMapIds() throws IOException {
try { try {
var files = client.listObjectsV2Paginator( var files = new ListObjectsV2ResponsePager(client,
ListObjectsV2Request ListObjectsV2Request
.builder() .builder()
.bucket(bucket) .bucketName(bucket)
.delimiter("/") .delimiter("/")
.build()
); );
List<String> ids = new ArrayList<>(); List<String> ids = new ArrayList<>();
for (var file: files.commonPrefixes()) { for (Iterator<CommonPrefix> it = files.getCommonPrefixes(); it.hasNext(); ) {
String id = file.prefix().split("/")[0]; var file = it.next();
String id = file.getPrefix().split("/")[0];
if (!ids.contains(id)) { if (!ids.contains(id)) {
ids.add(id); ids.add(id);
} }