mirror of
https://github.com/BlueMap-Minecraft/BlueMap.git
synced 2024-09-27 06:42:42 +02:00
S3 storage: use s3-lite
This commit is contained in:
parent
43adac0dbf
commit
dee263d96a
@ -70,7 +70,8 @@ dependencies {
|
|||||||
api ("org.apache.commons:commons-dbcp2:2.9.0")
|
api ("org.apache.commons:commons-dbcp2:2.9.0")
|
||||||
api ("io.airlift:aircompressor:0.24")
|
api ("io.airlift:aircompressor:0.24")
|
||||||
api ("org.lz4:lz4-java:1.8.0")
|
api ("org.lz4:lz4-java:1.8.0")
|
||||||
api ("software.amazon.awssdk:s3:2.24.9")
|
api ("com.github.vgskye.s3-lite:core:4a9e099bf8")
|
||||||
|
api ("com.github.vgskye.s3-lite:http-client-url-connection:4a9e099bf8")
|
||||||
|
|
||||||
api ("de.bluecolored.bluemap.api:BlueMapAPI")
|
api ("de.bluecolored.bluemap.api:BlueMapAPI")
|
||||||
|
|
||||||
|
@ -3,22 +3,27 @@
|
|||||||
import com.flowpowered.math.vector.Vector2i;
|
import com.flowpowered.math.vector.Vector2i;
|
||||||
import de.bluecolored.bluemap.core.storage.*;
|
import de.bluecolored.bluemap.core.storage.*;
|
||||||
import de.bluecolored.bluemap.core.util.OnCloseOutputStream;
|
import de.bluecolored.bluemap.core.util.OnCloseOutputStream;
|
||||||
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
|
import io.github.linktosriram.s3lite.api.client.S3Client;
|
||||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
|
||||||
import software.amazon.awssdk.core.exception.SdkException;
|
import io.github.linktosriram.s3lite.api.exception.S3Exception;
|
||||||
import software.amazon.awssdk.core.sync.RequestBody;
|
import io.github.linktosriram.s3lite.api.region.Region;
|
||||||
import software.amazon.awssdk.regions.Region;
|
import io.github.linktosriram.s3lite.api.request.DeleteObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.S3Client;
|
import io.github.linktosriram.s3lite.api.request.GetObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.model.*;
|
import io.github.linktosriram.s3lite.api.request.ListObjectsV2Request;
|
||||||
|
import io.github.linktosriram.s3lite.api.request.PutObjectRequest;
|
||||||
|
import io.github.linktosriram.s3lite.api.response.CommonPrefix;
|
||||||
|
import io.github.linktosriram.s3lite.api.response.ListObjectsV2ResponsePager;
|
||||||
|
import io.github.linktosriram.s3lite.core.auth.AwsBasicCredentials;
|
||||||
|
import io.github.linktosriram.s3lite.core.client.DefaultS3ClientBuilder;
|
||||||
|
import io.github.linktosriram.s3lite.http.spi.request.RequestBody;
|
||||||
|
import io.github.linktosriram.s3lite.http.urlconnection.URLConnectionSdkHttpClient;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.ArrayList;
|
import java.util.*;
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.StreamSupport;
|
||||||
|
|
||||||
public class S3Storage extends Storage {
|
public class S3Storage extends Storage {
|
||||||
private boolean closed = false;
|
private boolean closed = false;
|
||||||
@ -26,15 +31,14 @@ public class S3Storage extends Storage {
|
|||||||
private final Compression hiresCompression;
|
private final Compression hiresCompression;
|
||||||
private final String bucket;
|
private final String bucket;
|
||||||
public S3Storage(S3StorageSettings settings) {
|
public S3Storage(S3StorageSettings settings) {
|
||||||
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), "");
|
String endpoint = settings.getEndpoint().orElse(
|
||||||
var builder = S3Client
|
String.format("https://s3.%s.amazonaws.com", settings.getRegion())
|
||||||
.builder()
|
);
|
||||||
.credentialsProvider(StaticCredentialsProvider.create(awsCreds))
|
this.client = new DefaultS3ClientBuilder()
|
||||||
.region(Region.of(settings.getRegion()));
|
.credentialsProvider(() -> AwsBasicCredentials.create(settings.getAccessKey(), settings.getSecretKey()))
|
||||||
if (settings.getEndpoint().isPresent()) {
|
.region(Region.of(settings.getRegion(), URI.create(endpoint)))
|
||||||
builder = builder.endpointOverride(URI.create(settings.getEndpoint().get()));
|
.httpClient(URLConnectionSdkHttpClient.create()) // Or use URLConnectionSdkHttpClient
|
||||||
}
|
.build();
|
||||||
this.client = builder.build();
|
|
||||||
this.hiresCompression = settings.getCompression();
|
this.hiresCompression = settings.getCompression();
|
||||||
this.bucket = settings.getBucket();
|
this.bucket = settings.getBucket();
|
||||||
}
|
}
|
||||||
@ -58,14 +62,14 @@ public Optional<CompressedInputStream> readMapTile(String mapId, int lod, Vector
|
|||||||
InputStream is = client.getObject(
|
InputStream is = client.getObject(
|
||||||
GetObjectRequest
|
GetObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
return Optional.of(new CompressedInputStream(is, compression));
|
return Optional.of(new CompressedInputStream(is, compression));
|
||||||
} catch (NoSuchKeyException e) {
|
} catch (NoSuchKeyException e) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -76,9 +80,9 @@ public Optional<TileInfo> readMapTileInfo(String mapId, int lod, Vector2i tile)
|
|||||||
String path = getFilePath(mapId, lod, tile);
|
String path = getFilePath(mapId, lod, tile);
|
||||||
try {
|
try {
|
||||||
var info = client.headObject(
|
var info = client.headObject(
|
||||||
HeadObjectRequest
|
GetObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
@ -96,17 +100,17 @@ public Compression getCompression() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getSize() {
|
public long getSize() {
|
||||||
return info.contentLength();
|
return info.getContentLength();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getLastModified() {
|
public long getLastModified() {
|
||||||
return info.lastModified().getEpochSecond();
|
return info.getLastModified().getEpochSecond();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} catch (NoSuchKeyException e) {
|
} catch (NoSuchKeyException e) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -118,12 +122,12 @@ public void deleteMapTile(String mapId, int lod, Vector2i tile) throws IOExcepti
|
|||||||
client.deleteObject(
|
client.deleteObject(
|
||||||
DeleteObjectRequest
|
DeleteObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
} catch (NoSuchKeyException ignored) {
|
} catch (NoSuchKeyException ignored) {
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -142,9 +146,10 @@ private OutputStream makeUploadStream(String path, String FileMime) {
|
|||||||
client.putObject(
|
client.putObject(
|
||||||
PutObjectRequest
|
PutObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.contentType(FileMime)
|
.contentType(FileMime)
|
||||||
|
.contentLength((long) byteOut.toByteArray().length)
|
||||||
.build(),
|
.build(),
|
||||||
RequestBody.fromBytes(byteOut.toByteArray())
|
RequestBody.fromBytes(byteOut.toByteArray())
|
||||||
)
|
)
|
||||||
@ -158,14 +163,14 @@ public Optional<InputStream> readMeta(String mapId, String name) throws IOExcept
|
|||||||
InputStream is = client.getObject(
|
InputStream is = client.getObject(
|
||||||
GetObjectRequest
|
GetObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
return Optional.of(is);
|
return Optional.of(is);
|
||||||
} catch (NoSuchKeyException e) {
|
} catch (NoSuchKeyException e) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -175,9 +180,9 @@ public Optional<MetaInfo> readMetaInfo(String mapId, String name) throws IOExcep
|
|||||||
String path = getMetaFilePath(mapId, name);
|
String path = getMetaFilePath(mapId, name);
|
||||||
try {
|
try {
|
||||||
var info = client.headObject(
|
var info = client.headObject(
|
||||||
HeadObjectRequest
|
GetObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
@ -190,12 +195,12 @@ public InputStream readMeta() throws IOException {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getSize() {
|
public long getSize() {
|
||||||
return info.contentLength();
|
return info.getContentLength();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} catch (NoSuchKeyException e) {
|
} catch (NoSuchKeyException e) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -207,12 +212,12 @@ public void deleteMeta(String mapId, String name) throws IOException {
|
|||||||
client.deleteObject(
|
client.deleteObject(
|
||||||
DeleteObjectRequest
|
DeleteObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(path)
|
.key(path)
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
} catch (NoSuchKeyException ignored) {
|
} catch (NoSuchKeyException ignored) {
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -221,22 +226,26 @@ public void deleteMeta(String mapId, String name) throws IOException {
|
|||||||
public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) throws IOException {
|
public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) throws IOException {
|
||||||
String directory = getFilePath(mapId);
|
String directory = getFilePath(mapId);
|
||||||
try {
|
try {
|
||||||
var files = client.listObjectsV2Paginator(
|
var files = new ListObjectsV2ResponsePager(client,
|
||||||
ListObjectsV2Request
|
ListObjectsV2Request
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.prefix(directory + "/")
|
.prefix(directory + "/")
|
||||||
.build()
|
|
||||||
);
|
);
|
||||||
var filesList = files.contents().stream().collect(Collectors.toList());
|
var filesList = StreamSupport
|
||||||
|
.stream(
|
||||||
|
Spliterators.spliteratorUnknownSize(
|
||||||
|
files.getContents(), 0),
|
||||||
|
false
|
||||||
|
).collect(Collectors.toList());
|
||||||
for (int i = 0; i < filesList.size(); i++) {
|
for (int i = 0; i < filesList.size(); i++) {
|
||||||
var file = filesList.get(i);
|
var file = filesList.get(i);
|
||||||
try {
|
try {
|
||||||
client.deleteObject(
|
client.deleteObject(
|
||||||
DeleteObjectRequest
|
DeleteObjectRequest
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.key(file.key())
|
.key(file.getKey())
|
||||||
.build()
|
.build()
|
||||||
);
|
);
|
||||||
} catch (NoSuchKeyException ignored) {}
|
} catch (NoSuchKeyException ignored) {}
|
||||||
@ -245,7 +254,7 @@ public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) t
|
|||||||
new ProgressInfo((double) (i + 1) / filesList.size())
|
new ProgressInfo((double) (i + 1) / filesList.size())
|
||||||
)) return;
|
)) return;
|
||||||
}
|
}
|
||||||
} catch (SdkException e) {
|
} catch (S3Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -253,16 +262,16 @@ public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) t
|
|||||||
@Override
|
@Override
|
||||||
public Collection<String> collectMapIds() throws IOException {
|
public Collection<String> collectMapIds() throws IOException {
|
||||||
try {
|
try {
|
||||||
var files = client.listObjectsV2Paginator(
|
var files = new ListObjectsV2ResponsePager(client,
|
||||||
ListObjectsV2Request
|
ListObjectsV2Request
|
||||||
.builder()
|
.builder()
|
||||||
.bucket(bucket)
|
.bucketName(bucket)
|
||||||
.delimiter("/")
|
.delimiter("/")
|
||||||
.build()
|
|
||||||
);
|
);
|
||||||
List<String> ids = new ArrayList<>();
|
List<String> ids = new ArrayList<>();
|
||||||
for (var file: files.commonPrefixes()) {
|
for (Iterator<CommonPrefix> it = files.getCommonPrefixes(); it.hasNext(); ) {
|
||||||
String id = file.prefix().split("/")[0];
|
var file = it.next();
|
||||||
|
String id = file.getPrefix().split("/")[0];
|
||||||
if (!ids.contains(id)) {
|
if (!ids.contains(id)) {
|
||||||
ids.add(id);
|
ids.add(id);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user