Upstream
/
BlueMap
mirror of https://github.com/BlueMap-Minecraft/BlueMap.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

S3 support

This commit is contained in:
Skye 2024-02-23 16:03:34 +09:00
parent 3a1e723a51
commit 68aeaa65e5
No known key found for this signature in database
GPG Key ID: 0104BC05F41B77B8
6 changed files with 442 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java Normal file
View File

@ -0,0 +1,50 @@
package de.bluecolored.bluemap.common.config.storage;
import de.bluecolored.bluemap.api.debug.DebugDump;
import de.bluecolored.bluemap.core.storage.Compression;
import de.bluecolored.bluemap.core.storage.s3.S3StorageSettings;
import org.spongepowered.configurate.objectmapping.ConfigSerializable;
import java.util.Optional;
@SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"})
@DebugDump
@ConfigSerializable
public class S3Config extends StorageConfig implements S3StorageSettings {
private String endpoint = null;
private String region = "";
private String accessKey = "";
private String secretKey = "";
private String bucket = "";
private Compression compression = Compression.GZIP;
@Override
public Optional<String> getEndpoint() {
return Optional.ofNullable(endpoint);
}
@Override
public String getRegion() {
return region;
}
@Override
public String getAccessKey() {
return accessKey;
}
@Override
public String getSecretKey() {
return secretKey;
}
@Override
public String getBucket() {
return bucket;
}
@Override
public Compression getCompression() {
return compression;
}
}

4
BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java
View File

@ -26,12 +26,14 @@ package de.bluecolored.bluemap.common.config.storage;
import de.bluecolored.bluemap.core.storage.Storage;
import de.bluecolored.bluemap.core.storage.file.FileStorage;
import de.bluecolored.bluemap.core.storage.s3.S3Storage;
import de.bluecolored.bluemap.core.storage.sql.SQLStorage;
public enum StorageType {
FILE (FileConfig.class, FileStorage::new),
SQL (SQLConfig.class, SQLStorage::create);
SQL (SQLConfig.class, SQLStorage::create),
S3 (S3Config.class, S3Storage::new);
private final Class<? extends StorageConfig> configType;
private final StorageFactory<? extends StorageConfig> storageFactory;

32
BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf Normal file
View File

@ -0,0 +1,32 @@
## ##
## BlueMap ##
## Storage-Config ##
## ##
# The storage-type of this storage.
# Depending on this setting, different config-entries are allowed/expected in this config file.
# Don't change this value! (If you want a different storage-type, check out the other example-configs)
storage-type: S3
# This is the S3 endpoint that bluemap will use.
# If you're using a third-party S3 compatible provider,
# uncomment this and set it to your provider's S3 endpoint URL.
#endpoint: "https://s3.us-west-004.backblazeb2.com"
# This is the S3 region that bluemap will use.
# Set this to the AWS region of your S3 bucket.
region: "us-east-1"
# The S3 credentials that bluemap will use.
access-key: "changeme"
secret-key: "changeme"
# The S3 bucket name.
bucket: "bluemap"
# The compression-type that bluemap will use to compress generated map-data.
# Available compression-types are:
# - GZIP
# - NONE
# The default is: GZIP
compression: GZIP

1
BlueMapCore/build.gradle.kts
View File

@ -70,6 +70,7 @@ dependencies {
api ("org.apache.commons:commons-dbcp2:2.9.0")
api ("io.airlift:aircompressor:0.24")
api ("org.lz4:lz4-java:1.8.0")
api ("software.amazon.awssdk:s3:2.24.9")
api ("de.bluecolored.bluemap.api:BlueMapAPI")

342
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java Normal file
View File

@ -0,0 +1,342 @@
package de.bluecolored.bluemap.core.storage.s3;
import com.flowpowered.math.vector.Vector2i;
import de.bluecolored.bluemap.core.storage.*;
import de.bluecolored.bluemap.core.util.OnCloseOutputStream;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import java.io.*;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
public class S3Storage extends Storage {
private boolean closed = false;
private final S3Client client;
private final Compression hiresCompression;
private final String bucket;
public S3Storage(S3StorageSettings settings) {
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), "");
var builder = S3Client
.builder()
.credentialsProvider(StaticCredentialsProvider.create(awsCreds))
.region(Region.of(settings.getRegion()));
if (settings.getEndpoint().isPresent()) {
builder = builder.endpointOverride(URI.create(settings.getEndpoint().get()));
}
this.client = builder.build();
this.hiresCompression = settings.getCompression();
this.bucket = settings.getBucket();
}
@Override
public void initialize() throws IOException {}
@Override
public OutputStream writeMapTile(String mapId, int lod, Vector2i tile) throws IOException {
Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE;
String path = getFilePath(mapId, lod, tile);
OutputStream pipeOut = makeUploadStream(path, getFileMime(lod));
return new BufferedOutputStream(compression.compress(pipeOut));
}
@Override
public Optional<CompressedInputStream> readMapTile(String mapId, int lod, Vector2i tile) throws IOException {
Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE;
String path = getFilePath(mapId, lod, tile);
try {
InputStream is = client.getObject(
GetObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
return Optional.of(new CompressedInputStream(is, compression));
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return Optional.empty();
}
throw new IOException(e);
}
}
@Override
public Optional<TileInfo> readMapTileInfo(String mapId, int lod, Vector2i tile) throws IOException {
Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE;
String path = getFilePath(mapId, lod, tile);
try {
var info = client.headObject(
HeadObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
return Optional.of(new TileInfo() {
@Override
public CompressedInputStream readMapTile() throws IOException {
return S3Storage.this.readMapTile(mapId, lod, tile)
.orElseThrow(() -> new IOException("Tile no longer present!"));
}
@Override
public Compression getCompression() {
return compression;
}
@Override
public long getSize() {
return info.contentLength();
}
@Override
public long getLastModified() {
return info.lastModified().getEpochSecond();
}
});
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return Optional.empty();
}
throw new IOException(e);
}
}
@Override
public void deleteMapTile(String mapId, int lod, Vector2i tile) throws IOException {
String path = getFilePath(mapId, lod, tile);
try {
client.deleteObject(
DeleteObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return;
}
throw new IOException(e);
}
}
@Override
public OutputStream writeMeta(String mapId, String name) throws IOException {
String path = getMetaFilePath(mapId, name);
OutputStream pipeOut = makeUploadStream(path, null);
return new BufferedOutputStream(pipeOut);
}
private OutputStream makeUploadStream(String path, String FileMime) {
ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
return new OnCloseOutputStream(new BufferedOutputStream(byteOut), () ->
client.putObject(
PutObjectRequest
.builder()
.bucket(bucket)
.key(path)
.contentType(FileMime)
.build(),
RequestBody.fromBytes(byteOut.toByteArray())
)
);
}
@Override
public Optional<InputStream> readMeta(String mapId, String name) throws IOException {
String path = getMetaFilePath(mapId, name);
try {
InputStream is = client.getObject(
GetObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
return Optional.of(is);
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return Optional.empty();
}
throw new IOException(e);
}
}
@Override
public Optional<MetaInfo> readMetaInfo(String mapId, String name) throws IOException {
String path = getMetaFilePath(mapId, name);
try {
var info = client.headObject(
HeadObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
return Optional.of(new MetaInfo() {
@Override
public InputStream readMeta() throws IOException {
return S3Storage.this.readMeta(mapId, name)
.orElseThrow(() -> new IOException("Meta no longer present!"));
}
@Override
public long getSize() {
return info.contentLength();
}
});
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return Optional.empty();
}
throw new IOException(e);
}
}
@Override
public void deleteMeta(String mapId, String name) throws IOException {
String path = getMetaFilePath(mapId, name);
try {
client.deleteObject(
DeleteObjectRequest
.builder()
.bucket(bucket)
.key(path)
.build()
);
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return;
}
throw new IOException(e);
}
}
@Override
public void purgeMap(String mapId, Function<ProgressInfo, Boolean> onProgress) throws IOException {
String directory = getFilePath(mapId);
try {
var files = client.listObjectsV2Paginator(
ListObjectsV2Request
.builder()
.bucket(bucket)
.prefix(directory + "/")
.build()
);
var filesList = files.contents().stream().collect(Collectors.toList());
for (int i = 0; i < filesList.size(); i++) {
var file = filesList.get(i);
client.deleteObject(
DeleteObjectRequest
.builder()
.bucket(bucket)
.key(file.key())
.build()
);
if (!onProgress.apply(
new ProgressInfo((double) (i + 1) / filesList.size())
)) return;
}
} catch (Exception e) {
if (e instanceof NoSuchKeyException) {
return;
}
throw new IOException(e);
}
}
@Override
public Collection<String> collectMapIds() throws IOException {
try {
var files = client.listObjectsV2Paginator(
ListObjectsV2Request
.builder()
.bucket(bucket)
.delimiter("/")
.build()
);
List<String> ids = new ArrayList<>();
for (var file: files.contents()) {
String id = file.key().split("/")[0];
if (!ids.contains(id)) {
ids.add(id);
}
}
return ids;
} catch (Exception e) {
throw new IOException(e);
}
}
@Override
public boolean isClosed() {
return closed;
}
@Override
public void close() throws IOException {
client.close();
closed = true;
}
public String getFileMime(int lod){
if (lod == 0) {
if (hiresCompression.getFileSuffix().equals(".gz")) {
return "application/gzip";
} else if (hiresCompression == Compression.NONE) {
return "application/json";
} else {
return "application/octet-stream";
}
} else {
return "image/png";
}
}
public String getFilePath(String mapId, int lod, Vector2i tile){
String path = "x" + tile.getX() + "z" + tile.getY();
char[] cs = path.toCharArray();
List<String> folders = new ArrayList<>();
StringBuilder folder = new StringBuilder();
for (char c : cs){
folder.append(c);
if (c >= '0' && c <= '9'){
folders.add(folder.toString());
folder.delete(0, folder.length());
}
}
String fileName = folders.remove(folders.size() - 1);
StringBuilder p = new StringBuilder(getFilePath(mapId) + "/tiles/" + lod);
for (String s : folders){
p.append("/").append(s);
}
if (lod == 0) {
return p + "/" + (fileName + ".json" + hiresCompression.getFileSuffix());
} else {
return p + "/" + (fileName + ".png");
}
}
public String getFilePath(String mapId) {
return mapId;
}
public String getMetaFilePath(String mapId, String name) {
return getFilePath(mapId) + "/" + escapeMetaName(name);
}
}

14
BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java Normal file
View File

@ -0,0 +1,14 @@
package de.bluecolored.bluemap.core.storage.s3;
import de.bluecolored.bluemap.core.storage.Compression;
import java.util.Optional;
public interface S3StorageSettings {
Optional<String> getEndpoint();
String getRegion();
String getAccessKey();
String getSecretKey();
String getBucket();
Compression getCompression();
}