diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java new file mode 100644 index 00000000..ad4b4340 --- /dev/null +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/S3Config.java @@ -0,0 +1,50 @@ +package de.bluecolored.bluemap.common.config.storage; + +import de.bluecolored.bluemap.api.debug.DebugDump; +import de.bluecolored.bluemap.core.storage.Compression; +import de.bluecolored.bluemap.core.storage.s3.S3StorageSettings; +import org.spongepowered.configurate.objectmapping.ConfigSerializable; + +import java.util.Optional; + +@SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) +@DebugDump +@ConfigSerializable +public class S3Config extends StorageConfig implements S3StorageSettings { + private String endpoint = null; + private String region = ""; + private String accessKey = ""; + private String secretKey = ""; + private String bucket = ""; + private Compression compression = Compression.GZIP; + + @Override + public Optional getEndpoint() { + return Optional.ofNullable(endpoint); + } + + @Override + public String getRegion() { + return region; + } + + @Override + public String getAccessKey() { + return accessKey; + } + + @Override + public String getSecretKey() { + return secretKey; + } + + @Override + public String getBucket() { + return bucket; + } + + @Override + public Compression getCompression() { + return compression; + } +} diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java index 20a24e94..aae515d7 100644 --- a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java @@ -26,12 +26,14 @@ package de.bluecolored.bluemap.common.config.storage; import de.bluecolored.bluemap.core.storage.Storage; import de.bluecolored.bluemap.core.storage.file.FileStorage; +import de.bluecolored.bluemap.core.storage.s3.S3Storage; import de.bluecolored.bluemap.core.storage.sql.SQLStorage; public enum StorageType { FILE (FileConfig.class, FileStorage::new), - SQL (SQLConfig.class, SQLStorage::create); + SQL (SQLConfig.class, SQLStorage::create), + S3 (S3Config.class, S3Storage::new); private final Class configType; private final StorageFactory storageFactory; diff --git a/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf b/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf new file mode 100644 index 00000000..b882b9d8 --- /dev/null +++ b/BlueMapCommon/src/main/resources/de/bluecolored/bluemap/config/storages/s3.conf @@ -0,0 +1,32 @@ +## ## +## BlueMap ## +## Storage-Config ## +## ## + +# The storage-type of this storage. +# Depending on this setting, different config-entries are allowed/expected in this config file. +# Don't change this value! (If you want a different storage-type, check out the other example-configs) +storage-type: S3 + +# This is the S3 endpoint that bluemap will use. +# If you're using a third-party S3 compatible provider, +# uncomment this and set it to your provider's S3 endpoint URL. +#endpoint: "https://s3.us-west-004.backblazeb2.com" + +# This is the S3 region that bluemap will use. +# Set this to the AWS region of your S3 bucket. +region: "us-east-1" + +# The S3 credentials that bluemap will use. +access-key: "changeme" +secret-key: "changeme" + +# The S3 bucket name. +bucket: "bluemap" + +# The compression-type that bluemap will use to compress generated map-data. +# Available compression-types are: +# - GZIP +# - NONE +# The default is: GZIP +compression: GZIP diff --git a/BlueMapCore/build.gradle.kts b/BlueMapCore/build.gradle.kts index da0d6399..73131794 100644 --- a/BlueMapCore/build.gradle.kts +++ b/BlueMapCore/build.gradle.kts @@ -70,6 +70,7 @@ dependencies { api ("org.apache.commons:commons-dbcp2:2.9.0") api ("io.airlift:aircompressor:0.24") api ("org.lz4:lz4-java:1.8.0") + api ("software.amazon.awssdk:s3:2.24.9") api ("de.bluecolored.bluemap.api:BlueMapAPI") diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java new file mode 100644 index 00000000..76e5ba6d --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3Storage.java @@ -0,0 +1,342 @@ +package de.bluecolored.bluemap.core.storage.s3; + +import com.flowpowered.math.vector.Vector2i; +import de.bluecolored.bluemap.core.storage.*; +import de.bluecolored.bluemap.core.util.OnCloseOutputStream; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.*; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class S3Storage extends Storage { + private boolean closed = false; + private final S3Client client; + private final Compression hiresCompression; + private final String bucket; + public S3Storage(S3StorageSettings settings) { + AwsSessionCredentials awsCreds = AwsSessionCredentials.create(settings.getAccessKey(), settings.getSecretKey(), ""); + var builder = S3Client + .builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) + .region(Region.of(settings.getRegion())); + if (settings.getEndpoint().isPresent()) { + builder = builder.endpointOverride(URI.create(settings.getEndpoint().get())); + } + this.client = builder.build(); + this.hiresCompression = settings.getCompression(); + this.bucket = settings.getBucket(); + } + @Override + public void initialize() throws IOException {} + + @Override + public OutputStream writeMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + + OutputStream pipeOut = makeUploadStream(path, getFileMime(lod)); + return new BufferedOutputStream(compression.compress(pipeOut)); + } + + @Override + public Optional readMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + try { + InputStream is = client.getObject( + GetObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new CompressedInputStream(is, compression)); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public Optional readMapTileInfo(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + try { + var info = client.headObject( + HeadObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new TileInfo() { + @Override + public CompressedInputStream readMapTile() throws IOException { + return S3Storage.this.readMapTile(mapId, lod, tile) + .orElseThrow(() -> new IOException("Tile no longer present!")); + } + + @Override + public Compression getCompression() { + return compression; + } + + @Override + public long getSize() { + return info.contentLength(); + } + + @Override + public long getLastModified() { + return info.lastModified().getEpochSecond(); + } + }); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public void deleteMapTile(String mapId, int lod, Vector2i tile) throws IOException { + String path = getFilePath(mapId, lod, tile); + try { + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public OutputStream writeMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + + OutputStream pipeOut = makeUploadStream(path, null); + return new BufferedOutputStream(pipeOut); + } + + private OutputStream makeUploadStream(String path, String FileMime) { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + return new OnCloseOutputStream(new BufferedOutputStream(byteOut), () -> + client.putObject( + PutObjectRequest + .builder() + .bucket(bucket) + .key(path) + .contentType(FileMime) + .build(), + RequestBody.fromBytes(byteOut.toByteArray()) + ) + ); + } + + @Override + public Optional readMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + InputStream is = client.getObject( + GetObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(is); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public Optional readMetaInfo(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + var info = client.headObject( + HeadObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + return Optional.of(new MetaInfo() { + @Override + public InputStream readMeta() throws IOException { + return S3Storage.this.readMeta(mapId, name) + .orElseThrow(() -> new IOException("Meta no longer present!")); + } + + @Override + public long getSize() { + return info.contentLength(); + } + }); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return Optional.empty(); + } + throw new IOException(e); + } + } + + @Override + public void deleteMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(path) + .build() + ); + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public void purgeMap(String mapId, Function onProgress) throws IOException { + String directory = getFilePath(mapId); + try { + var files = client.listObjectsV2Paginator( + ListObjectsV2Request + .builder() + .bucket(bucket) + .prefix(directory + "/") + .build() + ); + var filesList = files.contents().stream().collect(Collectors.toList()); + for (int i = 0; i < filesList.size(); i++) { + var file = filesList.get(i); + client.deleteObject( + DeleteObjectRequest + .builder() + .bucket(bucket) + .key(file.key()) + .build() + ); + + if (!onProgress.apply( + new ProgressInfo((double) (i + 1) / filesList.size()) + )) return; + } + } catch (Exception e) { + if (e instanceof NoSuchKeyException) { + return; + } + throw new IOException(e); + } + } + + @Override + public Collection collectMapIds() throws IOException { + try { + var files = client.listObjectsV2Paginator( + ListObjectsV2Request + .builder() + .bucket(bucket) + .delimiter("/") + .build() + ); + List ids = new ArrayList<>(); + for (var file: files.contents()) { + String id = file.key().split("/")[0]; + if (!ids.contains(id)) { + ids.add(id); + } + } + return ids; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public void close() throws IOException { + client.close(); + closed = true; + } + + public String getFileMime(int lod){ + if (lod == 0) { + if (hiresCompression.getFileSuffix().equals(".gz")) { + return "application/gzip"; + } else if (hiresCompression == Compression.NONE) { + return "application/json"; + } else { + return "application/octet-stream"; + } + } else { + return "image/png"; + } + } + + public String getFilePath(String mapId, int lod, Vector2i tile){ + String path = "x" + tile.getX() + "z" + tile.getY(); + char[] cs = path.toCharArray(); + List folders = new ArrayList<>(); + StringBuilder folder = new StringBuilder(); + for (char c : cs){ + folder.append(c); + if (c >= '0' && c <= '9'){ + folders.add(folder.toString()); + folder.delete(0, folder.length()); + } + } + String fileName = folders.remove(folders.size() - 1); + + StringBuilder p = new StringBuilder(getFilePath(mapId) + "/tiles/" + lod); + for (String s : folders){ + p.append("/").append(s); + } + + if (lod == 0) { + return p + "/" + (fileName + ".json" + hiresCompression.getFileSuffix()); + } else { + return p + "/" + (fileName + ".png"); + } + } + + public String getFilePath(String mapId) { + return mapId; + } + + public String getMetaFilePath(String mapId, String name) { + return getFilePath(mapId) + "/" + escapeMetaName(name); + } +} diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java new file mode 100644 index 00000000..d62bff0b --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/s3/S3StorageSettings.java @@ -0,0 +1,14 @@ +package de.bluecolored.bluemap.core.storage.s3; + +import de.bluecolored.bluemap.core.storage.Compression; + +import java.util.Optional; + +public interface S3StorageSettings { + Optional getEndpoint(); + String getRegion(); + String getAccessKey(); + String getSecretKey(); + String getBucket(); + Compression getCompression(); +} \ No newline at end of file