From 0156766a0312e5b2ef146dc5ec230a12473d4210 Mon Sep 17 00:00:00 2001 From: Skye Date: Fri, 23 Feb 2024 12:11:32 +0900 Subject: [PATCH] b2 support --- .../common/config/storage/B2Config.java | 36 +++ .../common/config/storage/StorageType.java | 4 +- BlueMapCore/build.gradle.kts | 3 + .../bluemap/core/storage/b2/B2Storage.java | 294 ++++++++++++++++++ .../core/storage/b2/B2StorageSettings.java | 15 + 5 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/B2Config.java create mode 100644 BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2Storage.java create mode 100644 BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2StorageSettings.java diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/B2Config.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/B2Config.java new file mode 100644 index 00000000..1a4333dd --- /dev/null +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/B2Config.java @@ -0,0 +1,36 @@ +package de.bluecolored.bluemap.common.config.storage; + +import de.bluecolored.bluemap.api.debug.DebugDump; +import de.bluecolored.bluemap.core.storage.Compression; +import de.bluecolored.bluemap.core.storage.b2.B2StorageSettings; +import org.spongepowered.configurate.objectmapping.ConfigSerializable; + +@SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) +@DebugDump +@ConfigSerializable +public class B2Config extends StorageConfig implements B2StorageSettings { + private String applicationKeyId = ""; + private String applicationKey = ""; + private String bucket = ""; + private Compression compression = Compression.GZIP; + + @Override + public String getApplicationKeyId() { + return applicationKeyId; + } + + @Override + public String getApplicationKey() { + return applicationKey; + } + + @Override + public String getBucket() { + return bucket; + } + + @Override + public Compression getCompression() { + return compression; + } +} diff --git a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java index 20a24e94..3ed75066 100644 --- a/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java +++ b/BlueMapCommon/src/main/java/de/bluecolored/bluemap/common/config/storage/StorageType.java @@ -25,13 +25,15 @@ package de.bluecolored.bluemap.common.config.storage; import de.bluecolored.bluemap.core.storage.Storage; +import de.bluecolored.bluemap.core.storage.b2.B2Storage; import de.bluecolored.bluemap.core.storage.file.FileStorage; import de.bluecolored.bluemap.core.storage.sql.SQLStorage; public enum StorageType { FILE (FileConfig.class, FileStorage::new), - SQL (SQLConfig.class, SQLStorage::create); + SQL (SQLConfig.class, SQLStorage::create), + B2 (B2Config.class, B2Storage::new); private final Class configType; private final StorageFactory storageFactory; diff --git a/BlueMapCore/build.gradle.kts b/BlueMapCore/build.gradle.kts index da0d6399..0828fd76 100644 --- a/BlueMapCore/build.gradle.kts +++ b/BlueMapCore/build.gradle.kts @@ -71,6 +71,9 @@ dependencies { api ("io.airlift:aircompressor:0.24") api ("org.lz4:lz4-java:1.8.0") + api ("com.backblaze.b2:b2-sdk-core:6.1.1") + api ("com.backblaze.b2:b2-sdk-httpclient:6.1.1") + api ("de.bluecolored.bluemap.api:BlueMapAPI") compileOnly ("org.jetbrains:annotations:23.0.0") diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2Storage.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2Storage.java new file mode 100644 index 00000000..d564c8af --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2Storage.java @@ -0,0 +1,294 @@ +package de.bluecolored.bluemap.core.storage.b2; + +import com.backblaze.b2.client.B2ListFilesIterable; +import com.backblaze.b2.client.B2StorageClient; +import com.backblaze.b2.client.B2StorageClientFactory; +import com.backblaze.b2.client.contentHandlers.B2ContentMemoryWriter; +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.exceptions.B2NotFoundException; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ListFileNamesRequest; +import com.backblaze.b2.client.structures.B2UploadFileRequest; +import com.flowpowered.math.vector.Vector2i; +import de.bluecolored.bluemap.core.storage.*; +import de.bluecolored.bluemap.core.util.OnCloseOutputStream; + +import java.io.*; +import java.nio.file.Path; +import java.sql.Blob; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class B2Storage extends Storage { + private boolean closed = false; + private final B2StorageClient client; + private final Compression hiresCompression; + private final String bucket; + + public B2Storage(B2StorageSettings settings) { + this.client = B2StorageClientFactory.createDefaultFactory().create(settings.getApplicationKeyId(), settings.getApplicationKey(), "BlueMap/unknownversion"); + this.hiresCompression = settings.getCompression(); + this.bucket = settings.getBucket(); + } + @Override + public void initialize() throws IOException {} + + @Override + public OutputStream writeMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + return new OnCloseOutputStream(new BufferedOutputStream(compression.compress(byteOut)), () -> { + client.uploadSmallFile(B2UploadFileRequest.builder( + bucket, + path, + getFileMime(lod), + B2ByteArrayContentSource.build(byteOut.toByteArray()) + + ).build()); + }); + } + + @Override + public Optional readMapTile(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + B2ContentMemoryWriter writer = B2ContentMemoryWriter.build(); + try { + client.downloadByName(bucket, path, writer); + ByteArrayInputStream is = new ByteArrayInputStream(writer.getBytes()); + return Optional.of(new CompressedInputStream(is, compression)); + } catch (B2Exception e) { + if (e instanceof B2NotFoundException) { + return Optional.empty(); + } else { + throw new IOException(e); + } + } + } + + @Override + public Optional readMapTileInfo(String mapId, int lod, Vector2i tile) throws IOException { + Compression compression = lod == 0 ? this.hiresCompression : Compression.NONE; + String path = getFilePath(mapId, lod, tile); + try { + B2FileVersion info = client.getFileInfoByName(bucket, path); + return Optional.of(new TileInfo() { + @Override + public CompressedInputStream readMapTile() throws IOException { + return B2Storage.this.readMapTile(mapId, lod, tile) + .orElseThrow(() -> new IOException("Tile no longer present!")); + } + + @Override + public Compression getCompression() { + return compression; + } + + @Override + public long getSize() { + return info.getContentLength(); + } + + @Override + public long getLastModified() { + return info.getUploadTimestamp(); + } + }); + } catch (B2Exception e) { + if (e instanceof B2NotFoundException) { + return Optional.empty(); + } else { + throw new IOException(e); + } + } + } + + @Override + public void deleteMapTile(String mapId, int lod, Vector2i tile) throws IOException { + String path = getFilePath(mapId, lod, tile); + try { + client.hideFile(bucket, path); + } catch (B2Exception e) { + if (!(e instanceof B2NotFoundException)) { + throw new IOException(e); + } + } + } + + @Override + public OutputStream writeMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + return new OnCloseOutputStream(new BufferedOutputStream(byteOut), () -> { + client.uploadSmallFile(B2UploadFileRequest.builder( + bucket, + path, + "b2/x-auto", + B2ByteArrayContentSource.build(byteOut.toByteArray()) + + ).build()); + }); + } + + @Override + public Optional readMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + B2ContentMemoryWriter writer = B2ContentMemoryWriter.build(); + try { + client.downloadByName(bucket, path, writer); + ByteArrayInputStream is = new ByteArrayInputStream(writer.getBytes()); + return Optional.of(is); + } catch (B2Exception e) { + if (e instanceof B2NotFoundException) { + return Optional.empty(); + } else { + throw new IOException(e); + } + } + } + + @Override + public Optional readMetaInfo(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + B2FileVersion info = client.getFileInfoByName(bucket, path); + return Optional.of(new MetaInfo() { + @Override + public InputStream readMeta() throws IOException { + return B2Storage.this.readMeta(mapId, name) + .orElseThrow(() -> new IOException("Meta no longer present!")); + } + + @Override + public long getSize() { + return info.getContentLength(); + } + }); + } catch (B2Exception e) { + if (e instanceof B2NotFoundException) { + return Optional.empty(); + } else { + throw new IOException(e); + } + } + } + + @Override + public void deleteMeta(String mapId, String name) throws IOException { + String path = getMetaFilePath(mapId, name); + try { + client.hideFile(bucket, path); + } catch (B2Exception e) { + if (!(e instanceof B2NotFoundException)) { + throw new IOException(e); + } + } + } + + @Override + public void purgeMap(String mapId, Function onProgress) throws IOException { + String directory = getFilePath(mapId); + try { + B2ListFilesIterable files = client.fileNames(B2ListFileNamesRequest.builder(bucket).setWithinFolder(directory).build()); + List filesList = StreamSupport.stream(files.spliterator(), false).collect(Collectors.toList()); + + for (int i = 0; i < filesList.size(); i++) { + var file = filesList.get(i); + client.hideFile(bucket, file.getFileName()); + + if (!onProgress.apply( + new ProgressInfo((double) (i + 1) / filesList.size()) + )) return; + } + } catch (B2Exception e) { + if (!(e instanceof B2NotFoundException)) { + throw new IOException(e); + } + } + } + + @Override + public Collection collectMapIds() throws IOException { + try { + B2ListFilesIterable files = client.fileNames(B2ListFileNamesRequest.builder(bucket).setDelimiter("/").build()); + List ids = new ArrayList<>(); + for (var file: files) { + String id = file.getFileName().split("/")[0]; + if (!ids.contains(id)) { + ids.add(id); + } + } + return ids; + } catch (B2Exception e) { + throw new IOException(e); + } + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public void close() throws IOException { + this.closed = true; + client.close(); + } + + public String getFileMime(int lod){ + if (lod == 0) { + if (hiresCompression.getFileSuffix().equals(".gz")) { + return "application/gzip"; + } else if (hiresCompression == Compression.NONE) { + return "application/json"; + } else { + return "application/octet-stream"; + } + } else { + return "image/png"; + } + } + + public String getFilePath(String mapId, int lod, Vector2i tile){ + String path = "x" + tile.getX() + "z" + tile.getY(); + char[] cs = path.toCharArray(); + List folders = new ArrayList<>(); + StringBuilder folder = new StringBuilder(); + for (char c : cs){ + folder.append(c); + if (c >= '0' && c <= '9'){ + folders.add(folder.toString()); + folder.delete(0, folder.length()); + } + } + String fileName = folders.remove(folders.size() - 1); + + String p = getFilePath(mapId) + "/tiles/" + Integer.toString(lod); + for (String s : folders){ + p += "/" + s; + } + + if (lod == 0) { + return p + "/" + (fileName + ".json" + hiresCompression.getFileSuffix()); + } else { + return p + "/" + (fileName + ".png"); + } + } + + public String getFilePath(String mapId) { + return mapId; + } + + public String getMetaFilePath(String mapId, String name) { + return getFilePath(mapId) + "/" + escapeMetaName(name); + } +} diff --git a/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2StorageSettings.java b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2StorageSettings.java new file mode 100644 index 00000000..51f7098f --- /dev/null +++ b/BlueMapCore/src/main/java/de/bluecolored/bluemap/core/storage/b2/B2StorageSettings.java @@ -0,0 +1,15 @@ +package de.bluecolored.bluemap.core.storage.b2; + +import de.bluecolored.bluemap.core.storage.Compression; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.Optional; + +public interface B2StorageSettings { + String getApplicationKeyId(); + String getApplicationKey(); + String getBucket(); + Compression getCompression(); +} \ No newline at end of file