Attachment mirroring

api/src/main/java/com/discordsrv/api/discord/api/entity/channel/DiscordDMChannel.java

@@ -26,6 +26,7 @@ package com.discordsrv.api.discord.api.entity.channel;
 import com.discordsrv.api.DiscordSRVApi;
 import com.discordsrv.api.discord.api.entity.DiscordUser;
 import net.dv8tion.jda.api.entities.PrivateChannel;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * A Discord direct message channel.
@@ -36,6 +37,7 @@ public interface DiscordDMChannel extends DiscordMessageChannel {
      * Gets the {@link DiscordUser} that is associated with this direct message channel.
      * @return the user this direct message is with
      */
+    @Nullable
     DiscordUser getUser();
 
     /**

api/src/main/java/com/discordsrv/api/discord/api/entity/channel/DiscordMessageChannel.java

@@ -30,6 +30,9 @@ import com.discordsrv.api.discord.api.entity.message.SendableDiscordMessage;
 import net.dv8tion.jda.api.entities.MessageChannel;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -40,12 +43,25 @@ public interface DiscordMessageChannel extends Snowflake {
     /**
      * Sends the provided message to the channel.
      *
-     * @param message the channel to send to the channel
+     * @param message the message to send to the channel
      * @return a future returning the message after being sent
-     * @throws com.discordsrv.api.discord.api.exception.NotReadyException if DiscordSRV is not ready, {@link com.discordsrv.api.DiscordSRVApi#isReady()}
      */
     @NotNull
-    CompletableFuture<ReceivedDiscordMessage> sendMessage(@NotNull SendableDiscordMessage message);
+    default CompletableFuture<ReceivedDiscordMessage> sendMessage(@NotNull SendableDiscordMessage message) {
+        return sendMessage(message, Collections.emptyMap());
+    }
+
+    /**
+     * Sends the provided message to the channel with the provided attachments.
+     *
+     * @param message the message to send to the channel
+     * @param attachments the attachments (in a map of file name and input stream pairs) to include in the message, the streams will be closed upon execution
+     * @return a future returning the message after being sent
+     */
+    CompletableFuture<ReceivedDiscordMessage> sendMessage(
+            @NotNull SendableDiscordMessage message,
+            @NotNull Map<String, InputStream> attachments
+    );
 
     /**
      * Deletes the message identified by the id.
@@ -62,7 +78,6 @@ public interface DiscordMessageChannel extends Snowflake {
      * @param id the id of the message to edit
      * @param message the new message content
      * @return a future returning the message after being edited
-     * @throws com.discordsrv.api.discord.api.exception.NotReadyException if DiscordSRV is not ready, {@link com.discordsrv.api.DiscordSRVApi#isReady()}
      */
     @NotNull
     CompletableFuture<ReceivedDiscordMessage> editMessageById(long id, @NotNull SendableDiscordMessage message);

api/src/main/java/com/discordsrv/api/discord/api/entity/message/ReceivedDiscordMessage.java

@@ -137,10 +137,14 @@ public interface ReceivedDiscordMessage extends SendableDiscordMessage, Snowflak
 
         private final String fileName;
         private final String url;
+        private final String proxyUrl;
+        private final int sizeBytes;
 
-        public Attachment(String fileName, String url) {
+        public Attachment(String fileName, String url, String proxyUrl, int sizeBytes) {
             this.fileName = fileName;
             this.url = url;
+            this.proxyUrl = proxyUrl;
+            this.sizeBytes = sizeBytes;
         }
 
         public String fileName() {
@@ -150,5 +154,13 @@ public interface ReceivedDiscordMessage extends SendableDiscordMessage, Snowflak
         public String url() {
             return url;
         }
+
+        public String proxyUrl() {
+            return proxyUrl;
+        }
+
+        public int sizeBytes() {
+            return sizeBytes;
+        }
     }
 }

api/src/main/java/com/discordsrv/api/discord/api/entity/message/SendableDiscordMessage.java

@@ -198,9 +198,6 @@ public interface SendableDiscordMessage {
         @NotNull
         Builder setWebhookAvatarUrl(String webhookAvatarUrl);
 
-        @NotNull
-        Builder convertToNonWebhook();
-
         /**
          * Builds a {@link SendableDiscordMessage} from this builder.
          * @return the new {@link SendableDiscordMessage}
@@ -265,9 +262,6 @@ public interface SendableDiscordMessage {
         @NotNull
         Formatter applyPlaceholderService();
 
-        @NotNull
-        Formatter convertToNonWebhook();
-
         @NotNull
         SendableDiscordMessage build();
     }

api/src/main/java/com/discordsrv/api/discord/api/entity/message/impl/SendableDiscordMessageImpl.java

@@ -48,11 +48,14 @@ public class SendableDiscordMessageImpl implements SendableDiscordMessage {
     private final String webhookUsername;
     private final String webhookAvatarUrl;
 
-    protected SendableDiscordMessageImpl(String content,
-                                         List<DiscordMessageEmbed> embeds,
-                                         Set<AllowedMention> allowedMentions,
-                                         String webhookUsername,
-                                         String webhookAvatarUrl) {
+
+    protected SendableDiscordMessageImpl(
+            String content,
+            List<DiscordMessageEmbed> embeds,
+            Set<AllowedMention> allowedMentions,
+            String webhookUsername,
+            String webhookAvatarUrl
+    ) {
         this.content = content;
         this.embeds = embeds;
         this.allowedMentions = allowedMentions;
@@ -167,20 +170,6 @@ public class SendableDiscordMessageImpl implements SendableDiscordMessage {
             return this;
         }
 
-        @Override
-        public @NotNull Builder convertToNonWebhook() {
-            String webhookUsername = this.webhookUsername;
-            if (webhookUsername == null) {
-                return this;
-            }
-
-            // TODO: configuration?
-            this.content = webhookUsername + " > " + content;
-            this.webhookUsername = null;
-            this.webhookAvatarUrl = null;
-            return this;
-        }
-
         @Override
         public @NotNull SendableDiscordMessage build() {
             return new SendableDiscordMessageImpl(content, embeds, allowedMentions, webhookUsername, webhookAvatarUrl);
@@ -238,12 +227,6 @@ public class SendableDiscordMessageImpl implements SendableDiscordMessage {
             return this;
         }
 
-        @Override
-        public @NotNull Formatter convertToNonWebhook() {
-            builder.convertToNonWebhook();
-            return this;
-        }
-
         private Function<Matcher, Object> wrapFunction(Function<Matcher, Object> function) {
             return matcher -> {
                 Object result = function.apply(matcher);

common/src/main/java/com/discordsrv/common/config/main/channels/MirroringConfig.java

@@ -37,4 +37,21 @@ public class MirroringConfig {
 
     @Comment("Content to append to the beginning of a message if the message is replying to another")
     public String replyFormat = "[In reply to %user_effective_name|user_name%](%message_jump_url%)\n";
+
+    @Comment("Attachment related options")
+    public AttachmentConfig attachments = new AttachmentConfig();
+
+    @ConfigSerializable
+    public static class AttachmentConfig {
+
+        @Comment("Maximum size (in kB) to download and re-upload, set to 0 for unlimited or -1 to disable re-uploading.\n"
+                + "The default value is -1 (disabled)\n\n"
+                + "When this is enabled files smaller than the specified limit are downloaded and then re-uploaded to each mirror channel individually.\n"
+                + "Please consider limiting the users allowed to attach files if this is enabled,\n"
+                + "as spam of large files may result in a lot of downstream and upstream data usage")
+        public int maximumSizeKb = -1;
+
+        @Comment("If attachments should be placed into a embed in mirrored messages instead of re-uploading")
+        public boolean embedAttachments = true;
+    }
 }

common/src/main/java/com/discordsrv/common/discord/api/entity/channel/AbstractDiscordGuildMessageChannel.java

@@ -20,7 +20,7 @@ package com.discordsrv.common.discord.api.entity.channel;
 
 import club.minnced.discord.webhook.WebhookClient;
 import club.minnced.discord.webhook.receive.ReadonlyMessage;
-import club.minnced.discord.webhook.send.WebhookMessage;
+import club.minnced.discord.webhook.send.WebhookMessageBuilder;
 import com.discordsrv.api.discord.api.entity.channel.DiscordGuildMessageChannel;
 import com.discordsrv.api.discord.api.entity.guild.DiscordGuild;
 import com.discordsrv.api.discord.api.entity.message.ReceivedDiscordMessage;
@@ -31,10 +31,11 @@ import com.discordsrv.common.discord.api.entity.message.ReceivedDiscordMessageIm
 import com.discordsrv.common.discord.api.entity.message.util.SendableDiscordMessageUtil;
 import net.dv8tion.jda.api.entities.GuildMessageChannel;
 import net.dv8tion.jda.api.entities.Message;
-import net.dv8tion.jda.api.entities.MessageChannel;
 import net.dv8tion.jda.api.requests.restaction.MessageAction;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.InputStream;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.BiFunction;
 
@@ -69,22 +70,35 @@ public abstract class AbstractDiscordGuildMessageChannel<T extends GuildMessageC
     }
 
     @Override
-    public @NotNull CompletableFuture<ReceivedDiscordMessage> sendMessage(@NotNull SendableDiscordMessage message) {
-        return message(message, WebhookClient::send, MessageChannel::sendMessage);
+    public CompletableFuture<ReceivedDiscordMessage> sendMessage(
+            @NotNull SendableDiscordMessage message, @NotNull Map<String, InputStream> attachments
+    ) {
+        return message(message, (webhookClient, webhookMessage) -> {
+            for (Map.Entry<String, InputStream> entry : attachments.entrySet()) {
+                webhookMessage.addFile(entry.getKey(), entry.getValue());
+            }
+            return webhookClient.send(webhookMessage.build());
+        }, (channel, msg) -> {
+            MessageAction action = channel.sendMessage(msg);
+            for (Map.Entry<String, InputStream> entry : attachments.entrySet()) {
+                action = action.addFile(entry.getValue(), entry.getKey());
+            }
+            return action;
+        });
     }
 
     @Override
     public @NotNull CompletableFuture<ReceivedDiscordMessage> editMessageById(long id, @NotNull SendableDiscordMessage message) {
         return message(
                 message,
-                (client, msg) -> client.edit(id, msg),
+                (client, msg) -> client.edit(id, msg.build()),
                 (textChannel, msg) -> textChannel.editMessageById(id, msg)
         );
     }
 
     private CompletableFuture<ReceivedDiscordMessage> message(
             SendableDiscordMessage message,
-            BiFunction<WebhookClient, WebhookMessage, CompletableFuture<ReadonlyMessage>> webhookFunction,
+            BiFunction<WebhookClient, WebhookMessageBuilder, CompletableFuture<ReadonlyMessage>> webhookFunction,
             BiFunction<T, Message, MessageAction> jdaFunction) {
         return discordSRV.discordAPI().mapExceptions(() -> {
             CompletableFuture<ReceivedDiscordMessage> future;

common/src/main/java/com/discordsrv/common/discord/api/entity/channel/DiscordDMChannelImpl.java

@@ -27,8 +27,12 @@ import com.discordsrv.common.discord.api.entity.DiscordUserImpl;
 import com.discordsrv.common.discord.api.entity.message.ReceivedDiscordMessageImpl;
 import com.discordsrv.common.discord.api.entity.message.util.SendableDiscordMessageUtil;
 import net.dv8tion.jda.api.entities.PrivateChannel;
+import net.dv8tion.jda.api.entities.User;
+import net.dv8tion.jda.api.requests.restaction.MessageAction;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.InputStream;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 
@@ -38,7 +42,8 @@ public class DiscordDMChannelImpl extends AbstractDiscordMessageChannel<PrivateC
 
     public DiscordDMChannelImpl(DiscordSRV discordSRV, PrivateChannel privateChannel) {
         super(discordSRV, privateChannel);
-        this.user = new DiscordUserImpl(discordSRV, privateChannel.getUser());
+        User user = privateChannel.getUser();
+        this.user = user != null ? new DiscordUserImpl(discordSRV, user) : null;
     }
 
     @Override
@@ -47,14 +52,20 @@ public class DiscordDMChannelImpl extends AbstractDiscordMessageChannel<PrivateC
     }
 
     @Override
-    public @NotNull CompletableFuture<ReceivedDiscordMessage> sendMessage(@NotNull SendableDiscordMessage message) {
+    public CompletableFuture<ReceivedDiscordMessage> sendMessage(
+            @NotNull SendableDiscordMessage message,
+            @NotNull Map<String, InputStream> attachments
+    ) {
         if (message.isWebhookMessage()) {
             throw new IllegalArgumentException("Cannot send webhook messages to DMChannels");
         }
 
-        CompletableFuture<ReceivedDiscordMessage> future = channel
-                .sendMessage(SendableDiscordMessageUtil.toJDA(message))
-                .submit()
+        MessageAction action = channel.sendMessage(SendableDiscordMessageUtil.toJDA(message));
+        for (Map.Entry<String, InputStream> entry : attachments.entrySet()) {
+            action = action.addFile(entry.getValue(), entry.getKey());
+        }
+
+        CompletableFuture<ReceivedDiscordMessage> future = action.submit()
                 .thenApply(msg -> ReceivedDiscordMessageImpl.fromJDA(discordSRV, msg));
 
         return discordSRV.discordAPI().mapExceptions(future);

common/src/main/java/com/discordsrv/common/discord/api/entity/message/ReceivedDiscordMessageImpl.java

@@ -97,7 +97,12 @@ public class ReceivedDiscordMessageImpl extends SendableDiscordMessageImpl imple
 
         List<Attachment> attachments = new ArrayList<>();
         for (Message.Attachment attachment : message.getAttachments()) {
-            attachments.add(new Attachment(attachment.getFileName(), attachment.getUrl()));
+            attachments.add(new Attachment(
+                    attachment.getFileName(),
+                    attachment.getUrl(),
+                    attachment.getProxyUrl(),
+                    attachment.getSize()
+            ));
         }
 
         Message referencedMessage = message.getReferencedMessage();
@@ -166,7 +171,12 @@ public class ReceivedDiscordMessageImpl extends SendableDiscordMessageImpl imple
 
         List<Attachment> attachments = new ArrayList<>();
         for (ReadonlyAttachment attachment : webhookMessage.getAttachments()) {
-            attachments.add(new Attachment(attachment.getFileName(), attachment.getUrl()));
+            attachments.add(new Attachment(
+                    attachment.getFileName(),
+                    attachment.getUrl(),
+                    attachment.getProxyUrl(),
+                    attachment.getSize()
+            ));
         }
 
         return new ReceivedDiscordMessageImpl(

common/src/main/java/com/discordsrv/common/discord/api/entity/message/util/SendableDiscordMessageUtil.java

@@ -18,7 +18,6 @@
 
 package com.discordsrv.common.discord.api.entity.message.util;
 
-import club.minnced.discord.webhook.send.WebhookMessage;
 import club.minnced.discord.webhook.send.WebhookMessageBuilder;
 import com.discordsrv.api.discord.api.entity.message.AllowedMention;
 import com.discordsrv.api.discord.api.entity.message.DiscordMessageEmbed;
@@ -72,10 +71,9 @@ public final class SendableDiscordMessageUtil {
                 .build();
     }
 
-    public static WebhookMessage toWebhook(@NotNull SendableDiscordMessage message) {
+    public static WebhookMessageBuilder toWebhook(@NotNull SendableDiscordMessage message) {
         return WebhookMessageBuilder.fromJDA(toJDA(message))
                 .setUsername(message.getWebhookUsername().orElse(null))
-                .setAvatarUrl(message.getWebhookAvatarUrl().orElse(null))
-                .build();
+                .setAvatarUrl(message.getWebhookAvatarUrl().orElse(null));
     }
 }

common/src/main/java/com/discordsrv/common/messageforwarding/discord/DiscordMessageMirroringModule.java

@@ -43,8 +43,14 @@ import com.discordsrv.common.logging.NamedLogger;
 import com.discordsrv.common.module.type.AbstractModule;
 import com.github.benmanes.caffeine.cache.Cache;
 import net.dv8tion.jda.api.entities.Message;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
 import org.apache.commons.lang3.tuple.Pair;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -77,6 +83,8 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
 
         List<Pair<DiscordGuildMessageChannel, OrDefault<MirroringConfig>>> mirrorChannels = new ArrayList<>();
         List<CompletableFuture<DiscordThreadChannel>> futures = new ArrayList<>();
+        Map<ReceivedDiscordMessage.Attachment, byte[]> attachments = new LinkedHashMap<>();
+        DiscordMessageEmbed.Builder attachmentEmbed = DiscordMessageEmbed.builder().setDescription("Attachments");
 
         for (Map.Entry<GameChannel, OrDefault<BaseChannelConfig>> entry : channels.entrySet()) {
             OrDefault<BaseChannelConfig> channelConfig = entry.getValue();
@@ -95,6 +103,43 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
                 continue;
             }
 
+            OrDefault<MirroringConfig.AttachmentConfig> attachmentConfig = config.map(cfg -> cfg.attachments);
+            int maxSize = attachmentConfig.get(cfg -> cfg.maximumSizeKb, -1);
+            boolean embedAttachments = attachmentConfig.get(cfg -> cfg.embedAttachments, true);
+            if (maxSize >= 0 || embedAttachments) {
+                for (ReceivedDiscordMessage.Attachment attachment : message.getAttachments()) {
+                    if (attachments.containsKey(attachment)) {
+                        continue;
+                    }
+
+                    if (maxSize == 0 || attachment.sizeBytes() <= (maxSize * 1000)) {
+                        Request request = new Request.Builder()
+                                .url(attachment.proxyUrl())
+                                .get()
+                                .build();
+
+                        byte[] bytes = null;
+                        try (Response response = discordSRV.httpClient().newCall(request).execute()) {
+                            ResponseBody body = response.body();
+                            if (body != null) {
+                                bytes = body.bytes();
+                            }
+                        } catch (IOException e) {
+                            discordSRV.logger().error("Failed to download attachment for mirroring", e);
+                        }
+                        attachments.put(attachment, bytes);
+                        continue;
+                    }
+
+                    if (!embedAttachments) {
+                        continue;
+                    }
+
+                    attachments.put(attachment, null);
+                    attachmentEmbed.addField(attachment.fileName(), "[link](" + attachment.url() + ")", true);
+                }
+            }
+
             List<Long> channelIds = iChannelConfig.channelIds();
             if (channelIds != null) {
                 for (Long channelId : channelIds) {
@@ -118,10 +163,29 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
             for (Pair<DiscordGuildMessageChannel, OrDefault<MirroringConfig>> pair : mirrorChannels) {
                 DiscordGuildMessageChannel mirrorChannel = pair.getKey();
                 OrDefault<MirroringConfig> config = pair.getValue();
-                SendableDiscordMessage sendableMessage = convert(event.getDiscordMessage(), mirrorChannel, config);
+                OrDefault<MirroringConfig.AttachmentConfig> attachmentConfig = config.map(cfg -> cfg.attachments);
+
+                SendableDiscordMessage.Builder messageBuilder = convert(event.getDiscordMessage(), mirrorChannel, config);
+                if (!attachmentEmbed.getFields().isEmpty() && attachmentConfig.get(cfg -> cfg.embedAttachments, true)) {
+                    messageBuilder.addEmbed(attachmentEmbed.build());
+                }
+
+                int maxSize = attachmentConfig.get(cfg -> cfg.maximumSizeKb, -1);
+                Map<String, InputStream> currentAttachments;
+                if (!attachments.isEmpty() && maxSize > 0) {
+                    currentAttachments = new LinkedHashMap<>();
+                    attachments.forEach((attachment, bytes) -> {
+                        if (bytes != null && attachment.sizeBytes() <= maxSize) {
+                            currentAttachments.put(attachment.fileName(), new ByteArrayInputStream(bytes));
+                        }
+                    });
+                } else {
+                    currentAttachments = Collections.emptyMap();
+                }
 
                 CompletableFuture<Pair<ReceivedDiscordMessage, OrDefault<MirroringConfig>>> future =
-                        mirrorChannel.sendMessage(sendableMessage).thenApply(msg -> Pair.of(msg, config));
+                        mirrorChannel.sendMessage(messageBuilder.build(), currentAttachments)
+                                .thenApply(msg -> Pair.of(msg, config));
 
                 messageFutures.add(future);
                 future.exceptionally(t2 -> {
@@ -155,7 +219,7 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
                 continue;
             }
 
-            SendableDiscordMessage sendableMessage = convert(message, channel, reference.config);
+            SendableDiscordMessage sendableMessage = convert(message, channel, reference.config).build();
             channel.editMessageById(reference.messageId, sendableMessage).whenComplete((v, t) -> {
                 if (t != null) {
                     discordSRV.logger().error("Failed to update mirrored message in " + channel);
@@ -188,7 +252,7 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
     /**
      * Converts a given received message to a sendable message.
      */
-    private SendableDiscordMessage convert(
+    private SendableDiscordMessage.Builder convert(
             ReceivedDiscordMessage message,
             DiscordGuildMessageChannel destinationChannel,
             OrDefault<MirroringConfig> config
@@ -252,7 +316,7 @@ public class DiscordMessageMirroringModule extends AbstractModule<DiscordSRV> {
         for (DiscordMessageEmbed embed : message.getEmbeds()) {
             builder.addEmbed(embed);
         }
-        return builder.build();
+        return builder;
     }
 
     private MessageReference getReference(ReceivedDiscordMessage message, OrDefault<MirroringConfig> config) {