Add config options for enabling and filtering join address gathering

Affects issues:
- Close #3631

Plan/common/src/main/java/com/djrapitops/plan/delivery/domain/JoinAddressCount.java

@@ -28,7 +28,7 @@ import java.util.Objects;
  */
 public class JoinAddressCount implements Comparable<JoinAddressCount> {
 
-    private final int count;
+    private int count;
     private String joinAddress;
 
     public JoinAddressCount(Map.Entry<String, Integer> entry) {
@@ -52,6 +52,10 @@ public class JoinAddressCount implements Comparable<JoinAddressCount> {
         return count;
     }
 
+    public void setCount(int count) {
+        this.count = count;
+    }
+
     @Override
     public int compareTo(@NotNull JoinAddressCount other) {
         return String.CASE_INSENSITIVE_ORDER.compare(this.joinAddress, other.joinAddress);

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java

@@ -35,6 +35,7 @@ import com.djrapitops.plan.identification.Server;
 import com.djrapitops.plan.identification.ServerInfo;
 import com.djrapitops.plan.identification.ServerUUID;
 import com.djrapitops.plan.settings.config.PlanConfig;
+import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
 import com.djrapitops.plan.settings.config.paths.DisplaySettings;
 import com.djrapitops.plan.settings.config.paths.TimeSettings;
 import com.djrapitops.plan.settings.locale.Locale;
@@ -49,6 +50,7 @@ import com.djrapitops.plan.storage.database.queries.analysis.PlayerRetentionQuer
 import com.djrapitops.plan.storage.database.queries.objects.*;
 import com.djrapitops.plan.storage.database.queries.objects.playertable.NetworkTablePlayersQuery;
 import com.djrapitops.plan.storage.database.queries.objects.playertable.ServerTablePlayersQuery;
+import com.djrapitops.plan.storage.database.sql.tables.JoinAddressTable;
 import com.djrapitops.plan.utilities.comparators.SessionStartComparator;
 import com.djrapitops.plan.utilities.dev.Untrusted;
 import com.djrapitops.plan.utilities.java.Maps;
@@ -161,25 +163,52 @@ public class JSONFactory {
         return db.query(PlayerRetentionQueries.fetchRetentionData());
     }
 
+    private static void removeFiltered(Map<UUID, String> addressByPlayerUUID, List<String> filteredJoinAddresses) {
+        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of("play.example.com"))) return;
+
+        Set<UUID> toRemove = new HashSet<>();
+        // Remove filtered addresses from the data
+        for (Map.Entry<UUID, String> entry : addressByPlayerUUID.entrySet()) {
+            if (filteredJoinAddresses.contains(entry.getValue())) {
+                toRemove.add(entry.getKey());
+            }
+        }
+        for (UUID playerUUID : toRemove) {
+            addressByPlayerUUID.put(playerUUID, JoinAddressTable.DEFAULT_VALUE_FOR_LOOKUP);
+        }
+    }
+
     public PlayerJoinAddresses playerJoinAddresses(ServerUUID serverUUID, boolean includeByPlayerMap) {
         Database db = dbSystem.getDatabase();
+        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
         if (includeByPlayerMap) {
             Map<UUID, String> addresses = db.query(JoinAddressQueries.latestJoinAddressesOfPlayers(serverUUID));
+
+            removeFiltered(addresses, filteredJoinAddresses);
+
             return new PlayerJoinAddresses(
                     addresses.values().stream().distinct().sorted().collect(Collectors.toList()),
                     addresses
             );
         } else {
-            return new PlayerJoinAddresses(db.query(JoinAddressQueries.uniqueJoinAddresses(serverUUID)), null);
+            List<String> addresses = db.query(JoinAddressQueries.uniqueJoinAddresses(serverUUID));
+            addresses.removeAll(filteredJoinAddresses);
+            return new PlayerJoinAddresses(addresses, null);
         }
     }
 
     public PlayerJoinAddresses playerJoinAddresses(boolean includeByPlayerMap) {
         Database db = dbSystem.getDatabase();
-        return new PlayerJoinAddresses(
+        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
-                db.query(JoinAddressQueries.uniqueJoinAddresses()),
+        List<String> unique = db.query(JoinAddressQueries.uniqueJoinAddresses());
-                includeByPlayerMap ? db.query(JoinAddressQueries.latestJoinAddressesOfPlayers()) : null
+        unique.removeAll(filteredJoinAddresses);
-        );
+        if (includeByPlayerMap) {
+            Map<UUID, String> latest = db.query(JoinAddressQueries.latestJoinAddressesOfPlayers());
+            removeFiltered(latest, filteredJoinAddresses);
+            return new PlayerJoinAddresses(unique, latest);
+        } else {
+            return new PlayerJoinAddresses(unique, null);
+        }
     }
 
     public List<Map<String, Object>> serverSessionsAsJSONMap(ServerUUID serverUUID) {

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/graphs/GraphJSONCreator.java

@@ -478,19 +478,51 @@ public class GraphJSONCreator {
         return mapToJson(pieColors, joinAddresses);
     }
 
+    private static void removeFilteredAddresses(List<JoinAddressCount> addresses, List<String> filteredJoinAddresses) {
+        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of("play.example.com"))) return;
+
+        List<JoinAddressCount> addressesToRemove = addresses.stream()
+                .filter(address -> filteredJoinAddresses.contains(address.getJoinAddress()))
+                .collect(Collectors.toList());
+
+        if (!addressesToRemove.isEmpty()) {
+            Optional<JoinAddressCount> foundUnknownAddressCount = addresses.stream()
+                    .filter(address -> address.getJoinAddress().equals(JoinAddressTable.DEFAULT_VALUE_FOR_LOOKUP))
+                    .findFirst();
+            JoinAddressCount unknownAddressCount;
+            if (foundUnknownAddressCount.isEmpty()) {
+                unknownAddressCount = new JoinAddressCount(JoinAddressTable.DEFAULT_VALUE_FOR_LOOKUP, 0);
+                addresses.add(unknownAddressCount);
+            } else {
+                unknownAddressCount = foundUnknownAddressCount.get();
+            }
+
+            for (JoinAddressCount toRemove : addressesToRemove) {
+                unknownAddressCount.setCount(unknownAddressCount.getCount() + toRemove.getCount());
+                addresses.remove(toRemove);
+            }
+        }
+    }
+
     private Map<String, Object> mapToJson(String[] pieColors, List<DateObj<Map<String, Integer>>> joinAddresses) {
         for (DateObj<Map<String, Integer>> addressesByDate : joinAddresses) {
             translateUnknown(addressesByDate.getValue());
         }
 
+        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+
         List<JoinAddressCounts> joinAddressCounts = joinAddresses.stream()
-                .map(addressesOnDay -> new JoinAddressCounts(
+                .map(addressesOnDay -> {
-                        addressesOnDay.getDate(),
+                    List<JoinAddressCount> addresses = addressesOnDay.getValue().entrySet()
-                        addressesOnDay.getValue().entrySet()
+                            .stream()
-                                .stream()
+                            .map(JoinAddressCount::new)
-                                .map(JoinAddressCount::new)
+                            .sorted()
-                                .sorted()
+                            .collect(Collectors.toList());
-                                .collect(Collectors.toList())))
+
+                    removeFilteredAddresses(addresses, filteredJoinAddresses);
+
+                    return new JoinAddressCounts(addressesOnDay.getDate(), addresses);
+                })
                 .sorted(new DateHolderOldestComparator())
                 .collect(Collectors.toList());
 

Plan/common/src/main/java/com/djrapitops/plan/gathering/JoinAddressValidator.java

@@ -25,6 +25,7 @@ import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.List;
 
 /**
  * Utility for validating and sanitizing join addresses.
@@ -35,6 +36,7 @@ import java.net.URISyntaxException;
 public class JoinAddressValidator {
 
     private final PlanConfig config;
+    private List<String> filteredAddresses;
 
     @Inject
     public JoinAddressValidator(PlanConfig config) {
@@ -42,9 +44,15 @@ public class JoinAddressValidator {
         this.config = config;
     }
 
+    private void prepareFilteredAddresses() {
+        if (filteredAddresses == null) {
+            filteredAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+        }
+    }
+
     @Untrusted
     public String sanitize(@Untrusted String address) {
-        if (address == null) return "";
+        if (address == null || config.isFalse(DataGatheringSettings.JOIN_ADDRESSES)) return "";
         if (!address.isEmpty()) {
             // Remove port
             if (address.contains(":")) {
@@ -61,6 +69,10 @@ public class JoinAddressValidator {
             if (config.isFalse(DataGatheringSettings.PRESERVE_JOIN_ADDRESS_CASE)) {
                 address = StringUtils.lowerCase(address);
             }
+            prepareFilteredAddresses();
+            if (filteredAddresses.contains(address)) {
+                address = "";
+            }
         }
         return address;
     }

Plan/common/src/main/java/com/djrapitops/plan/settings/config/changes/ConfigUpdater.java

@@ -177,6 +177,9 @@ public class ConfigUpdater {
                 new ConfigChange.Removed("Plugin.Use_Legacy_Frontend"),
                 new ConfigChange.Removed("Customized_files.Enable_web_dev_mode"),
                 new ConfigChange.Removed("Customized_files.Plan"),
+
+                new ConfigChange.Moved("Data_gathering.Preserve_join_address_case", "Data_gathering.Join_addresses.Preserve_case"),
+                new ConfigChange.Moved("Data_gathering.Preserve_invalid_join_addresses", "Data_gathering.Join_addresses.Preserve_invalid"),
         };
     }
 

Plan/common/src/main/java/com/djrapitops/plan/settings/config/paths/DataGatheringSettings.java

@@ -18,8 +18,11 @@ package com.djrapitops.plan.settings.config.paths;
 
 import com.djrapitops.plan.settings.config.paths.key.BooleanSetting;
 import com.djrapitops.plan.settings.config.paths.key.Setting;
+import com.djrapitops.plan.settings.config.paths.key.StringListSetting;
 import com.djrapitops.plan.settings.config.paths.key.StringSetting;
 
+import java.util.List;
+
 /**
  * {@link Setting} values that are in "Data_gathering" section.
  *
@@ -34,8 +37,10 @@ public class DataGatheringSettings {
     public static final Setting<Boolean> DISK_SPACE = new BooleanSetting("Data_gathering.Disk_space");
     public static final Setting<Boolean> LOG_UNKNOWN_COMMANDS = new BooleanSetting("Data_gathering.Commands.Log_unknown");
     public static final Setting<Boolean> COMBINE_COMMAND_ALIASES = new BooleanSetting("Data_gathering.Commands.Log_aliases_as_main_command");
-    public static final Setting<Boolean> PRESERVE_JOIN_ADDRESS_CASE = new BooleanSetting("Data_gathering.Preserve_join_address_case");
+    public static final Setting<Boolean> JOIN_ADDRESSES = new BooleanSetting("Data_gathering.Join_addresses.Enabled");
-    public static final Setting<Boolean> PRESERVE_INVALID_JOIN_ADDRESS = new BooleanSetting("Data_gathering.Preserve_invalid_join_addresses");
+    public static final Setting<Boolean> PRESERVE_JOIN_ADDRESS_CASE = new BooleanSetting("Data_gathering.Join_addresses.Preserve_case");
+    public static final Setting<Boolean> PRESERVE_INVALID_JOIN_ADDRESS = new BooleanSetting("Data_gathering.Join_addresses.Preserve_invalid");
+    public static final Setting<List<String>> FILTER_JOIN_ADDRESSES = new StringListSetting("Data_gathering.Join_addresses.Filter_out_from_data");
 
     private DataGatheringSettings() {
         /* static variable class */

Plan/common/src/main/resources/assets/plan/bungeeconfig.yml

@@ -114,9 +114,14 @@ Data_gathering:
   Geolocation_Download_URL: "https://geodb.playeranalytics.net/GeoLite2-Country.mmdb"
   Ping: true
   Disk_space: true
-  # Does not affect already gathered data
+  Join_addresses:
-  Preserve_join_address_case: false
+    Enabled: true
-  Preserve_invalid_join_addresses: false
+    # Does not affect already gathered data
+    Preserve_case: false
+    Preserve_invalid: false
+    # Replaces these join addresses with unknown
+    Filter_out_from_data:
+      - "play.example.com"
 # -----------------------------------------------------
 # Supported time units: MILLISECONDS, SECONDS, MINUTES, HOURS, DAYS
 # -----------------------------------------------------

Plan/common/src/main/resources/assets/plan/config.yml

@@ -118,9 +118,14 @@ Data_gathering:
   Commands:
     Log_unknown: false
     Log_aliases_as_main_command: true
-  # Does not affect already gathered data
+  Join_addresses:
-  Preserve_join_address_case: false
+    Enabled: true
-  Preserve_invalid_join_addresses: false
+    # Does not affect already gathered data
+    Preserve_case: false
+    Preserve_invalid: false
+    # Replaces these join addresses with unknown
+    Filter_out_from_data:
+      - "play.example.com"
 # -----------------------------------------------------
 # Supported time units: MILLISECONDS, SECONDS, MINUTES, HOURS, DAYS
 # -----------------------------------------------------