From 0b061f0a5e26381364c835cc940488e4dc6d5bd7 Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Sat, 3 Jun 2017 12:58:14 +0300 Subject: [PATCH] Unique Players, Outlier removal, Gender pie removed --- .../java/com/djrapitops/plan/Settings.java | 4 +- .../djrapitops/plan/data/AnalysisData.java | 99 ++++++++++++----- .../djrapitops/plan/data/RawAnalysisData.java | 40 +++---- .../plan/database/databases/SQLDB.java | 102 +++--------------- .../plan/database/tables/UsersTable.java | 4 - .../ui/graphs/PlayerActivityGraphCreator.java | 27 +++++ .../plan/ui/graphs/PunchCardGraphCreator.java | 75 +++++++++---- .../plan/ui/webserver/Response.java | 8 +- .../plan/utilities/PlaceholderUtils.java | 15 ++- .../plan/utilities/analysis/Analysis.java | 39 ++++--- .../utilities/analysis/AnalysisUtils.java | 78 +++++++++++++- .../analysis/locations/LocationAnalysis.java | 96 +++++++++++++++++ .../utilities/analysis/locations/Point.java | 64 +++++++++++ Plan/src/main/resources/analysis.html | 76 ++----------- Plan/src/main/resources/config.yml | 5 +- Plan/src/main/resources/plugin.yml | 2 +- 16 files changed, 457 insertions(+), 277 deletions(-) create mode 100644 Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/LocationAnalysis.java create mode 100644 Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/Point.java diff --git a/Plan/src/main/java/com/djrapitops/plan/Settings.java b/Plan/src/main/java/com/djrapitops/plan/Settings.java index 16539cd4c..3812ec9fc 100644 --- a/Plan/src/main/java/com/djrapitops/plan/Settings.java +++ b/Plan/src/main/java/com/djrapitops/plan/Settings.java @@ -16,6 +16,7 @@ public enum Settings { ANALYSIS_REFRESH_ON_ENABLE("Settings.Cache.AnalysisCache.RefreshAnalysisCacheOnEnable"), ANALYSIS_LOG_TO_CONSOLE("Settings.Analysis.LogProgressOnConsole"), ANALYSIS_LOG_FINISHED("Settings.Analysis.NotifyWhenFinished"), + ANALYSIS_REMOVE_OUTLIERS("Settings.Analysis.RemoveOutliersFromVisualization"), ANALYSIS_EXPORT("Settings.Analysis.Export.Enabled"), SHOW_ALTERNATIVE_IP("Settings.WebServer.ShowAlternativeServerIP"), USE_ALTERNATIVE_UI("Settings.UseTextUI"), @@ -78,9 +79,6 @@ public enum Settings { HCOLOR_GMP_1("Customization.Colors.HTML.GamemodePie.Creative"), HCOLOR_GMP_2("Customization.Colors.HTML.GamemodePie.Adventure"), HCOLOR_GMP_3("Customization.Colors.HTML.GamemodePie.Spectator"), - HCOLOR_GENP_M("Customization.Colors.HTML.GenderPie.Male"), - HCOLOR_GENP_F("Customization.Colors.HTML.GenderPie.Female"), - HCOLOR_GENP_U("Customization.Colors.HTML.GenderPie.Unknown"), // StringList HIDE_FACTIONS("Customization.Plugins.Factions.HideFactions"), HIDE_TOWNS("Customization.Plugins.Towny.HideTowns"); diff --git a/Plan/src/main/java/com/djrapitops/plan/data/AnalysisData.java b/Plan/src/main/java/com/djrapitops/plan/data/AnalysisData.java index e490764f7..9d47624db 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/AnalysisData.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/AnalysisData.java @@ -70,7 +70,14 @@ public class AnalysisData { private String geomapZ; private String geomapCodes; - private int[] genderData; + private int avgUniqJoins; + private int avgUniqJoinsDay; + private int avgUniqJoinsWeek; + private int avgUniqJoinsMonth; + + private int uniqueJoinsDay; + private int uniqueJoinsWeek; + private int uniqueJoinsMonth; /** * Class constructor. @@ -78,6 +85,13 @@ public class AnalysisData { * All data has to be set with setters to avoid NPEs. */ public AnalysisData() { + avgUniqJoins = 0; + avgUniqJoinsDay = 0; + avgUniqJoinsWeek = 0; + avgUniqJoinsMonth = 0; + uniqueJoinsDay = 0; + uniqueJoinsWeek = 0; + uniqueJoinsMonth = 0; sortablePlayersTable = Html.ERROR_NOT_SET + ""; commandUseTableHtml = Html.ERROR_NOT_SET + ""; recentPlayers = Html.ERROR_NOT_SET + ""; @@ -88,7 +102,6 @@ public class AnalysisData { sessionDistributionData = new String[]{"[]", "[]"}; playtimeDistributionData = new String[]{"[]", "[]"}; playersDataArray = new String[]{"[0]", "[\"No data\"]", "[0]", "[\"No data\"]", "[0]", "[\"No data\"]"}; - genderData = new int[]{0, 0, 0}; additionalDataReplaceMap = new HashMap<>(); } @@ -194,9 +207,6 @@ public class AnalysisData { if (!Arrays.deepEquals(this.playersDataArray, other.playersDataArray)) { return false; } - if (!Arrays.equals(this.genderData, other.genderData)) { - return false; - } return true; } @@ -862,29 +872,6 @@ public class AnalysisData { this.sessionAverage = sessionAverage; } - /** - * Get the integer array containing 3 numbers. - * - * 0 Male, 1 Female, 2 Unknown. - * - * @return for example [0, 4, 5] when 0 male, 4 female and 5 unknown. - */ - public int[] getGenderData() { - return genderData; - } - - /** - * - * Set the integer array containing 3 numbers. - * - * 0 Male, 1 Female, 2 Unknown. - * - * @param genderData for example [0, 4, 5] - */ - public void setGenderData(int[] genderData) { - this.genderData = genderData; - } - /** * Get the data for the Session Punchcard. * @@ -940,4 +927,60 @@ public class AnalysisData { public void setPlaytimeDistributionData(String[] playtimeDistributionData) { this.playtimeDistributionData = playtimeDistributionData; } + + public int getAvgUniqJoins() { + return avgUniqJoins; + } + + public int getAvgUniqJoinsDay() { + return avgUniqJoinsDay; + } + + public int getAvgUniqJoinsWeek() { + return avgUniqJoinsWeek; + } + + public int getAvgUniqJoinsMonth() { + return avgUniqJoinsMonth; + } + + public void setAvgUniqJoins(int avgUniqJoins) { + this.avgUniqJoins = avgUniqJoins; + } + + public void setAvgUniqJoinsDay(int avgUniqJoinsDay) { + this.avgUniqJoinsDay = avgUniqJoinsDay; + } + + public void setAvgUniqJoinsWeek(int avgUniqJoinsWeek) { + this.avgUniqJoinsWeek = avgUniqJoinsWeek; + } + + public void setAvgUniqJoinsMonth(int avgUniqJoinsMonth) { + this.avgUniqJoinsMonth = avgUniqJoinsMonth; + } + + public int getUniqueJoinsDay() { + return uniqueJoinsDay; + } + + public void setUniqueJoinsDay(int uniqueJoinsDay) { + this.uniqueJoinsDay = uniqueJoinsDay; + } + + public int getUniqueJoinsWeek() { + return uniqueJoinsWeek; + } + + public void setUniqueJoinsWeek(int uniqueJoinsWeek) { + this.uniqueJoinsWeek = uniqueJoinsWeek; + } + + public int getUniqueJoinsMonth() { + return uniqueJoinsMonth; + } + + public void setUniqueJoinsMonth(int uniqueJoinsMonth) { + this.uniqueJoinsMonth = uniqueJoinsMonth; + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/data/RawAnalysisData.java b/Plan/src/main/java/com/djrapitops/plan/data/RawAnalysisData.java index 8735cbf63..a73f01ba3 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/RawAnalysisData.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/RawAnalysisData.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import main.java.com.djrapitops.plan.utilities.analysis.Analysis; /** @@ -34,11 +35,11 @@ public class RawAnalysisData { private Map latestLogins; private Map playtimes; private List sessiondata; + private Map> sortedSessionData; private Map commandUse; private Map geolocations; private Map geocodes; private List registered; - private int[] genders; /** * Constructor for a new empty dataset. @@ -56,16 +57,17 @@ public class RawAnalysisData { inactive = 0; totalKills = 0; totalMobKills = 0; + totalDeaths = 0; ops = 0; ages = new ArrayList<>(); latestLogins = new HashMap<>(); playtimes = new HashMap<>(); sessiondata = new ArrayList<>(); + sortedSessionData = new HashMap<>(); commandUse = new HashMap<>(); geolocations = new HashMap<>(); geocodes = new HashMap<>(); registered = new ArrayList<>(); - genders = new int[]{0, 0, 0}; } /** @@ -368,6 +370,15 @@ public class RawAnalysisData { return sessiondata; } + public Map> getSortedSessionData() { + return sortedSessionData; + } + + public void addSessions(UUID uuid, List sessions) { + sessiondata.addAll(sessions); + sortedSessionData.put(uuid, sessions); + } + /** * * @param commandUse @@ -391,29 +402,4 @@ public class RawAnalysisData { public List getRegistered() { return registered; } - - /** - * - * @return - */ - public int[] getGenders() { - return genders; - } - - /** - * - * @param gender - */ - public void setGenders(int[] gender) { - this.genders = gender; - } - - /** - * - * @param i - * @param amount - */ - public void addToGender(int i, int amount) { - this.genders[i] = this.genders[i] + amount; - } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java index 31bf5cab2..91894d122 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java @@ -31,8 +31,6 @@ import org.bukkit.scheduler.BukkitRunnable; */ public abstract class SQLDB extends Database { - final Plan plugin; - private final boolean supportsModification; private Connection connection; @@ -44,7 +42,6 @@ public abstract class SQLDB extends Database { */ public SQLDB(Plan plugin, boolean supportsModification) { super(plugin); - this.plugin = plugin; this.supportsModification = supportsModification; boolean usingMySQL = getName().equals("MySQL"); @@ -158,6 +155,7 @@ public abstract class SQLDB extends Database { Set uuids = usersTable.getSavedUUIDs(); uuids.removeAll(usersTable.getContainsBukkitData(uuids)); if (uuids.isEmpty()) { + Log.debug("No conversion necessary."); return; } Log.info("Beginning Bukkit Data -> DB Conversion for " + uuids.size() + " players"); @@ -313,9 +311,9 @@ public abstract class SQLDB extends Database { List sessions = sessionsTable.getSessionData(userId); data.addSessions(sessions); data.setPlayerKills(killsTable.getPlayerKills(userId)); - for (DBCallableProcessor processor : processors) { + processors.stream().forEach((processor) -> { processor.process(data); - } + }); Benchmark.stop("DB Give userdata to processors"); } @@ -333,12 +331,9 @@ public abstract class SQLDB extends Database { Benchmark.start("DB get UserData for " + uuidsCol.size()); Map userIds = usersTable.getAllUserIds(); - Set remove = new HashSet<>(); - for (UUID uuid : uuidsCol) { - if (!userIds.containsKey(uuid)) { - remove.add(uuid); - } - } + Set remove = uuidsCol.stream() + .filter((uuid) -> (!userIds.containsKey(uuid))) + .collect(Collectors.toSet()); List uuids = new ArrayList<>(uuidsCol); Log.debug("Data not found for: " + remove.size()); uuids.removeAll(remove); @@ -426,23 +421,20 @@ public abstract class SQLDB extends Database { gmTimesTable.saveGMTimes(id, gmTimes.get(id)); } Benchmark.stop("Save GMTimes"); - for (Integer id : locations.keySet()) { - UUID uuid = uuids.get(id); - if (uuid != null) { - UserData uData = userDatas.get(uuid); - if (uData != null) { + userDatas.values().stream() + .filter(u -> u != null) + .filter(uData -> uData.isAccessed()) + .forEach(uData -> { uData.stopAccessing(); - } - } - } + }); // Save leftovers - for (UserData userData : saveLast) { + saveLast.stream().forEach((userData) -> { try { saveUserData(userData); } catch (SQLException | NullPointerException e) { exceptions.add(e); } - } + }); if (!exceptions.isEmpty()) { Log.error("SEVERE: MULTIPLE ERRORS OCCURRED: " + exceptions.size()); Log.toLog(this.getClass().getName(), exceptions); @@ -478,74 +470,6 @@ public abstract class SQLDB extends Database { data.stopAccessing(); } - /** - * - * @param userId - * @param locations - * @throws SQLException - */ - @Deprecated - public void saveAdditionalLocationsList(int userId, List locations) throws SQLException { - locationsTable.saveAdditionalLocationsList(userId, locations); - } - - /** - * - * @param userId - * @param names - * @param lastNick - * @throws SQLException - */ - @Deprecated - public void saveNickList(int userId, Set names, String lastNick) throws SQLException { - nicknamesTable.saveNickList(userId, names, lastNick); - } - - /** - * - * @param userId - * @param sessions - * @throws SQLException - * @deprecated Use sessionsTable instead. - */ - @Deprecated - public void saveSessionList(int userId, List sessions) throws SQLException { - sessionsTable.saveSessionData(userId, sessions); - } - - /** - * - * @param userId - * @param kills - * @throws SQLException - */ - @Deprecated - public void savePlayerKills(int userId, List kills) throws SQLException { - killsTable.savePlayerKills(userId, kills); - } - - /** - * - * @param userId - * @param ips - * @throws SQLException - */ - @Deprecated - public void saveIPList(int userId, Set ips) throws SQLException { - ipsTable.saveIPList(userId, ips); - } - - /** - * - * @param userId - * @param gamemodeTimes - * @throws SQLException - */ - @Deprecated - public void saveGMTimes(int userId, Map gamemodeTimes) throws SQLException { - gmTimesTable.saveGMTimes(userId, gamemodeTimes); - } - /** * */ diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java index ff5bd79db..10f7beb2b 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java @@ -4,7 +4,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -21,10 +20,7 @@ import main.java.com.djrapitops.plan.data.UserData; import main.java.com.djrapitops.plan.database.databases.SQLDB; import main.java.com.djrapitops.plan.utilities.Benchmark; import main.java.com.djrapitops.plan.utilities.UUIDFetcher; -import static org.bukkit.Bukkit.getOfflinePlayer; -import static org.bukkit.Bukkit.getOfflinePlayers; import org.bukkit.GameMode; -import org.bukkit.OfflinePlayer; import static org.bukkit.Bukkit.getOfflinePlayer; /** diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java index 71d8190a0..9694ba467 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java @@ -1,5 +1,6 @@ package main.java.com.djrapitops.plan.ui.graphs; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -10,9 +11,11 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import main.java.com.djrapitops.plan.Settings; import main.java.com.djrapitops.plan.data.SessionData; import main.java.com.djrapitops.plan.utilities.Benchmark; import main.java.com.djrapitops.plan.utilities.FormatUtils; +import main.java.com.djrapitops.plan.utilities.analysis.MathUtils; /** * @@ -65,13 +68,37 @@ public class PlayerActivityGraphCreator { playersOnline.add(lastPValue); } } + if (Settings.ANALYSIS_REMOVE_OUTLIERS.isTrue()) { + long average = MathUtils.averageLong(playersOnline.stream()); + double standardDiviation = getStandardDiviation(playersOnline, average); + if (standardDiviation > 3) { + for (int i = 0; i < playersOnline.size(); i++) { + long value = playersOnline.get(i); + if (value - average > 3 * standardDiviation) { + playersOnline.set(i, (long) maxPlayers + 10); + } + } + } + } Benchmark.stop("Player Activity Graph Amount Calculation"); playersOnline.add(0L); + playersOnline.add(0L); + playersOnline.add(0L); + playersOnline.add(0L); playersOnline.add((long) maxPlayers); Benchmark.stop("Generate Player Activity Graph " + sessionData.size() + " " + scale + " |"); return new String[]{playersOnline.toString(), labels.toString()}; } + private static double getStandardDiviation(List players, long avg) { + List valueMinusAvg = players.stream() + .map(p -> Math.pow(Math.abs(p - avg), 2)) + .collect(Collectors.toList()); + int size = valueMinusAvg.size(); + double sum = MathUtils.sumDouble(valueMinusAvg.stream().map(p -> (Serializable) p)); + return Math.sqrt(sum / size); + } + private static Map transformIntoChangeMap(List sessionStarts, List sessionEnds) { Benchmark.start("Player Activity Graph Calc. Change"); Map starts = sessionStarts.stream().distinct().collect(Collectors.toMap(Function.identity(), start -> Collections.frequency(sessionStarts, start))); diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java index 1ff17c9cc..1ffe16f1e 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java @@ -6,12 +6,15 @@ package main.java.com.djrapitops.plan.ui.graphs; import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.Settings; import main.java.com.djrapitops.plan.data.SessionData; +import main.java.com.djrapitops.plan.utilities.MiscUtils; +import main.java.com.djrapitops.plan.utilities.analysis.AnalysisUtils; +import main.java.com.djrapitops.plan.utilities.analysis.MathUtils; /** * @@ -27,7 +30,7 @@ public class PunchCardGraphCreator { public static String generateDataArray(Collection data) { // Initialize dataset List sessionStarts = getSessionStarts(data); - List daysAndHours = getDaysAndHours(sessionStarts); + List daysAndHours = AnalysisUtils.getDaysAndHours(sessionStarts); int[][] dataArray = createDataArray(daysAndHours); int big = findBiggestValue(dataArray); int[][] scaled = scale(dataArray, big); @@ -65,36 +68,63 @@ public class PunchCardGraphCreator { int h = dAndH[1]; dataArray[d][h] = dataArray[d][h] + 1; } + for (int i = 0; i < 7; i++) { + Log.debug(" " + Arrays.toString(dataArray[i])); + } + if (Settings.ANALYSIS_REMOVE_OUTLIERS.isTrue()) { + int avg = findAverage(dataArray); + double standardDiviation = getStandardDiviation(dataArray, avg); + Log.debug("Diviation: " + standardDiviation); + for (int i = 0; i < 7; i++) { + for (int j = 0; j < 24; j++) { + int value = dataArray[i][j]; + if (value - avg > 3 * standardDiviation) { + dataArray[i][j] = (int) (avg); + } + } + } + for (int i = 0; i < 7; i++) { + Log.debug(" " + Arrays.toString(dataArray[i])); + } + } return dataArray; } - private static List getDaysAndHours(List sessionStarts) { - List daysAndHours = sessionStarts.stream().map(start -> { - Calendar day = Calendar.getInstance(); - day.setTimeInMillis(start); - int hourOfDay = day.get(Calendar.HOUR_OF_DAY); + private static double getStandardDiviation(int[][] array, int avg) { + int[][] valueMinusAvg = new int[7][24]; + for (int i = 0; i < 7; i++) { + for (int j = 0; j < 24; j++) { + valueMinusAvg[i][j] = (int) Math.pow(Math.abs(array[i][j] - avg), 2); + } + } + int size = array.length * array[0].length; + int sum = sum(valueMinusAvg); + return Math.sqrt(sum / size); + } - int dayOfWeek = day.get(Calendar.DAY_OF_WEEK) - 2; - if (hourOfDay == 24) { - hourOfDay = 0; - dayOfWeek += 1; + private static int findAverage(int[][] array) { + int total = sum(array); + int size = array.length * array[0].length; + return (int) MathUtils.average(total, size); + } + + private static int sum(int[][] array) { + int total = 0; + for (int[] is : array) { + for (int i : is) { + total += i; } - if (dayOfWeek > 6) { - dayOfWeek = 0; - } - if (dayOfWeek < 0) { - dayOfWeek = 6; - } - return new int[]{dayOfWeek, hourOfDay}; - }).collect(Collectors.toList()); - return daysAndHours; + } + return total; } private static List getSessionStarts(Collection data) { + long now = MiscUtils.getTime(); List sessionStarts = data.stream() .filter(s -> s != null) .filter(s -> s.isValid()) .map(s -> s.getSessionStart()) + .filter(start -> now - start < (long) 2592000 * (long) 1000) .sorted() .collect(Collectors.toList()); return sessionStarts; @@ -135,7 +165,10 @@ public class PunchCardGraphCreator { scaled[i][j] = value; } } - + Log.debug("Biggest value: " + big); + for (int i = 0; i < 7; i++) { + Log.debug(" " + Arrays.toString(scaled[i])); + } return scaled; } } diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/webserver/Response.java b/Plan/src/main/java/com/djrapitops/plan/ui/webserver/Response.java index aeb6c6943..eafb5c0dc 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/webserver/Response.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/webserver/Response.java @@ -69,11 +69,11 @@ public class Response { String playerName = requestArgs[3].trim(); UUID uuid = UUIDFetcher.getUUIDOf(playerName); if (uuid == null) { - String errorMessage = "HTTP/1.1 404 UUID not Found\r\n" + String errorMessage = "HTTP/1.1 500 UUID not Found\r\n" + "Content-Type: text/html;\r\n" + "Content-Length: 30\r\n" + "\r\n" - + "

404 - Player doesn't exist

"; + + "

500 - Player has no UUID.

"; output.write(errorMessage.getBytes()); return; } @@ -87,11 +87,11 @@ public class Response { output.write((htmlDef + dataHtml).getBytes()); } catch (NullPointerException e) { Log.toLog(this.getClass().getName(), e); - String errorMessage = "HTTP/1.1 404 Error\r\n" + String errorMessage = "HTTP/1.1 500 Error\r\n" + "Content-Type: text/html;\r\n" + "Content-Length: 30\r\n" + "\r\n" - + "

404 - Error has occurred..

"; + + "

500 - Error has occurred..

"; output.write(errorMessage.getBytes()); } return; diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/PlaceholderUtils.java b/Plan/src/main/java/com/djrapitops/plan/utilities/PlaceholderUtils.java index d1727c680..ffb9009dc 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/PlaceholderUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/PlaceholderUtils.java @@ -65,6 +65,13 @@ public class PlaceholderUtils { replaceMap.put("%version%", plugin.getDescription().getVersion()); replaceMap.put("%planlite%", ""); replaceMap.put("%sortabletable%", data.getSortablePlayersTable()); + replaceMap.put("%uniquejoinsday%", data.getUniqueJoinsDay()+""); + replaceMap.put("%uniquejoinsweek%", data.getUniqueJoinsWeek()+""); + replaceMap.put("%uniquejoinsmonth%", data.getUniqueJoinsMonth()+""); + replaceMap.put("%avguniquejoins%", data.getAvgUniqJoins()+""); + replaceMap.put("%avguniquejoinsday%", data.getAvgUniqJoinsDay()+""); + replaceMap.put("%avguniquejoinsweek%", data.getAvgUniqJoinsWeek()+""); + replaceMap.put("%avguniquejoinsmonth%", data.getAvgUniqJoinsMonth()+""); replaceMap.put("%dataday%", data.getPlayersDataArray()[0]); replaceMap.put("%labelsday%", data.getPlayersDataArray()[1]); replaceMap.put("%dataweek%", data.getPlayersDataArray()[2]); @@ -102,14 +109,6 @@ public class PlaceholderUtils { replaceMap.put("%gmlabels%", "[\"Survival\", \"Creative\", \"Adventure\", \"Spectator\"]"); replaceMap.put("%gmcolors%", "\"#" + Settings.HCOLOR_GMP_0 + "\",\"#" + Settings.HCOLOR_GMP_1 + "\",\"#" + Settings.HCOLOR_GMP_2 + "\",\"#" + Settings.HCOLOR_GMP_3 + "\""); - replaceMap.put("%genderdata%", Arrays.toString(data.getGenderData())); - replaceMap.put("%gendermale%", data.getGenderData()[0] + ""); - replaceMap.put("%genderfemale%", data.getGenderData()[1] + ""); - replaceMap.put("%genderlabels%", "[\"Male\", \"Female\", \"Unknown\"]"); - replaceMap.put("%gendercolors%", "\"#" + Settings.HCOLOR_GENP_M + "\",\"#" + Settings.HCOLOR_GENP_F - + "\",\"#" + Settings.HCOLOR_GENP_U + "\""); - replaceMap.put("%genderfcolor%", "#" + Settings.HCOLOR_GENP_F); - replaceMap.put("%gendermcolor%", "#" + Settings.HCOLOR_GENP_M); replaceMap.put("%sessionaverage%", FormatUtils.formatTimeAmount(data.getSessionAverage())); replaceMap.put("%geomapcountries%", data.getGeomapCountries()); replaceMap.put("%geomapz%", data.getGeomapZ()); diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java index c0b71cbde..8ebf0755b 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java @@ -128,7 +128,7 @@ public class Analysis { // Analyze & Save RawAnalysisData to AnalysisData createCloroplethMap(analysisData, sorted.getGeolocations(), sorted.getGeocodes()); - createPlayerActivityGraphs(analysisData, sorted.getSessiondata(), sorted.getRegistered()); + createPlayerActivityGraphs(analysisData, sorted.getSessiondata(), sorted.getRegistered(), sorted.getSortedSessionData()); analysisData.setRecentPlayers(RecentPlayersButtonsCreator.createRecentLoginsButtons(sorted.getLatestLogins(), 20)); long totalPlaytime = sorted.getTotalPlaytime(); analysisData.setTotalPlayTime(totalPlaytime); @@ -144,18 +144,18 @@ public class Analysis { analysisData.setTotalkills(sorted.getTotalKills()); analysisData.setTotalmobkills(sorted.getTotalMobKills()); analysisData.setRefreshDate(now); - analysisData.setGenderData(sorted.getGenders()); analysisData.setPunchCardData(PunchCardGraphCreator.generateDataArray(sorted.getSessiondata())); analysisData.setSessionDistributionData(SessionLengthDistributionGraphCreator.generateDataArraySessions(sorted.getSessiondata())); analysisData.setPlaytimeDistributionData(SessionLengthDistributionGraphCreator.generateDataArray(sorted.getPlaytimes().values())); - + analysisData.setAdditionalDataReplaceMap(analyzeAdditionalPluginData(uuids)); - + analysisCache.cache(analysisData); Benchmark.stop("Analysis"); if (Settings.ANALYSIS_LOG_FINISHED.isTrue()) { Log.info(Phrase.ANALYSIS_COMPLETE + ""); } +// LocationAnalysis.performAnalysis(analysisData, plugin.getDB()); if (Settings.ANALYSIS_EXPORT.isTrue()) { ExportUtility.export(plugin, analysisData, rawData); } @@ -227,25 +227,11 @@ public class Analysis { sorted.addTotalDeaths(uData.getDeaths()); List sessions = uData.getSessions(); if (!sessions.isEmpty()) { - sorted.getSessiondata().addAll(sessions); + sorted.addSessions(uData.getUuid(), sessions); } sorted.getRegistered().add(uData.getRegistered()); sorted.addGeoloc(demData.getGeoLocation()); uData.stopAccessing(); - Gender gender = demData.getGender(); - if (null != gender) { - switch (gender) { - case MALE: - sorted.addToGender(0, 1); - break; - case FEMALE: - sorted.addToGender(1, 1); - break; - default: - sorted.addToGender(2, 1); - break; - } - } }); Benchmark.stop("Analysis Fill Dataset"); return sorted; @@ -300,7 +286,7 @@ public class Analysis { Benchmark.stop("Analysis GMVisualization"); } - private void createPlayerActivityGraphs(AnalysisData data, List sData, List registered) { + private void createPlayerActivityGraphs(AnalysisData data, List sData, List registered, Map> sortedSData) { long now = new Date().toInstant().getEpochSecond() * (long) 1000; long scaleDay = 86400 * 1000; @@ -312,6 +298,19 @@ public class Analysis { data.setNewPlayersWeek(AnalysisUtils.getNewPlayers(registered, scaleWeek, now)); data.setNewPlayersMonth(AnalysisUtils.getNewPlayers(registered, scaleMonth, now)); + Benchmark.start("Analysis Unique/day"); + data.setAvgUniqJoins(AnalysisUtils.getUniqueJoinsPerDay(sortedSData, -1)); + data.setAvgUniqJoinsDay(AnalysisUtils.getUniqueJoinsPerDay(sortedSData, scaleDay)); + data.setAvgUniqJoinsWeek(AnalysisUtils.getUniqueJoinsPerDay(sortedSData, scaleWeek)); + data.setAvgUniqJoinsMonth(AnalysisUtils.getUniqueJoinsPerDay(sortedSData, scaleMonth)); + Benchmark.stop("Analysis Unique/day"); + + Benchmark.start("Analysis Unique"); + data.setUniqueJoinsDay(AnalysisUtils.getUniqueJoins(sortedSData, scaleDay)); + data.setUniqueJoinsWeek(AnalysisUtils.getUniqueJoins(sortedSData, scaleWeek)); + data.setUniqueJoinsMonth(AnalysisUtils.getUniqueJoins(sortedSData, scaleMonth)); + Benchmark.stop("Analysis Unique"); + List sessions = sData.stream() .filter(session -> (session != null)) .filter(session -> session.isValid()) diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java index 6bb60af0b..b2dbb4bb4 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java @@ -1,8 +1,13 @@ package main.java.com.djrapitops.plan.utilities.analysis; import java.io.Serializable; +import java.util.Calendar; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -11,7 +16,6 @@ import main.java.com.djrapitops.plan.Settings; import main.java.com.djrapitops.plan.data.SessionData; import main.java.com.djrapitops.plan.data.additional.AnalysisType; import main.java.com.djrapitops.plan.data.additional.PluginData; -import main.java.com.djrapitops.plan.utilities.Benchmark; import main.java.com.djrapitops.plan.utilities.FormatUtils; import main.java.com.djrapitops.plan.utilities.MiscUtils; @@ -65,7 +69,6 @@ public class AnalysisUtils { * @return */ public static int getNewPlayers(List registered, long scale, long now) { - Benchmark.start("Get new players for "+registered.size()+" "+scale+" | "); int newPlayers = 0; if (!registered.isEmpty()) { newPlayers = registered.stream() @@ -74,7 +77,6 @@ public class AnalysisUtils { .map((_item) -> 1).reduce(newPlayers, Integer::sum); } // Filters out register dates before scale - Benchmark.stop("Get new players for "+registered.size()+" "+scale+" | "); return newPlayers; } @@ -223,4 +225,74 @@ public class AnalysisUtils { Log.toLog("com.djrapitops.plan.utilities.AnalysisUtils", e); return source.parseContainer("", "Exception during calculation."); } + + public static Integer getUniqueJoins(Map> sessions, long scale) { + long now = MiscUtils.getTime(); + long nowMinusScale = now - scale; + Set uniqueJoins = new HashSet<>(); + sessions.keySet().stream().forEach((uuid) -> { + List s = sessions.get(uuid); + for (SessionData session : s) { + if (session.getSessionStart() < nowMinusScale) { + continue; + } + uniqueJoins.add(uuid); + } + }); + return uniqueJoins.size(); + } + + public static Integer getUniqueJoinsPerDay(Map> sessions, long scale) { + Map> uniqueJoins = new HashMap<>(); + long now = MiscUtils.getTime(); + long nowMinusScale = now - scale; + sessions.keySet().stream().forEach((uuid) -> { + List s = sessions.get(uuid); + for (SessionData session : s) { + if (scale != -1) { + if (session.getSessionStart() < nowMinusScale) { + continue; + } + } + int day = getDayOfYear(session); + if (!uniqueJoins.containsKey(day)) { + uniqueJoins.put(day, new HashSet<>()); + } + uniqueJoins.get(day).add(uuid); + } + }); + int total = MathUtils.sumInt(uniqueJoins.values().stream().map(s -> s.size())); + int size = uniqueJoins.keySet().size(); + if (size == 0) { + return 0; + } + return total / size; + } + + public static List getDaysAndHours(List sessionStarts) { + List daysAndHours = sessionStarts.stream().map((Long start) -> { + Calendar day = Calendar.getInstance(); + day.setTimeInMillis(start); + int hourOfDay = day.get(Calendar.HOUR_OF_DAY); + int dayOfWeek = day.get(Calendar.DAY_OF_WEEK) - 2; + if (hourOfDay == 24) { + hourOfDay = 0; + dayOfWeek += 1; + } + if (dayOfWeek > 6) { + dayOfWeek = 0; + } + if (dayOfWeek < 0) { + dayOfWeek = 6; + } + return new int[]{dayOfWeek, hourOfDay}; + }).collect(Collectors.toList()); + return daysAndHours; + } + + private static int getDayOfYear(SessionData session) { + Calendar day = Calendar.getInstance(); + day.setTimeInMillis(session.getSessionStart()); + return day.get(Calendar.DAY_OF_YEAR); + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/LocationAnalysis.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/LocationAnalysis.java new file mode 100644 index 000000000..4bc18bfb3 --- /dev/null +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/LocationAnalysis.java @@ -0,0 +1,96 @@ +package main.java.com.djrapitops.plan.utilities.analysis.locations; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.data.AnalysisData; +import main.java.com.djrapitops.plan.database.Database; +import main.java.com.djrapitops.plan.utilities.Benchmark; +import main.java.com.djrapitops.plan.utilities.analysis.MathUtils; +import static org.bukkit.Bukkit.getWorlds; +import org.bukkit.Location; +import org.bukkit.World; + +/** + * + * @author Rsl1122 + * @since 3.4.0 + */ +public class LocationAnalysis { + + public static void performAnalysis(AnalysisData data, Database db) { + Benchmark.start("Location Analysis"); + try { + Map> playerLocations = db.getLocationsTable().getAllLocations(getWorlds().stream().collect(Collectors.toMap(w -> w.getName(), Function.identity()))); + List locations = new ArrayList<>(); + for (Integer id : playerLocations.keySet()) { + locations.addAll(playerLocations.get(id)); + } + Map> worldPoints = getWorldPoints(locations); + for (String world : worldPoints.keySet()) { + Map worldLocs = worldPoints.get(world); + Set frequentPoints = getFrequentPoints(worldLocs); + Log.debug(frequentPoints.toString()); + } + } catch (Exception ex) { + Log.toLog("LocationAnalysis.performAnalysis", ex); + } + Benchmark.stop("Location Analysis"); + } + + public static Map cluster(Collection freqPoints, Collection allPoints) { + Benchmark.start("LocAnalysis cluster"); + allPoints.removeAll(freqPoints); + for (Point point : freqPoints) { + Set cluster = allPoints.stream().filter(p -> distance(point, p) < 5).collect(Collectors.toSet()); + } + Benchmark.stop("LocAnalysis cluster"); + return new HashMap<>(); + } + + public static Set getFrequentPoints(Map points) { + Benchmark.start("LocAnalysis getFrequentPoints"); + if (points.isEmpty()) { + return new HashSet<>(); + } + double averageFreq = MathUtils.averageInt(points.values().stream()); + Set freqPoints = points.entrySet().stream().filter(e -> e.getValue() > averageFreq).map(e -> e.getKey()).collect(Collectors.toSet()); + Benchmark.stop("LocAnalysis getFrequentPoints"); + return freqPoints; + } + + public static Map> getWorldPoints(Collection locations) { + Benchmark.start("LocAnalysis getWorldPoints"); + Map> pointMap = new HashMap<>(); + for (Location location : locations) { + World world = location.getWorld(); + if (world == null) { + continue; + } + String worldName = world.getName(); + if (!pointMap.containsKey(worldName)) { + pointMap.put(worldName, new HashMap<>()); + } + Map numOfLocs = pointMap.get(worldName); + Point point = new Point(location.getBlockX(), location.getBlockZ()); + if (!numOfLocs.containsKey(point)) { + numOfLocs.put(point, 0); + } + numOfLocs.replace(point, numOfLocs.get(point) + 1); + } + Benchmark.stop("LocAnalysis getWorldPoints"); + return pointMap; + } + + public static double distance(Point one, Point two) { + return Math.hypot(one.getX() - two.getX(), one.getY() - one.getY()); + } + +} diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/Point.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/Point.java new file mode 100644 index 000000000..410a68f34 --- /dev/null +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/locations/Point.java @@ -0,0 +1,64 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package main.java.com.djrapitops.plan.utilities.analysis.locations; + +/** + * + * @author Risto + */ +public class Point { + + final private int x; + final private int y; + + public Point(int x, int y) { + this.x = x; + this.y = y; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + @Override + public int hashCode() { + int hash = 3; + hash = 97 * hash + this.x; + hash = 97 * hash + this.y; + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final Point other = (Point) obj; + if (this.x != other.x) { + return false; + } + if (this.y != other.y) { + return false; + } + return true; + } + + @Override + public String toString() { + return "P[x:" + x + "|y:" + y + ']'; + } + +} diff --git a/Plan/src/main/resources/analysis.html b/Plan/src/main/resources/analysis.html index 0e0a4499d..df40df06d 100644 --- a/Plan/src/main/resources/analysis.html +++ b/Plan/src/main/resources/analysis.html @@ -336,7 +336,8 @@ header p { -
+ +

Unique Players: %uniquejoinsday% | Unique/Day: %avguniquejoinsday%

Recent Logins

@@ -367,7 +368,8 @@ header p {

Total Playtime: %totalplaytime% | Player Average: %avgplaytime%
Average Session Length: %sessionaverage%
- Total Login times: %totallogins%
+ Total Login times: %totallogins%
+ Average Unique Players/Day: %avguniquejoins%
Player kills: %playerkills% | Mob kills: %mobkills% | Deaths: %deaths%

@@ -452,6 +454,7 @@ header p {
+

Unique Players: %uniquejoinsday% | Unique/Day: %avguniquejoinsday%

@@ -473,6 +476,7 @@ header p {
+

Unique Players: %uniquejoinsweek% | Unique/Day: %avguniquejoinsweek%

@@ -494,6 +498,7 @@ header p {
+

Unique Playes: %uniquejoinsmonth% | Unique/Day: %avguniquejoinsmonth%

@@ -585,7 +590,7 @@ header p {
-
PunchCard
+
PunchCard - Last 30d
@@ -730,7 +735,7 @@ header p {
-
+
@@ -753,42 +758,6 @@ header p {
-
-
-
-
-
Gender Distribution
-
-
-
- -
-
-
- %genderfemale% -
-
- Female -
-
-
-
-
- -
-
-
- %gendermale% -
-
- Male -
-
-
-
- -
-
%plugins% @@ -931,8 +900,7 @@ function countUpTimer() { var ctxweek = document.getElementById("playerChartWeek"); var ctxmonth = document.getElementById("playerChartMonth"); var ctxactivitypie = document.getElementById("activityPie"); - var ctxgmpie = document.getElementById("gmPie"); - var ctxgenderpie = document.getElementById("genderPie"); + var ctxgmpie = document.getElementById("gmPie"); var dataday = { labels: %labelsday%, datasets: [ @@ -1054,29 +1022,7 @@ function countUpTimer() { } } } - }); - var dataGenderPie = { - labels: %genderlabels%, - datasets: [ - { - data: %genderdata%, - backgroundColor: [%gendercolors%], - hoverBackgroundColor: [%gendercolors%] - } - ] - } - var GenderPie = new Chart(ctxgenderpie, { - type: 'doughnut', - data: dataGenderPie, - options: { - legend: { - position: 'right', - labels: { - padding: 7 - } - } - } - }); + }); var playersChartDay = new Chart(ctxday, { type: 'line', data: dataday, diff --git a/Plan/src/main/resources/config.yml b/Plan/src/main/resources/config.yml index 5457614c2..f972ea0e2 100644 --- a/Plan/src/main/resources/config.yml +++ b/Plan/src/main/resources/config.yml @@ -8,6 +8,7 @@ Settings: LogProgressOnConsole: false NotifyWhenFinished: true MinutesPlayedUntilConsidiredActive: 10 + RemoveOutliersFromVisualization: true Export: Enabled: false DestinationFolder: 'Analysis Results' @@ -69,10 +70,6 @@ Customization: Banned: '951800' Inactive: 'A9A9A9' JoinedOnce: '808080' - GenderPie: - Female: ED97E3 - Male: 7CB9D6 - Unknown: A9A9A9 DemographicsTriggers: Trigger: "i'm, am, im, bin" Female: 'female, girl, gurl, woman, gal, mrs, she, miss, feminin, weiblich, mädchen, frau' diff --git a/Plan/src/main/resources/plugin.yml b/Plan/src/main/resources/plugin.yml index 65d17d1ff..a5e0ad5d4 100644 --- a/Plan/src/main/resources/plugin.yml +++ b/Plan/src/main/resources/plugin.yml @@ -1,7 +1,7 @@ name: Plan author: Rsl1122 main: main.java.com.djrapitops.plan.Plan -version: 3.3.0 +version: 3.4.0 softdepend: - OnTime