diff --git a/.gitignore b/.gitignore index c72e974fc..b4d1bc86c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ /ProjectHelper/target/ /Filetool/nbproject/private/ /Filetool/build/ -/PlanPluginBridge/target/ \ No newline at end of file +/PlanPluginBridge/target/ +/MakroS/nbproject/private/ \ No newline at end of file diff --git a/Plan/src/main/java/com/djrapitops/plan/Phrase.java b/Plan/src/main/java/com/djrapitops/plan/Phrase.java index dec01fb1d..c5caa4c08 100644 --- a/Plan/src/main/java/com/djrapitops/plan/Phrase.java +++ b/Plan/src/main/java/com/djrapitops/plan/Phrase.java @@ -138,6 +138,7 @@ public enum Phrase { CMD_USG_MANAGE_COMBINE("Copy data from one database to another & combine values"), CMD_USG_MANAGE_IMPORT("Import Data from supported plugins to Active Database."), CMD_USG_MANAGE_CLEAR("Clear data from one database"), + CMD_USG_MANAGE_CLEAN("Clear incorrect data from the database"), CMD_USG_MANAGE_REMOVE("Remove players's data from the Active Database."), CMD_USG_MANAGE_STATUS("Check the status of the Active Database."), CMD_USG_MANAGE_HELP("Show managment help."), diff --git a/Plan/src/main/java/com/djrapitops/plan/command/commands/AnalyzeCommand.java b/Plan/src/main/java/com/djrapitops/plan/command/commands/AnalyzeCommand.java index c8d70c513..ae70ed578 100644 --- a/Plan/src/main/java/com/djrapitops/plan/command/commands/AnalyzeCommand.java +++ b/Plan/src/main/java/com/djrapitops/plan/command/commands/AnalyzeCommand.java @@ -61,7 +61,7 @@ public class AnalyzeCommand extends SubCommand { @Override public void run() { timesrun++; - if (analysisCache.isCached()) { + if (analysisCache.isCached() && !analysisCache.isAnalysisBeingRun()) { sendAnalysisMessage(sender); this.cancel(); return; diff --git a/Plan/src/main/java/com/djrapitops/plan/command/commands/ManageCommand.java b/Plan/src/main/java/com/djrapitops/plan/command/commands/ManageCommand.java index 787636334..b5f2eb291 100644 --- a/Plan/src/main/java/com/djrapitops/plan/command/commands/ManageCommand.java +++ b/Plan/src/main/java/com/djrapitops/plan/command/commands/ManageCommand.java @@ -43,6 +43,7 @@ public class ManageCommand extends TreeCommand { commands.add(new ManageStatusCommand(plugin)); commands.add(new ManageImportCommand(plugin)); commands.add(new ManageRemoveCommand(plugin)); +// commands.add(new ManageCleanCommand(plugin)); commands.add(new ManageClearCommand(plugin)); } } diff --git a/Plan/src/main/java/com/djrapitops/plan/command/commands/QuickAnalyzeCommand.java b/Plan/src/main/java/com/djrapitops/plan/command/commands/QuickAnalyzeCommand.java index 734f2e567..0afa7db1e 100644 --- a/Plan/src/main/java/com/djrapitops/plan/command/commands/QuickAnalyzeCommand.java +++ b/Plan/src/main/java/com/djrapitops/plan/command/commands/QuickAnalyzeCommand.java @@ -56,7 +56,7 @@ public class QuickAnalyzeCommand extends SubCommand { @Override public void run() { timesrun++; - if (analysisCache.isCached()) { + if (analysisCache.isCached() && !analysisCache.isAnalysisBeingRun()) { sender.sendMessage(Phrase.CMD_ANALYZE_HEADER + ""); sender.sendMessage(TextUI.getAnalysisMessages()); sender.sendMessage(Phrase.CMD_FOOTER + ""); diff --git a/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageCleanCommand.java b/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageCleanCommand.java new file mode 100644 index 000000000..5431089ed --- /dev/null +++ b/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageCleanCommand.java @@ -0,0 +1,74 @@ +package main.java.com.djrapitops.plan.command.commands.manage; + +import com.djrapitops.javaplugin.command.CommandType; +import com.djrapitops.javaplugin.command.SubCommand; +import java.sql.SQLException; +import java.util.Arrays; +import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.Permissions; +import main.java.com.djrapitops.plan.Phrase; +import main.java.com.djrapitops.plan.Plan; +import main.java.com.djrapitops.plan.database.Database; +import org.bukkit.command.Command; +import org.bukkit.command.CommandSender; +import org.bukkit.scheduler.BukkitRunnable; + +/** + * This manage subcommand is used to clear a database of all data. + * + * @author Rsl1122 + * @since 2.3.0 + */ +public class ManageCleanCommand extends SubCommand { + + private final Plan plugin; + + /** + * Class Constructor. + * + * @param plugin Current instance of Plan + */ + public ManageCleanCommand(Plan plugin) { + super("clean", CommandType.CONSOLE_WITH_ARGUMENTS, Permissions.MANAGE.getPermission(), Phrase.CMD_USG_MANAGE_CLEAN + "", ""); + + this.plugin = plugin; + } + + @Override + public boolean onCommand(CommandSender sender, Command cmd, String commandLabel, String[] args) { + if (args.length == 0) { + sender.sendMessage(Phrase.COMMAND_REQUIRES_ARGUMENTS_ONE + ""); + return true; + } + String dbToClear = args[0].toLowerCase(); + if (!dbToClear.equals("mysql") && !dbToClear.equals("sqlite")) { + sender.sendMessage(Phrase.MANAGE_ERROR_INCORRECT_DB + dbToClear); + return true; + } + + Database clearDB = null; + for (Database database : plugin.getDatabases()) { + if (dbToClear.equalsIgnoreCase(database.getConfigName())) { + clearDB = database; + clearDB.init(); + } + } + if (clearDB == null) { + sender.sendMessage(Phrase.MANAGE_DATABASE_FAILURE + ""); + Log.error(dbToClear + " was null!"); + return true; + } + + final Database clearThisDB = clearDB; + (new BukkitRunnable() { + @Override + public void run() { + sender.sendMessage(Phrase.MANAGE_PROCESS_START.parse()); + clearThisDB.clean(); + sender.sendMessage(Phrase.MANAGE_SUCCESS + ""); + this.cancel(); + } + }).runTaskAsynchronously(plugin); + return true; + } +} diff --git a/Plan/src/main/java/com/djrapitops/plan/data/cache/AnalysisCacheHandler.java b/Plan/src/main/java/com/djrapitops/plan/data/cache/AnalysisCacheHandler.java index 3a4567f11..47b9382fd 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/cache/AnalysisCacheHandler.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/cache/AnalysisCacheHandler.java @@ -15,7 +15,7 @@ public class AnalysisCacheHandler { private final Plan plugin; private AnalysisData cache; - private Analysis analysis; + private final Analysis analysis; /** * Class Constructor. @@ -33,7 +33,6 @@ public class AnalysisCacheHandler { * Runs analysis, cache method is called after analysis is complete. */ public void updateCache() { - cache = null; analysis.runAnalysis(this); } @@ -63,4 +62,8 @@ public class AnalysisCacheHandler { public boolean isCached() { return (cache != null); } + + public boolean isAnalysisBeingRun() { + return analysis.isAnalysisBeingRun(); + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/data/cache/queue/Setup.java b/Plan/src/main/java/com/djrapitops/plan/data/cache/queue/Setup.java index 5421da2e3..abd55508f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/cache/queue/Setup.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/cache/queue/Setup.java @@ -8,7 +8,7 @@ package main.java.com.djrapitops.plan.data.cache.queue; */ public abstract class Setup { - private Consumer[] consumers; + private final Consumer[] consumers; /** * Constructor, defines consumers. diff --git a/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanCommandPreprocessListener.java b/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanCommandPreprocessListener.java index ca45864be..d068e542e 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanCommandPreprocessListener.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanCommandPreprocessListener.java @@ -43,6 +43,6 @@ public class PlanCommandPreprocessListener implements Listener { Log.debug("Ignored command, player had ignore permission."); return; } - handler.handleCommand(event.getMessage().split(" ")[0]); + handler.handleCommand(event.getMessage().split(" ")[0].toLowerCase()); } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java index 4f6cad67c..12a25e8db 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java @@ -18,6 +18,7 @@ import java.util.Map.Entry; public class DBUtils { public static List>> splitIntoBatches(Map> objects) { + int batchSize = 2048; List>> wrappedBatches = new ArrayList<>(); int i = 0; @@ -29,8 +30,8 @@ public class DBUtils { wrappedBatches.add(new ArrayList<>()); } wrappedBatches.get(j).add(new Container<>(object, entry.getKey())); - i++; - if (i % 1024 == 0) { + i++; + if (i % batchSize == 0) { j++; } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java index 58c4bf99e..eb385f00c 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java @@ -154,7 +154,6 @@ public abstract class SQLDB extends Database { @Override public void run() { try { - clean(); Benchmark.start("Convert Bukkitdata to DB data"); Set uuids = usersTable.getSavedUUIDs(); uuids.removeAll(usersTable.getContainsBukkitData(uuids)); @@ -472,41 +471,8 @@ public abstract class SQLDB extends Database { Log.info("Cleaning the database."); try { checkConnection(); - Map> allSessions = sessionsTable.getSessionData(usersTable.getAllUserIds().values()); - Benchmark.start("Combine Sessions"); - int before = MathUtils.sumInt(allSessions.values().stream().map(l -> l.size())); - Log.debug("Sessions before: " + before); - Map beforeM = new HashMap<>(); - Map afterM = new HashMap<>(); - for (Integer id : allSessions.keySet()) { - List sessions = allSessions.get(id); - beforeM.put(id, sessions.size()); - if (sessions.isEmpty()) { - afterM.put(id, 0); - continue; - } - List combined = ManageUtils.combineSessions(sessions); - afterM.put(id, combined.size()); - allSessions.put(id, combined); - } - int after = MathUtils.sumInt(allSessions.values().stream().map(l -> l.size())); - Log.debug("Sessions after: " + after); - if (before - after > 50) { - Benchmark.start("Save combined sessions"); - Iterator iterator = new HashSet<>(allSessions.keySet()).iterator(); - while (iterator.hasNext()) { - int id = iterator.next(); - if (afterM.get(id) < beforeM.get(id)) { - sessionsTable.removeUserSessions(id); - } else { - allSessions.remove(id); - } - } - sessionsTable.saveSessionData(allSessions); - Benchmark.stop("Save combined sessions"); - } - Benchmark.stop("Combine Sessions"); - Log.info("Combined " + (before - after) + " sessions."); + commandUseTable.clean(); + sessionsTable.clean(); Log.info("Clean complete."); } catch (SQLException e) { Log.toLog(this.getClass().getName(), e); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java index 0bf97334b..bb867d73f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java @@ -6,6 +6,7 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.Plan; import main.java.com.djrapitops.plan.database.databases.SQLDB; import main.java.com.djrapitops.plan.utilities.Benchmark; @@ -61,7 +62,13 @@ public class CommandUseTable extends Table { statement = prepareStatement("SELECT * FROM " + tableName); set = statement.executeQuery(); while (set.next()) { - commandUse.put(set.getString(columnCommand), set.getInt(columnTimesUsed)); + String cmd = set.getString(columnCommand); + int amountUsed = set.getInt(columnTimesUsed); + Integer get = commandUse.get(cmd); + if (get != null && get > amountUsed) { + continue; + } + commandUse.put(cmd, amountUsed); } return commandUse; } finally { @@ -150,4 +157,11 @@ public class CommandUseTable extends Table { close(statement); } } + + public void clean() throws SQLException { + Map commandUse = getCommandUse(); + removeAllData(); + saveCommandUse(commandUse); + Plan.getInstance().getHandler().getCommandUseFromDb(); + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java index 918bd52c4..828149ae7 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java @@ -6,6 +6,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import main.java.com.djrapitops.plan.Log; @@ -13,6 +15,8 @@ import main.java.com.djrapitops.plan.data.SessionData; import main.java.com.djrapitops.plan.database.Container; import main.java.com.djrapitops.plan.database.databases.SQLDB; import main.java.com.djrapitops.plan.utilities.Benchmark; +import main.java.com.djrapitops.plan.utilities.ManageUtils; +import main.java.com.djrapitops.plan.utilities.analysis.MathUtils; /** * @@ -251,4 +255,48 @@ public class SessionsTable extends Table { close(statement); } } + + public void clean() throws SQLException { + Map loginTimes = db.getUsersTable().getLoginTimes(); + Map> allSessions = getSessionData(loginTimes.keySet()); + Benchmark.start("Combine Sessions"); + int before = MathUtils.sumInt(allSessions.values().stream().map(l -> l.size())); + Log.debug("Sessions before: " + before); + Map beforeM = new HashMap<>(); + Map afterM = new HashMap<>(); + for (Integer id : allSessions.keySet()) { + List sessions = allSessions.get(id); + beforeM.put(id, sessions.size()); + if (sessions.isEmpty()) { + afterM.put(id, 0); + continue; + } + Integer times = loginTimes.get(id); + if (sessions.size() == times) { + afterM.put(id, times); + continue; + } + List combined = ManageUtils.combineSessions(sessions, times); + afterM.put(id, combined.size()); + allSessions.put(id, combined); + } + int after = MathUtils.sumInt(allSessions.values().stream().map(l -> l.size())); + Log.debug("Sessions after: " + after); + if (before - after > 50) { + Benchmark.start("Save combined sessions"); + Iterator iterator = new HashSet<>(allSessions.keySet()).iterator(); + while (iterator.hasNext()) { + int id = iterator.next(); + if (afterM.get(id) < beforeM.get(id)) { + removeUserSessions(id); + } else { + allSessions.remove(id); + } + } + saveSessionData(allSessions); + Benchmark.stop("Save combined sessions"); + } + Benchmark.stop("Combine Sessions"); + Log.info("Combined " + (before - after) + " sessions."); + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java index 24890a74b..aa3671305 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java @@ -330,7 +330,7 @@ public class UsersTable extends Table { List containsBukkitData = getContainsBukkitData(uuids); List datas = new ArrayList<>(); datas.addAll(getUserDataForKnown(containsBukkitData)); - + uuids.removeAll(containsBukkitData); if (!uuids.isEmpty()) { List noBukkitData = new ArrayList<>(); @@ -343,7 +343,7 @@ public class UsersTable extends Table { addUserInformationToUserData(noBukkitData); datas.addAll(noBukkitData); } - + Benchmark.stop("Get UserData Multiple " + uuids.size()); return datas; } @@ -753,7 +753,7 @@ public class UsersTable extends Table { if (!savedUUIDs.contains(uuid)) { if (!saveLast.contains(uData)) { saveLast.add(uData); - } + } continue; } uData.access(); @@ -848,6 +848,26 @@ public class UsersTable extends Table { } } + public Map getLoginTimes() throws SQLException { + Benchmark.start("Get Logintimes"); + PreparedStatement statement = null; + ResultSet set = null; + try { + Map ids = new HashMap<>(); + statement = prepareStatement("SELECT " + columnID + ", " + columnLoginTimes + " FROM " + tableName); + set = statement.executeQuery(); + while (set.next()) { + Integer id = set.getInt(columnID); + ids.put(id, set.getInt(columnLoginTimes)); + } + return ids; + } finally { + close(set); + close(statement); + Benchmark.stop("Get Logintimes"); + } + } + /** * * @return @@ -874,7 +894,7 @@ public class UsersTable extends Table { close(statement); } } - + public Map getLoginTimes(Collection uuids) { //TODO return new HashMap<>(); diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java index 88fdd1734..f60c0ca36 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PlayerActivityGraphCreator.java @@ -71,7 +71,7 @@ public class PlayerActivityGraphCreator { if (Settings.ANALYSIS_REMOVE_OUTLIERS.isTrue()) { long average = MathUtils.averageLong(playersOnline.stream()); double standardDiviation = getStandardDiviation(playersOnline, average); - if (standardDiviation > 3) { + if (standardDiviation > 3.5) { for (int i = 0; i < playersOnline.size(); i++) { long value = playersOnline.get(i); if (value - average > 3 * standardDiviation) { diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java index dccdf5973..4b700d415 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/graphs/PunchCardGraphCreator.java @@ -67,7 +67,7 @@ public class PunchCardGraphCreator { int avg = findAverage(dataArray); double standardDiviation = getStandardDiviation(dataArray, avg); Log.debug("Diviation: " + standardDiviation); - if (standardDiviation > 3) { + if (standardDiviation > 3.5) { for (int i = 0; i < 7; i++) { for (int j = 0; j < 24; j++) { int value = dataArray[i][j]; diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/ManageUtils.java b/Plan/src/main/java/com/djrapitops/plan/utilities/ManageUtils.java index 20212cb16..743d7f25b 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/ManageUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/ManageUtils.java @@ -83,15 +83,26 @@ public class ManageUtils { } public static boolean containsCombinable(List sessions) { - // Checks if there are starts & ends that are the same, or less than 5000 ms away from each other. + return containsCombinable(sessions, 5000); + } + + private static boolean containsCombinable(List sessions, int threshold) { + // Checks if there are starts & ends that are the same, or less than threshold ms away from each other. return sessions.stream() .anyMatch(s -> sessions.stream() .filter(ses -> !ses.equals(s)) .map(ses -> ses.getSessionStart()) - .anyMatch((Long start) -> (Math.abs(s.getSessionEnd() - start) < 5000))); + .anyMatch((Long start) -> (Math.abs(s.getSessionEnd() - start) < threshold))); } - public static List combineSessions(List sessions) { + public static List combineSessions(List sessions, Integer loginTimes) { + return combineSessions(sessions, loginTimes, 5000); + } + + private static List combineSessions(List sessions, Integer loginTimes, int threshold) { + if (threshold >= 35000) { + return sessions; + } List newSessions = new ArrayList<>(); List removed = new ArrayList<>(); Iterator iterator = sessions.iterator(); @@ -100,7 +111,7 @@ public class ManageUtils { if (removed.contains(session)) { continue; } - List close = sessions.stream().filter(ses -> Math.abs(session.getSessionEnd() - ses.getSessionStart()) < 5000).collect(Collectors.toList()); + List close = sessions.stream().filter(ses -> Math.abs(session.getSessionEnd() - ses.getSessionStart()) < threshold).collect(Collectors.toList()); if (!close.isEmpty()) { long big = MathUtils.getBiggestLong(close.stream().map((SessionData ses) -> ses.getSessionEnd()).collect(Collectors.toList())); session.endSession(big); @@ -108,9 +119,12 @@ public class ManageUtils { } newSessions.add(session); } - boolean containsCombinable = containsCombinable(newSessions); + if (loginTimes == newSessions.size()) { + return newSessions; + } + boolean containsCombinable = containsCombinable(newSessions, threshold); if (containsCombinable) { - return combineSessions(newSessions); + return combineSessions(newSessions, threshold + 1000); } else { return newSessions; } diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java index 5acf44865..9c63bac6f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/Analysis.java @@ -45,6 +45,7 @@ public class Analysis { private final Plan plugin; private final InspectCacheHandler inspectCache; + private int taskId = -1; /** * Class Constructor. @@ -66,13 +67,18 @@ public class Analysis { * @param analysisCache Cache that the data is saved to. */ public void runAnalysis(AnalysisCacheHandler analysisCache) { + if (isAnalysisBeingRun()) { + return; + } Benchmark.start("Analysis"); log(Phrase.ANALYSIS_START + ""); // Async task for Analysis BukkitTask asyncAnalysisTask = (new BukkitRunnable() { @Override public void run() { + taskId = this.getTaskId(); analyze(analysisCache, plugin.getDB()); + taskId = -1; this.cancel(); } }).runTaskAsynchronously(plugin); @@ -414,4 +420,8 @@ public class Analysis { Benchmark.stop("Analysis 3rd party"); return replaceMap; } + + public boolean isAnalysisBeingRun() { + return taskId != -1; + } } diff --git a/Plan/src/main/resources/analysis.html b/Plan/src/main/resources/analysis.html index 1a48540d7..08eb6f426 100644 --- a/Plan/src/main/resources/analysis.html +++ b/Plan/src/main/resources/analysis.html @@ -766,6 +766,9 @@ header p {