From 4a20d4417a07c812bb4d2827e59ac3c3ac9eb1bd Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Sun, 10 Sep 2017 14:07:23 +0200 Subject: [PATCH] PR for 4.0.0 (Fuzzlemann) (14) (#285) (Finalizes importer) * Finalized Importer * Test method for ManageImportCommand (runs OfflinePlayerImporter all the time, temporary) Fix Importer --- .../main/java/com/djrapitops/plan/Plan.java | 3 + .../commands/manage/ManageImportCommand.java | 23 ++++- .../plan/database/tables/TPSTable.java | 2 +- .../listeners/PlanGamemodeChangeListener.java | 4 +- .../listeners/PlanWorldChangeListener.java | 4 +- .../importing/importers/Importer.java | 90 +++++++++++++++---- .../importers/OfflinePlayerImporter.java | 18 +++- Plan/test/test/java/utils/TestInit.java | 3 - 8 files changed, 118 insertions(+), 29 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/Plan.java b/Plan/src/main/java/com/djrapitops/plan/Plan.java index 0c8bf0a34..9850fb8d7 100644 --- a/Plan/src/main/java/com/djrapitops/plan/Plan.java +++ b/Plan/src/main/java/com/djrapitops/plan/Plan.java @@ -43,6 +43,7 @@ import main.java.com.djrapitops.plan.systems.info.InformationManager; import main.java.com.djrapitops.plan.systems.info.server.ServerInfoManager; import main.java.com.djrapitops.plan.systems.listeners.*; import main.java.com.djrapitops.plan.systems.processing.Processor; +import main.java.com.djrapitops.plan.systems.processing.importing.importers.OfflinePlayerImporter; import main.java.com.djrapitops.plan.systems.queue.ProcessingQueue; import main.java.com.djrapitops.plan.systems.tasks.TPSCountTimer; import main.java.com.djrapitops.plan.systems.webserver.PageCache; @@ -192,6 +193,8 @@ public class Plan extends BukkitPlugin implements IPlan { hookHandler = new HookHandler(this); Benchmark.stop("Enable", "Hook to 3rd party plugins"); + ImporterManager.registerImporter(new OfflinePlayerImporter()); + BStats bStats = new BStats(this); bStats.registerMetrics(); diff --git a/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageImportCommand.java b/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageImportCommand.java index 8534b915a..d3cd8ec3f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageImportCommand.java +++ b/Plan/src/main/java/com/djrapitops/plan/command/commands/manage/ManageImportCommand.java @@ -3,12 +3,17 @@ package main.java.com.djrapitops.plan.command.commands.manage; import com.djrapitops.plugin.command.CommandType; import com.djrapitops.plugin.command.ISender; import com.djrapitops.plugin.command.SubCommand; +import com.djrapitops.plugin.task.AbsRunnable; +import main.java.com.djrapitops.plan.Log; import main.java.com.djrapitops.plan.Permissions; import main.java.com.djrapitops.plan.Plan; import main.java.com.djrapitops.plan.locale.Locale; import main.java.com.djrapitops.plan.locale.Msg; +import main.java.com.djrapitops.plan.systems.info.ImporterManager; import main.java.com.djrapitops.plan.utilities.Check; +import java.sql.SQLException; + /** * This manage subcommand is used to import data from 3rd party plugins. *

@@ -47,7 +52,23 @@ public class ManageImportCommand extends SubCommand { if (!Check.isTrue(args.length >= 1, Locale.get(Msg.CMD_FAIL_REQ_ONE_ARG) + " " + this.getArguments(), sender)) { return true; } - // TODO Rewrite Importer + + runImport("offlineimporter"); return true; } + + private void runImport(String importer) { + plugin.getRunnableFactory().createNew("Import", new AbsRunnable() { + @Override + public void run() { + try { + ImporterManager.getImporter(importer).processImport(); + } catch (SQLException e) { + Log.toLog(this.getClass().getName(), e); + } finally { + this.cancel(); + } + } + }).runTaskAsynchronously(); + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java index 341eaa9e4..dafaed9c6 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java @@ -146,7 +146,7 @@ public class TPSTable extends Table { statement.setInt(1, p); // More than 2 Months ago. long fiveWeeks = TimeAmount.MONTH.ms() * 2L; - statement.setLong(1, MiscUtils.getTime() - fiveWeeks); + statement.setLong(2, MiscUtils.getTime() - fiveWeeks); statement.execute(); commit(statement.getConnection()); diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanGamemodeChangeListener.java b/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanGamemodeChangeListener.java index ac679b6b9..c9eeca62c 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanGamemodeChangeListener.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanGamemodeChangeListener.java @@ -47,8 +47,6 @@ public class PlanGamemodeChangeListener implements Listener { String worldName = p.getWorld().getName(); Optional cachedSession = plugin.getDataCache().getCachedSession(uuid); - cachedSession.ifPresent(session -> { - session.changeState(worldName, gameMode, time); - }); + cachedSession.ifPresent(session -> session.changeState(worldName, gameMode, time)); } } diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanWorldChangeListener.java b/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanWorldChangeListener.java index afc8552a4..cd41aa38f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanWorldChangeListener.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/listeners/PlanWorldChangeListener.java @@ -29,8 +29,6 @@ public class PlanWorldChangeListener implements Listener { long time = MiscUtils.getTime(); Optional cachedSession = plugin.getDataCache().getCachedSession(uuid); - cachedSession.ifPresent(session -> { - session.changeState(worldName, gameMode, time); - }); + cachedSession.ifPresent(session -> session.changeState(worldName, gameMode, time)); } } diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java index cc3af4acc..fd7f8db82 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java @@ -44,13 +44,33 @@ public abstract class Importer { Benchmark.start(benchmarkName); - Benchmark.start(serverBenchmarkName); - processServerData(); - Benchmark.stop(serverBenchmarkName); + ExecutorService service = Executors.newCachedThreadPool(); - Benchmark.start(userDataBenchmarkName); - processUserData(); - Benchmark.stop(userDataBenchmarkName); + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + Benchmark.start(serverBenchmarkName); + processServerData(); + Benchmark.stop(serverBenchmarkName); + } + }.submit(service); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + Benchmark.start(userDataBenchmarkName); + processUserData(); + Benchmark.stop(userDataBenchmarkName); + } + }.submit(service); + + service.shutdown(); + + try { + service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Log.toLog(this.getClass().getName(), e); + } Benchmark.stop(benchmarkName); } @@ -58,6 +78,7 @@ public abstract class Importer { private void processServerData() throws SQLException { String benchmarkName = "Processing Server Data"; String getDataBenchmarkName = "Getting Server Data"; + String insertDataIntoDatabaseBenchmarkName = "Insert Server Data into Database"; Benchmark.start(benchmarkName); Benchmark.start(getDataBenchmarkName); @@ -75,21 +96,47 @@ public abstract class Importer { UUID uuid = plan.getServerInfoManager().getServerUUID(); Database db = plan.getDB(); - db.getTpsTable().insertAllTPS(ImmutableMap.of(uuid, serverImportData.getTpsData())); - db.getCommandUseTable().insertCommandUsage(ImmutableMap.of(uuid, serverImportData.getCommandUsages())); + ExecutorService service = Executors.newCachedThreadPool(); - Benchmark.start(benchmarkName); + Benchmark.start(insertDataIntoDatabaseBenchmarkName); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + db.getTpsTable().insertAllTPS(ImmutableMap.of(uuid, serverImportData.getTpsData())); + } + }.submit(service); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + db.getCommandUseTable().insertCommandUsage(ImmutableMap.of(uuid, serverImportData.getCommandUsages())); + } + }.submit(service); + + service.shutdown(); + + try { + service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Log.toLog(this.getClass().getName(), e); + } + + Benchmark.stop(insertDataIntoDatabaseBenchmarkName); + Benchmark.stop(benchmarkName); } private void processUserData() throws SQLException { String benchmarkName = "Processing User Data"; String getDataBenchmarkName = "Getting User Data"; + String insertDataIntoCollectionsBenchmarkName = "Insert User Data into Collections"; + String insertDataIntoDatabaseBenchmarkName = "Insert User Data into Database"; Benchmark.start(benchmarkName); Benchmark.start(getDataBenchmarkName); List userImportData = getUserImportData(); - Benchmark.stop(getDataBenchmarkName); if (Verify.isEmpty(userImportData)) { @@ -106,10 +153,14 @@ public abstract class Importer { Database db = plan.getDB(); Set existingUUIDs = db.getSavedUUIDs(); + Set existingUserInfoTableUUIDs = db.getUserInfoTable().getSavedUUIDs().get(serverUUID); + + Benchmark.start(insertDataIntoCollectionsBenchmarkName); + Map users = new Hashtable<>(); List userInfo = new Vector<>(); Map> nickNames = new Hashtable<>(); - Map sessions = new Hashtable<>(); + Map> sessions = new Hashtable<>(); Map> ips = new Hashtable<>(); Map timesKicked = new Hashtable<>(); @@ -118,24 +169,31 @@ public abstract class Importer { UserInfo info = toUserInfo(data); if (!existingUUIDs.contains(uuid)) { + users.put(uuid, info); + } + + if (!existingUserInfoTableUUIDs.contains(uuid)) { userInfo.add(info); } - users.put(uuid, info); nickNames.put(uuid, data.getNicknames()); ips.put(uuid, convertIPs(data)); timesKicked.put(uuid, data.getTimesKicked()); - sessions.put(uuid, toSession(data)); + sessions.put(uuid, Collections.singletonList(toSession(data))); }); + Benchmark.stop(insertDataIntoCollectionsBenchmarkName); + ExecutorService service = Executors.newCachedThreadPool(); + Benchmark.start(insertDataIntoDatabaseBenchmarkName); + db.getUsersTable().insertUsers(users); new ImportExecutorHelper() { @Override void execute() throws SQLException { - // TODO db.getSessionsTable().insertSessions(ImmutableMap.of(serverUUID, sessions)); + db.getSessionsTable().insertSessions(ImmutableMap.of(serverUUID, sessions), true); } }.submit(service); @@ -144,7 +202,7 @@ public abstract class Importer { void execute() throws SQLException { db.getUsersTable().updateKicked(timesKicked); } - }.submit(service); + }.submit(service); new ImportExecutorHelper() { @Override @@ -172,9 +230,11 @@ public abstract class Importer { try { service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); Log.toLog(this.getClass().getName(), e); } + Benchmark.stop(insertDataIntoDatabaseBenchmarkName); Benchmark.stop(benchmarkName); } diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/OfflinePlayerImporter.java b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/OfflinePlayerImporter.java index 92029d4a0..5d4c7f764 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/OfflinePlayerImporter.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/OfflinePlayerImporter.java @@ -7,9 +7,11 @@ package main.java.com.djrapitops.plan.systems.processing.importing.importers; import main.java.com.djrapitops.plan.systems.processing.importing.ServerImportData; import main.java.com.djrapitops.plan.systems.processing.importing.UserImportData; import org.bukkit.Bukkit; +import org.bukkit.OfflinePlayer; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.Vector; /** @@ -32,13 +34,23 @@ public class OfflinePlayerImporter extends Importer { public List getUserImportData() { List dataList = new Vector<>(); + Set operators = Bukkit.getOperators(); + Set banned = Bukkit.getBannedPlayers(); + Arrays.stream(Bukkit.getOfflinePlayers()).parallel().forEach(player -> { UserImportData.UserImportDataBuilder builder = UserImportData.builder(); builder.name(player.getName()) .uuid(player.getUniqueId()) - .op(player.isOp()) - .registered(player.getFirstPlayed()) - .banned(player.isBanned()); + .registered(player.getFirstPlayed()); + + if (operators.contains(player)) { + builder.op(); + } + + if (banned.contains(player)) { + builder.banned(); + } + dataList.add(builder.build()); }); diff --git a/Plan/test/test/java/utils/TestInit.java b/Plan/test/test/java/utils/TestInit.java index 99fb64e32..dc676261f 100644 --- a/Plan/test/test/java/utils/TestInit.java +++ b/Plan/test/test/java/utils/TestInit.java @@ -219,9 +219,6 @@ public class TestInit { return configuration; } - /** - * @return - */ public Plan getPlanMock() { return planMock; }