From 4acfcdaddb5c03b92d684fea24dda1ef49e5a0e0 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Sun, 3 Sep 2017 02:25:37 +0200 Subject: [PATCH] Halfway completion of Importer (Still needs some Batch Statements) --- .../processing/importing/UserImportData.java | 16 +-- .../importing/UserImportRefiner.java | 16 ++- .../importing/importers/Importer.java | 136 ++++++++++++++++-- 3 files changed, 146 insertions(+), 22 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportData.java b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportData.java index e8ddfd9d5..2ec16a467 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportData.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportData.java @@ -16,7 +16,7 @@ import java.util.*; public class UserImportData { private String name; - private String uuid; + private UUID uuid; private List nicknames; private long registered; @@ -32,7 +32,7 @@ public class UserImportData { private int mobKills; private int deaths; - private UserImportData(String name, String uuid, List nicknames, long registered, boolean op, boolean banned, int timesKicked, List ips, Map worldTimes, List kills, int mobKills, int deaths) { + private UserImportData(String name, UUID uuid, List nicknames, long registered, boolean op, boolean banned, int timesKicked, List ips, Map worldTimes, List kills, int mobKills, int deaths) { this.name = name; this.uuid = uuid; this.nicknames = nicknames; @@ -59,11 +59,11 @@ public class UserImportData { this.name = name; } - public String getUuid() { + public UUID getUuid() { return uuid; } - public void setUuid(String uuid) { + public void setUuid(UUID uuid) { this.uuid = uuid; } @@ -153,7 +153,7 @@ public class UserImportData { private final Map worldTimes = new HashMap<>(); private final List kills = new ArrayList<>(); private String name; - private String uuid; + private UUID uuid; private long registered; private boolean op; private boolean banned; @@ -171,12 +171,12 @@ public class UserImportData { } public UserImportDataBuilder uuid(UUID uuid) { - return uuid(uuid.toString()); + this.uuid = uuid; + return this; } public UserImportDataBuilder uuid(String uuid) { - this.uuid = uuid; - return this; + return uuid(UUID.fromString(uuid)); } public UserImportDataBuilder registered(long registered) { diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportRefiner.java b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportRefiner.java index 8afa969c5..56ef30e38 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportRefiner.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/UserImportRefiner.java @@ -96,17 +96,15 @@ public class UserImportRefiner { importers.parallelStream().forEach(importer -> { String name = importer.getName(); - String uuid = importer.getUuid(); + UUID uuid = importer.getUuid(); boolean nameNull = name == null; boolean uuidNull = uuid == null; if (nameNull && uuidNull) { invalidData.add(importer); - } - - if (nameNull) { - namesMissing.put(importer, uuid); + } else if (nameNull) { + namesMissing.put(importer, uuid.toString()); } else if (uuidNull) { uuidsMissing.put(importer, name); } @@ -133,7 +131,13 @@ public class UserImportRefiner { addMissingUUIDsOverFetcher(); } - foundUUIDs.entrySet().parallelStream().forEach(entry -> entry.getKey().setUuid(entry.getValue())); + foundUUIDs.entrySet().parallelStream() + .forEach(entry -> { + UserImportData userImportData = entry.getKey(); + UUID uuid = UUID.fromString(entry.getValue()); + + userImportData.setUuid(uuid); + }); importers.removeAll(uuidsMissing.keySet()); diff --git a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java index 785aec4b6..847493b3c 100644 --- a/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java +++ b/Plan/src/main/java/com/djrapitops/plan/systems/processing/importing/importers/Importer.java @@ -5,15 +5,26 @@ package main.java.com.djrapitops.plan.systems.processing.importing.importers; import com.djrapitops.plugin.utilities.Verify; +import com.google.common.collect.ImmutableMap; import main.java.com.djrapitops.plan.Log; import main.java.com.djrapitops.plan.Plan; +import main.java.com.djrapitops.plan.data.PlayerKill; +import main.java.com.djrapitops.plan.data.UserInfo; +import main.java.com.djrapitops.plan.data.time.WorldTimes; import main.java.com.djrapitops.plan.database.Database; +import main.java.com.djrapitops.plan.systems.cache.GeolocationCache; import main.java.com.djrapitops.plan.systems.processing.importing.ServerImportData; import main.java.com.djrapitops.plan.systems.processing.importing.UserImportData; import main.java.com.djrapitops.plan.systems.processing.importing.UserImportRefiner; import main.java.com.djrapitops.plan.utilities.Benchmark; -import java.util.List; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RecursiveAction; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * @author Fuzzlemann @@ -27,7 +38,7 @@ public abstract class Importer { public abstract List getUserImportData(); - public final void processImport() { + public final void processImport() throws SQLException { String benchmarkName = "Import processing"; String serverBenchmarkName = "Server Data processing"; String userDataBenchmarkName = "User Data processing"; @@ -45,7 +56,7 @@ public abstract class Importer { Benchmark.stop(benchmarkName); } - private void processServerData() { + private void processServerData() throws SQLException { String benchmarkName = "Processing Server Data"; String getDataBenchmarkName = "Getting Server Data"; @@ -61,14 +72,17 @@ public abstract class Importer { return; } - Database db = Plan.getInstance().getDB(); + Plan plan = Plan.getInstance(); + UUID uuid = plan.getServerInfoManager().getServerUUID(); + Database db = plan.getDB(); - //TODO + db.getTpsTable().insertAllTPS(ImmutableMap.of(uuid, serverImportData.getTpsData())); + db.getCommandUseTable().insertCommandUsage(ImmutableMap.of(uuid, serverImportData.getCommandUsages())); Benchmark.start(benchmarkName); } - private void processUserData() { + private void processUserData() throws SQLException { String benchmarkName = "Processing User Data"; String getDataBenchmarkName = "Getting User Data"; @@ -84,11 +98,117 @@ public abstract class Importer { return; } - UserImportRefiner userImportRefiner = new UserImportRefiner(Plan.getInstance(), userImportData); + Plan plan = Plan.getInstance(); + + UserImportRefiner userImportRefiner = new UserImportRefiner(plan, userImportData); userImportData = userImportRefiner.refineData(); - //TODO + UUID serverUUID = plan.getServerInfoManager().getServerUUID(); + Database db = plan.getDB(); + + Map> nickNames = new Hashtable<>(); + List userInfo = new Vector<>(); + Map worldTimes = new Hashtable<>(); + Map> ips = new Hashtable<>(); + Map> playerKills = new Hashtable<>(); + Map mobKills = new Hashtable<>(); + Map deaths = new Hashtable<>(); + Map timesKicked = new Hashtable<>(); + + userImportData.parallelStream().forEach(data -> { + UUID uuid = data.getUuid(); + + nickNames.put(uuid, data.getNicknames()); + userInfo.add(toUserInfo(data)); + worldTimes.put(uuid, new WorldTimes(data.getWorldTimes())); + ips.put(uuid, convertIPs(data)); + playerKills.put(uuid, data.getKills()); + mobKills.put(uuid, data.getMobKills()); + deaths.put(uuid, data.getDeaths()); + timesKicked.put(uuid, data.getTimesKicked()); + }); + + new RecursiveAction() { + @Override + protected void compute() { + try { + db.getUserInfoTable().insertUserInfo(ImmutableMap.of(serverUUID, userInfo)); + } catch (SQLException e) { + Log.toLog(this.getClass().getName(), e); + } + } + }; + + ExecutorService service = Executors.newCachedThreadPool(); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + db.getUserInfoTable().insertUserInfo(ImmutableMap.of(serverUUID, userInfo)); + } + }.submit(service); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + db.getNicknamesTable().insertNicknames(ImmutableMap.of(serverUUID, nickNames)); + } + }.submit(service); + + new ImportExecutorHelper() { + @Override + void execute() throws SQLException { + db.getIpsTable().insertIPsAndGeolocations(ips); + } + }.submit(service); + + //TODO deaths, mobkills, worldTimes, timesKicked & playerKills insertion + + service.shutdown(); + + try { + service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Log.toLog(this.getClass().getName(), e); + } Benchmark.stop(benchmarkName); } + + private UserInfo toUserInfo(UserImportData userImportData) { + UUID uuid = userImportData.getUuid(); + String name = userImportData.getName(); + long registered = userImportData.getRegistered(); + boolean op = userImportData.isOp(); + boolean banned = userImportData.isBanned(); + + return new UserInfo(uuid, name, registered, op, banned); + } + + private Map convertIPs(UserImportData userImportData) { + Map convertedIPs; + List ips = userImportData.getIps(); + + convertedIPs = ips.parallelStream() + .collect(Collectors.toMap(ip -> ip, GeolocationCache::getCountry, (a, b) -> b, HashMap::new)); + + return convertedIPs; + } + + private abstract class ImportExecutorHelper { + abstract void execute() throws SQLException; + + void submit(ExecutorService service) { + service.submit(new Runnable() { + @Override + public void run() { + try { + execute(); + } catch (SQLException e) { + Log.toLog(this.getClass().getName(), e); + } + } + }); + } + } }