From ee74fc328eb985af6003de48af7c4aff5794d68c Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Fri, 15 Feb 2019 11:02:18 +0200 Subject: [PATCH] Refactored BukkitImporter to use new db things: - Refactored UsersTable#getSavedUUIDs to a query - Removed #getRegisterDates, #updateName, #getAllTimesKicked, #getUUIDsAndNamesByID - not used - Refactored UserInfoTable#getSavedUUIDs to a query - Removed UserInfoTable#getServerUserInfo - not used --- .../importing/importers/BukkitImporter.java | 96 +++++++------------ .../queries/objects/BaseUserQueries.java | 49 +++++++++- .../plan/db/sql/tables/UserInfoTable.java | 60 +----------- .../plan/db/sql/tables/UsersTable.java | 94 +----------------- .../databases/sql/operation/SQLFetchOps.java | 6 +- 5 files changed, 88 insertions(+), 217 deletions(-) diff --git a/Plan/bukkit/src/main/java/com/djrapitops/plan/system/importing/importers/BukkitImporter.java b/Plan/bukkit/src/main/java/com/djrapitops/plan/system/importing/importers/BukkitImporter.java index 721f6dee6..e9d1fd63f 100644 --- a/Plan/bukkit/src/main/java/com/djrapitops/plan/system/importing/importers/BukkitImporter.java +++ b/Plan/bukkit/src/main/java/com/djrapitops/plan/system/importing/importers/BukkitImporter.java @@ -17,25 +17,24 @@ package com.djrapitops.plan.system.importing.importers; import com.djrapitops.plan.Plan; -import com.djrapitops.plan.api.exceptions.database.DBException; -import com.djrapitops.plan.api.exceptions.database.DBOpException; import com.djrapitops.plan.data.container.BaseUser; import com.djrapitops.plan.data.container.GeoInfo; import com.djrapitops.plan.data.container.Session; import com.djrapitops.plan.data.container.UserInfo; import com.djrapitops.plan.data.store.objects.Nickname; import com.djrapitops.plan.data.time.WorldTimes; +import com.djrapitops.plan.db.Database; +import com.djrapitops.plan.db.access.queries.LargeStoreQueries; +import com.djrapitops.plan.db.access.queries.objects.BaseUserQueries; +import com.djrapitops.plan.db.access.transactions.Transaction; import com.djrapitops.plan.system.cache.GeolocationCache; import com.djrapitops.plan.system.database.DBSystem; -import com.djrapitops.plan.system.database.databases.operation.FetchOperations; -import com.djrapitops.plan.system.database.databases.operation.SaveOperations; import com.djrapitops.plan.system.importing.data.BukkitUserImportRefiner; import com.djrapitops.plan.system.importing.data.ServerImportData; import com.djrapitops.plan.system.importing.data.UserImportData; import com.djrapitops.plan.system.info.server.ServerInfo; import com.djrapitops.plan.utilities.SHA256Hash; import com.djrapitops.plugin.utilities.Verify; -import com.google.common.collect.ImmutableMap; import java.security.NoSuchAlgorithmException; import java.util.*; @@ -46,6 +45,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; /** + * Generic importer for user data into Plan on the Bukkit platform. + * * @author Fuzzlemann */ public abstract class BukkitImporter implements Importer { @@ -90,8 +91,8 @@ public abstract class BukkitImporter implements Importer { ExecutorService service = Executors.newCachedThreadPool(); try { - submitTo(service, this::processServerData); - submitTo(service, this::processUserData); + service.submit(this::processServerData); + service.submit(this::processUserData); } finally { shutdownService(service); } @@ -104,15 +105,13 @@ public abstract class BukkitImporter implements Importer { return; } - ExecutorService service = Executors.newCachedThreadPool(); - - try { - SaveOperations save = dbSystem.getDatabase().save(); - submitTo(service, () -> save.insertTPS(ImmutableMap.of(serverUUID.get(), serverImportData.getTpsData()))); - submitTo(service, () -> save.insertCommandUsage(ImmutableMap.of(serverUUID.get(), serverImportData.getCommandUsages()))); - } finally { - shutdownService(service); - } + dbSystem.getDatabase().executeTransaction(new Transaction() { + @Override + protected void performOperations() { + execute(LargeStoreQueries.storeAllTPSData(Collections.singletonMap(serverUUID.get(), serverImportData.getTpsData()))); + execute(LargeStoreQueries.storeAllCommandUsageData(Collections.singletonMap(serverUUID.get(), serverImportData.getCommandUsages()))); + } + }); } private void processUserData() { @@ -125,50 +124,44 @@ public abstract class BukkitImporter implements Importer { BukkitUserImportRefiner userImportRefiner = new BukkitUserImportRefiner(plugin, userImportData); userImportData = userImportRefiner.refineData(); - FetchOperations fetch = dbSystem.getDatabase().fetch(); - Set existingUUIDs = fetch.getSavedUUIDs(); - Set existingUserInfoTableUUIDs = fetch.getSavedUUIDs(serverUUID.get()); + Database db = dbSystem.getDatabase(); - Map users = new HashMap<>(); + Set existingUUIDs = db.query(BaseUserQueries.fetchCommonUserUUIDs()); + Set existingUserInfoTableUUIDs = db.query(BaseUserQueries.fetchServerUserUUIDs(serverUUID.get())); + + Map users = new HashMap<>(); List userInfo = new ArrayList<>(); Map> nickNames = new HashMap<>(); - Map> sessions = new HashMap<>(); + List sessions = new ArrayList<>(); Map> geoInfo = new HashMap<>(); - Map timesKicked = new HashMap<>(); userImportData.parallelStream().forEach(data -> { UUID uuid = data.getUuid(); - UserInfo info = toUserInfo(data); if (!existingUUIDs.contains(uuid)) { - users.put(uuid, info); + users.put(uuid, toBaseUser(data)); } if (!existingUserInfoTableUUIDs.contains(uuid)) { - userInfo.add(info); + userInfo.add(toUserInfo(data)); } nickNames.put(uuid, data.getNicknames()); geoInfo.put(uuid, convertGeoInfo(data)); - timesKicked.put(uuid, data.getTimesKicked()); - sessions.put(uuid, Collections.singletonList(toSession(data))); + sessions.add(toSession(data)); }); - ExecutorService service = Executors.newCachedThreadPool(); - - try { - SaveOperations save = dbSystem.getDatabase().save(); - - // TODO Replace with a transaction - save.insertUsers(users); - submitTo(service, () -> save.insertSessions(ImmutableMap.of(serverUUID.get(), sessions), true)); - submitTo(service, () -> save.kickAmount(timesKicked)); - submitTo(service, () -> save.insertUserInfo(ImmutableMap.of(serverUUID.get(), userInfo))); - submitTo(service, () -> save.insertNicknames(ImmutableMap.of(serverUUID.get(), nickNames))); - submitTo(service, () -> save.insertAllGeoInfo(geoInfo)); - } finally { - shutdownService(service); - } + db.executeTransaction(new Transaction() { + @Override + protected void performOperations() { + execute(LargeStoreQueries.storeAllCommonUserInformation(users.values())); + execute(LargeStoreQueries.storeAllSessionsWithKillAndWorldData(sessions)); + Map> userInformation = Collections.singletonMap(serverUUID.get(), userInfo); + execute(LargeStoreQueries.storePerServerUserInformation(userInformation)); + execute(LargeStoreQueries.storeAllNicknameData(Collections.singletonMap(serverUUID.get(), nickNames))); + execute(LargeStoreQueries.storeAllGeoInformation(geoInfo)); + } + }); } private void shutdownService(ExecutorService service) { @@ -183,11 +176,6 @@ public abstract class BukkitImporter implements Importer { } } - private void submitTo(ExecutorService service, ImportExecutorHelper helper) { - helper.submit(service); - } - - // TODO Refactor parts to use the base user private BaseUser toBaseUser(UserImportData userImportData) { UUID uuid = userImportData.getUuid(); String name = userImportData.getName(); @@ -230,18 +218,4 @@ public abstract class BukkitImporter implements Importer { } }).collect(Collectors.toList()); } - - private interface ImportExecutorHelper { - void execute() throws DBException; - - default void submit(ExecutorService service) { - service.submit(() -> { - try { - execute(); - } catch (DBException e) { - throw new DBOpException("Import Execution failed", e); - } - }); - } - } } diff --git a/Plan/common/src/main/java/com/djrapitops/plan/db/access/queries/objects/BaseUserQueries.java b/Plan/common/src/main/java/com/djrapitops/plan/db/access/queries/objects/BaseUserQueries.java index 7295eb9d1..5d59e4f9d 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/db/access/queries/objects/BaseUserQueries.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/db/access/queries/objects/BaseUserQueries.java @@ -55,18 +55,34 @@ public class BaseUserQueries { public Collection processResults(ResultSet set) throws SQLException { Collection users = new HashSet<>(); while (set.next()) { - UUID uuid = UUID.fromString(set.getString(UsersTable.USER_UUID)); + UUID playerUUID = UUID.fromString(set.getString(UsersTable.USER_UUID)); String name = set.getString(UsersTable.USER_NAME); long registered = set.getLong(UsersTable.REGISTERED); int kicked = set.getInt(UsersTable.TIMES_KICKED); - users.add(new BaseUser(uuid, name, registered, kicked)); + users.add(new BaseUser(playerUUID, name, registered, kicked)); } return users; } }; } + public static Query> fetchCommonUserUUIDs() { + String sql = Select.from(UsersTable.TABLE_NAME, UsersTable.USER_UUID).toString(); + + return new QueryAllStatement>(sql, 20000) { + @Override + public Set processResults(ResultSet set) throws SQLException { + Set playerUUIDs = new HashSet<>(); + while (set.next()) { + UUID playerUUID = UUID.fromString(set.getString(UsersTable.USER_UUID)); + playerUUIDs.add(playerUUID); + } + return playerUUIDs; + } + }; + } + public static Query> fetchServerBaseUsers(UUID serverUUID) { String sql = "SELECT " + UsersTable.TABLE_NAME + "." + UsersTable.USER_UUID + ", " + @@ -87,15 +103,40 @@ public class BaseUserQueries { public List processResults(ResultSet set) throws SQLException { List users = new ArrayList<>(); while (set.next()) { - UUID uuid = UUID.fromString(set.getString(UsersTable.USER_UUID)); + UUID playerUUID = UUID.fromString(set.getString(UsersTable.USER_UUID)); String name = set.getString(UsersTable.USER_NAME); long registered = set.getLong(UsersTable.REGISTERED); int kicked = set.getInt(UsersTable.TIMES_KICKED); - users.add(new BaseUser(uuid, name, registered, kicked)); + users.add(new BaseUser(playerUUID, name, registered, kicked)); } return users; } }; } + + public static Query> fetchServerUserUUIDs(UUID serverUUID) { + String sql = "SELECT " + + UsersTable.TABLE_NAME + "." + UsersTable.USER_UUID + ", " + + " FROM " + UsersTable.TABLE_NAME + + " INNER JOIN " + UserInfoTable.TABLE_NAME + " on " + + UsersTable.TABLE_NAME + "." + UsersTable.USER_UUID + "=" + UserInfoTable.TABLE_NAME + "." + UserInfoTable.USER_UUID + + " WHERE " + UserInfoTable.SERVER_UUID + "=?"; + return new QueryStatement>(sql, 1000) { + @Override + public void prepare(PreparedStatement statement) throws SQLException { + statement.setString(1, serverUUID.toString()); + } + + @Override + public Set processResults(ResultSet set) throws SQLException { + Set playerUUIDs = new HashSet<>(); + while (set.next()) { + UUID playerUUID = UUID.fromString(set.getString(UsersTable.USER_UUID)); + playerUUIDs.add(playerUUID); + } + return playerUUIDs; + } + }; + } } \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UserInfoTable.java b/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UserInfoTable.java index f8729f731..cccff5593 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UserInfoTable.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UserInfoTable.java @@ -16,11 +16,9 @@ */ package com.djrapitops.plan.db.sql.tables; -import com.djrapitops.plan.data.container.UserInfo; import com.djrapitops.plan.db.DBType; import com.djrapitops.plan.db.SQLDB; import com.djrapitops.plan.db.access.ExecStatement; -import com.djrapitops.plan.db.access.QueryStatement; import com.djrapitops.plan.db.patches.UserInfoOptimizationPatch; import com.djrapitops.plan.db.patches.Version10Patch; import com.djrapitops.plan.db.sql.parsing.CreateTableParser; @@ -28,9 +26,8 @@ import com.djrapitops.plan.db.sql.parsing.Sql; import com.djrapitops.plan.db.sql.parsing.Update; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; -import java.util.*; +import java.util.UUID; /** * Table that is in charge of storing server specific player data. @@ -104,59 +101,4 @@ public class UserInfoTable extends Table { } }); } - - public List getServerUserInfo(UUID serverUUID) { - String sql = "SELECT " + - TABLE_NAME + "." + REGISTERED + ", " + - BANNED + ", " + - OP + ", " + - TABLE_NAME + "." + USER_UUID + - " FROM " + TABLE_NAME + - " WHERE " + SERVER_UUID + "=?"; - - return query(new QueryStatement>(sql, 20000) { - @Override - public void prepare(PreparedStatement statement) throws SQLException { - statement.setString(1, serverUUID.toString()); - } - - @Override - public List processResults(ResultSet set) throws SQLException { - List userInfo = new ArrayList<>(); - while (set.next()) { - UUID uuid = UUID.fromString(set.getString(USER_UUID)); - long registered = set.getLong(REGISTERED); - boolean op = set.getBoolean(OP); - boolean banned = set.getBoolean(BANNED); - - UserInfo info = new UserInfo(uuid, serverUUID, registered, op, banned); - if (!userInfo.contains(info)) { - userInfo.add(info); - } - } - return userInfo; - } - }); - } - - public Set getSavedUUIDs(UUID serverUUID) { - String sql = "SELECT " + USER_UUID + " FROM " + TABLE_NAME + " WHERE " + SERVER_UUID + "=?"; - - return query(new QueryStatement>(sql, 50000) { - @Override - public void prepare(PreparedStatement statement) throws SQLException { - statement.setString(1, serverUUID.toString()); - } - - @Override - public Set processResults(ResultSet set) throws SQLException { - Set uuids = new HashSet<>(); - while (set.next()) { - UUID uuid = UUID.fromString(set.getString(USER_UUID)); - uuids.add(uuid); - } - return uuids; - } - }); - } } diff --git a/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UsersTable.java b/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UsersTable.java index 37b4072de..13a381b19 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UsersTable.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/db/sql/tables/UsersTable.java @@ -25,7 +25,10 @@ import com.djrapitops.plan.db.SQLDB; import com.djrapitops.plan.db.access.ExecStatement; import com.djrapitops.plan.db.access.QueryAllStatement; import com.djrapitops.plan.db.access.QueryStatement; -import com.djrapitops.plan.db.sql.parsing.*; +import com.djrapitops.plan.db.sql.parsing.CreateTableParser; +import com.djrapitops.plan.db.sql.parsing.Insert; +import com.djrapitops.plan.db.sql.parsing.Select; +import com.djrapitops.plan.db.sql.parsing.Sql; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -65,25 +68,6 @@ public class UsersTable extends Table { .toString(); } - /** - * @return a {@link Set} of the saved UUIDs. - */ - public Set getSavedUUIDs() { - String sql = Select.from(tableName, USER_UUID).toString(); - - return query(new QueryAllStatement>(sql, 50000) { - @Override - public Set processResults(ResultSet set) throws SQLException { - Set uuids = new HashSet<>(); - while (set.next()) { - UUID uuid = UUID.fromString(set.getString(USER_UUID)); - uuids.add(uuid); - } - return uuids; - } - }); - } - /** * Get UUID of a player. * @@ -112,35 +96,6 @@ public class UsersTable extends Table { }); } - public List getRegisterDates() { - String sql = Select.from(tableName, REGISTERED).toString(); - - return query(new QueryAllStatement>(sql, 50000) { - @Override - public List processResults(ResultSet set) throws SQLException { - List registerDates = new ArrayList<>(); - while (set.next()) { - registerDates.add(set.getLong(REGISTERED)); - } - return registerDates; - } - }); - } - - public void updateName(UUID uuid, String name) { - String sql = Update.values(tableName, USER_NAME) - .where(USER_UUID + "=?") - .toString(); - - execute(new ExecStatement(sql) { - @Override - public void prepare(PreparedStatement statement) throws SQLException { - statement.setString(1, name); - statement.setString(2, uuid.toString()); - } - }); - } - public int getTimesKicked(UUID uuid) { String sql = Select.from(tableName, TIMES_KICKED) .where(USER_UUID + "=?") @@ -229,24 +184,6 @@ public class UsersTable extends Table { }); } - public Map getAllTimesKicked() { - String sql = Select.from(tableName, USER_UUID, TIMES_KICKED).toString(); - - return query(new QueryAllStatement>(sql, 20000) { - @Override - public Map processResults(ResultSet set) throws SQLException { - Map timesKicked = new HashMap<>(); - while (set.next()) { - UUID uuid = UUID.fromString(set.getString(USER_UUID)); - int kickCount = set.getInt(TIMES_KICKED); - - timesKicked.put(uuid, kickCount); - } - return timesKicked; - } - }); - } - public Map getPlayerNames() { String sql = Select.from(tableName, USER_UUID, USER_NAME).toString(); @@ -265,29 +202,6 @@ public class UsersTable extends Table { }); } - /** - * Gets the {@code UUID} and the name of the player mapped to the user ID - * - * @return a {@code Map>} where the key is the user ID - * and the value is an {@code Map.Entry>} of the player's {@code UUID} and name - */ - public Map> getUUIDsAndNamesByID() { - String sql = Select.from(tableName, ID, USER_UUID, USER_NAME).toString(); - return query(new QueryAllStatement>>(sql, 20000) { - @Override - public Map> processResults(ResultSet set) throws SQLException { - Map> uuidsAndNamesByID = new TreeMap<>(); - while (set.next()) { - int id = set.getInt(ID); - UUID uuid = UUID.fromString(set.getString(USER_UUID)); - String name = set.getString(USER_NAME); - uuidsAndNamesByID.put(id, new AbstractMap.SimpleEntry<>(uuid, name)); - } - return uuidsAndNamesByID; - } - }); - } - public DataContainer getUserInformation(UUID uuid) { Key key = new Key<>(DataContainer.class, "plan_users_data"); DataContainer returnValue = new SupplierDataContainer(); diff --git a/Plan/common/src/main/java/com/djrapitops/plan/system/database/databases/sql/operation/SQLFetchOps.java b/Plan/common/src/main/java/com/djrapitops/plan/system/database/databases/sql/operation/SQLFetchOps.java index a9e47db0d..6edf702ad 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/system/database/databases/sql/operation/SQLFetchOps.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/system/database/databases/sql/operation/SQLFetchOps.java @@ -64,12 +64,12 @@ public class SQLFetchOps extends SQLOps implements FetchOperations { @Override public Set getSavedUUIDs() { - return usersTable.getSavedUUIDs(); + return db.query(BaseUserQueries.fetchCommonUserUUIDs()); } @Override - public Set getSavedUUIDs(UUID server) { - return userInfoTable.getSavedUUIDs(server); + public Set getSavedUUIDs(UUID serverUUID) { + return db.query(BaseUserQueries.fetchServerUserUUIDs(serverUUID)); } @Override