diff --git a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java index 8548c3136..ec9426305 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java @@ -72,6 +72,7 @@ public class DBUtils { if (wrappedBatches.size() <= j) { wrappedBatches.add(new ArrayList<>()); } + wrappedBatches.get(j).add(new Container<>(object, entry.getKey())); i++; if (i % BATCH_SIZE == 0) { @@ -81,4 +82,25 @@ public class DBUtils { } return wrappedBatches; } + + public static List>> splitIntoBatchesWithID(Map objects) { + List>> wrappedBatches = new ArrayList<>(); + + int i = 0; + int j = 0; + + for (Entry entry : objects.entrySet()) { + T object = entry.getValue(); + if (wrappedBatches.size() <= j) { + wrappedBatches.add(new ArrayList<>()); + } + + wrappedBatches.get(j).add(new Container<>(object, entry.getKey())); + i++; + if (i % BATCH_SIZE == 0) { + j++; + } + } + return wrappedBatches; + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java index 21632a8f6..83fa8a665 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java @@ -35,7 +35,9 @@ public class MySQLDB extends SQLDB { try { Class.forName("com.mysql.jdbc.Driver"); - String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/" + config.getString("mysql.database"); + String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/" + + config.getString("mysql.database") + + "?rewriteBatchedStatements=true"; return DriverManager.getConnection(url, config.getString("mysql.user"), config.getString("mysql.password")); } catch (ClassNotFoundException | SQLException e) { diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java index b78955a41..68ad0a490 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java @@ -424,19 +424,24 @@ public abstract class SQLDB extends Database { */ @Override public void saveMultipleUserData(Collection data) throws SQLException { - Benchmark.start("Database: Save multiple Userdata"); - checkConnection(); - if (data.isEmpty()) { + if (data == null || data.isEmpty()) { return; } + + Benchmark.start("Database: Save multiple Userdata"); + data.removeIf(Objects::isNull); + + checkConnection(); setStatus("Save userdata (multiple) for " + data.size()); usersTable.saveUserDataInformationBatch(data); // Transform to map - Map userDatas = data.stream().collect(Collectors.toMap(UserData::getUuid, Function.identity())); + Map userDatas = data.stream() + .collect(Collectors.toMap(UserData::getUuid, Function.identity())); + // Get UserIDs Map userIds = usersTable.getAllUserIds(); - // Empty dataset + // Create empty data sets Map> nicknames = new HashMap<>(); Map lastNicks = new HashMap<>(); Map> ips = new HashMap<>(); @@ -445,7 +450,8 @@ public abstract class SQLDB extends Database { Map> sessions = new HashMap<>(); Map> gmTimes = new HashMap<>(); Map> worldTimes = new HashMap<>(); - // Put to dataset + + // Put in data set List worldNames = data.stream() .map(UserData::getWorldTimes) .map(WorldTimes::getTimes) @@ -453,6 +459,7 @@ public abstract class SQLDB extends Database { .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); + for (Map.Entry entrySet : userDatas.entrySet()) { UUID uuid = entrySet.getKey(); UserData uData = entrySet.getValue(); @@ -472,6 +479,7 @@ public abstract class SQLDB extends Database { gmTimes.put(id, new HashMap<>(uData.getGmTimes().getTimes())); worldTimes.put(id, new HashMap<>(uData.getWorldTimes().getTimes())); } + // Save nicknamesTable.saveNickLists(nicknames, lastNicks); ipsTable.saveIPList(ips); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java index a9261f919..249625ae5 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java @@ -83,6 +83,7 @@ public class CommandUseTable extends Table { if (data.isEmpty()) { return; } + Benchmark.start("Database: Save Commanduse"); Map newData = new HashMap<>(data); Map saved = getCommandUse(); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java index c75481364..cdec6ff1f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java @@ -2,6 +2,9 @@ package main.java.com.djrapitops.plan.database.tables; import com.djrapitops.plugin.utilities.Verify; import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.data.time.GMTimes; +import main.java.com.djrapitops.plan.database.Container; +import main.java.com.djrapitops.plan.database.DBUtils; import main.java.com.djrapitops.plan.database.databases.SQLDB; import main.java.com.djrapitops.plan.utilities.Benchmark; @@ -93,12 +96,14 @@ public class GMTimesTable extends Table { statement.setInt(1, userId); set = statement.executeQuery(); HashMap times = new HashMap<>(); + while (set.next()) { times.put("SURVIVAL", set.getLong(columnSurvivalTime)); times.put("CREATIVE", set.getLong(columnCreativeTime)); times.put("ADVENTURE", set.getLong(columnAdventureTime)); times.put("SPECTATOR", set.getLong(columnSpectatorTime)); } + return times; } finally { close(set); @@ -113,18 +118,21 @@ public class GMTimesTable extends Table { try { statement = prepareStatement("SELECT * FROM " + tableName); set = statement.executeQuery(); + while (set.next()) { Map gmTimes = new HashMap<>(); int id = set.getInt(columnUserID); if (!userIds.contains(id)) { continue; } + gmTimes.put("SURVIVAL", set.getLong(columnSurvivalTime)); gmTimes.put("CREATIVE", set.getLong(columnCreativeTime)); gmTimes.put("ADVENTURE", set.getLong(columnAdventureTime)); gmTimes.put("SPECTATOR", set.getLong(columnSpectatorTime)); times.put(id, gmTimes); } + return times; } finally { close(set); @@ -141,8 +149,10 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + PreparedStatement statement = null; String[] gms = getGMKeyArray(); + int update; try { statement = prepareStatement( @@ -153,22 +163,17 @@ public class GMTimesTable extends Table { + columnSpectatorTime + "=? " + " WHERE (" + columnUserID + "=?)"); statement.setInt(5, userId); + for (int i = 0; i < gms.length; i++) { - try { - Long time = gamemodeTimes.get(gms[i]); - if (time != null) { - statement.setLong(i + 1, time); - } else { - statement.setLong(i + 1, 0); - } - } catch (NoSuchFieldError e) { - statement.setLong(i + 1, 0); - } + Long time = gamemodeTimes.get(gms[i]); + statement.setLong(i + 1, time != null ? time : 0); } + update = statement.executeUpdate(); } finally { close(statement); } + if (update == 0) { addNewGMTimesRow(userId, gamemodeTimes); } @@ -195,11 +200,52 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + Benchmark.start("Database: Save GMTimes"); - PreparedStatement statement = null; - String[] gms = getGMKeyArray(); Set savedIDs = getSavedIDs(); + + Map gmTimes = new HashMap<>(); + + for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { + int userID = entrySet.getKey(); + + if (!savedIDs.contains(userID)) { + continue; + } + + Map gmTimesMap = entrySet.getValue(); + gmTimes.put(userID, new GMTimes(gmTimesMap)); + } + + List>> batches = DBUtils.splitIntoBatchesWithID(gmTimes); + + batches.parallelStream().forEach(batch -> { + try { + saveGMTimesBatch(batch); + } catch (SQLException e) { + Log.toLog("GMTimesTable.saveGMTimes", e); + } + }); + + gamemodeTimes.keySet().removeAll(savedIDs); + + addNewGMTimesRows(gamemodeTimes); + Benchmark.stop("Database: Save GMTimes"); + } + + private void saveGMTimesBatch(List> batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize); + + String[] gms = getGMKeyArray(); + Set savedIDs = getSavedIDs(); + + PreparedStatement statement = null; try { statement = prepareStatement( "UPDATE " + tableName + " SET " @@ -208,48 +254,72 @@ public class GMTimesTable extends Table { + columnAdventureTime + "=?, " + columnSpectatorTime + "=? " + " WHERE (" + columnUserID + "=?)"); - boolean commitRequired = false; - for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { - Integer id = entrySet.getKey(); + + for (Container data : batch) { + int id = data.getId(); if (!savedIDs.contains(id)) { continue; } statement.setInt(5, id); + for (int i = 0; i < gms.length; i++) { - try { - Map times = entrySet.getValue(); - Long time = times.get(gms[i]); + Map times = data.getObject().getTimes(); + Long time = times.get(gms[i]); - statement.setLong(i + 1, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 1, 0); - } + statement.setLong(i + 1, time != null ? time : 0); } + statement.addBatch(); - commitRequired = true; } - if (commitRequired) { - statement.executeBatch(); - } - - gamemodeTimes.keySet().removeAll(savedIDs); + Log.debug("Executing GM Times batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } - - addNewGMTimesRows(gamemodeTimes); - Benchmark.stop("Database: Save GMTimes"); } private void addNewGMTimesRows(Map> gamemodeTimes) throws SQLException { if (Verify.isEmpty(gamemodeTimes)) { return; } - PreparedStatement statement = null; + + Benchmark.start("Database: Add GMTimes Rows"); + + Map gmTimes = new HashMap<>(); + + for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { + int userID = entrySet.getKey(); + Map gmTimesMap = entrySet.getValue(); + gmTimes.put(userID, new GMTimes(gmTimesMap)); + } + + List>> batches = DBUtils.splitIntoBatchesWithID(gmTimes); + + batches.parallelStream().forEach(batch -> { + try { + addNewGMTimesBatch(batch); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + + Benchmark.stop("Database: Add GMTimes Rows"); + } + + private void addNewGMTimesBatch(List> batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize); + String[] gms = getGMKeyArray(); + + PreparedStatement statement = null; try { statement = prepareStatement( "INSERT INTO " + tableName + " (" @@ -259,28 +329,22 @@ public class GMTimesTable extends Table { + columnAdventureTime + ", " + columnSpectatorTime + ") VALUES (?, ?, ?, ?, ?)"); - boolean commitRequired = false; - for (Map.Entry> entry : gamemodeTimes.entrySet()) { - Integer id = entry.getKey(); - statement.setInt(1, id); + for (Container data : batch) { + statement.setInt(1, data.getId()); + for (int i = 0; i < gms.length; i++) { - try { - Map times = entry.getValue(); - Long time = times.get(gms[i]); + Map times = data.getObject().getTimes(); + Long time = times.get(gms[i]); - statement.setLong(i + 2, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 2, 0); - } + statement.setLong(i + 2, time != null ? time : 0); } + statement.addBatch(); - commitRequired = true; } - if (commitRequired) { - statement.executeBatch(); - } + Log.debug("Executing GM Times batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } @@ -290,6 +354,7 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + PreparedStatement statement = null; String[] gms = getGMKeyArray(); try { @@ -302,14 +367,10 @@ public class GMTimesTable extends Table { + ") VALUES (?, ?, ?, ?, ?)"); statement.setInt(1, userId); - for (int i = 0; i < gms.length; i++) { - try { - Long time = gamemodeTimes.get(gms[i]); - statement.setLong(i + 2, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 2, 0); - } + for (int i = 0; i < gms.length; i++) { + Long time = gamemodeTimes.get(gms[i]); + statement.setLong(i + 2, time != null ? time : 0); } statement.execute(); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java index e07fd29d4..3e3cee558 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java @@ -90,6 +90,7 @@ public class IPsTable extends Table { Log.error("Host not found at getIPAddresses: " + ipAddressName); //Shouldn't ever happen } } + return ips; } finally { close(set); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java index b96052423..2eb135925 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java @@ -107,36 +107,38 @@ public class SessionsTable extends Table { if (sessions == null) { return; } + Benchmark.start("Database: Save Sessions"); sessions.removeAll(getSessionData(userId)); + if (sessions.isEmpty()) { + Benchmark.stop("Database: Save Sessions"); return; } - PreparedStatement statement = null; - try { + PreparedStatement statement = null; + + try { statement = prepareStatement("INSERT INTO " + tableName + " (" + columnUserID + ", " + columnSessionStart + ", " + columnSessionEnd + ") VALUES (?, ?, ?)"); - boolean commitRequired = false; for (SessionData session : sessions) { long end = session.getSessionEnd(); long start = session.getSessionStart(); if (end < start) { continue; } + statement.setInt(1, userId); statement.setLong(2, start); statement.setLong(3, end); statement.addBatch(); - commitRequired = true; - } - if (commitRequired) { - statement.executeBatch(); } + + statement.executeBatch(); } finally { close(statement); Benchmark.stop("Database: Save Sessions"); @@ -152,25 +154,31 @@ public class SessionsTable extends Table { if (ids == null || ids.isEmpty()) { return new HashMap<>(); } + Benchmark.start("Database: Get Sessions multiple"); PreparedStatement statement = null; ResultSet set = null; + try { Map> sessions = new HashMap<>(); statement = prepareStatement("SELECT * FROM " + tableName); set = statement.executeQuery(); + for (Integer id : ids) { sessions.put(id, new ArrayList<>()); } + while (set.next()) { Integer id = set.getInt(columnUserID); if (!ids.contains(id)) { continue; } - sessions.get(id).add(new SessionData(set.getLong(columnSessionStart), set.getLong(columnSessionEnd))); + + long sessionStart = set.getLong(columnSessionStart); + long sessionEnd = set.getLong(columnSessionEnd); + + sessions.get(id).add(new SessionData(sessionStart, sessionEnd)); } - set.close(); - statement.close(); return sessions; } finally { @@ -195,8 +203,8 @@ public class SessionsTable extends Table { for (Map.Entry> entrySet : sessions.entrySet()) { Integer id = entrySet.getKey(); List sessionList = entrySet.getValue(); - List s = saved.get(id); + if (s != null) { sessionList.removeAll(s); } @@ -207,11 +215,17 @@ public class SessionsTable extends Table { saved.put(id, sessionList); } + List>> batches = splitIntoBatches(sessions); - for (List> batch : batches) { - saveSessionBatch(batch); - } + batches.parallelStream().forEach(batch -> { + try { + saveSessionBatch(batch); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + Benchmark.stop("Database: Save Sessions multiple"); } @@ -219,6 +233,10 @@ public class SessionsTable extends Table { if (batch.isEmpty()) { return; } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of sessions... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement("INSERT INTO " + tableName + " (" @@ -227,25 +245,21 @@ public class SessionsTable extends Table { + columnSessionEnd + ") VALUES (?, ?, ?)"); - boolean commitRequired = false; - int i = 0; for (Container data : batch) { SessionData session = data.getObject(); int id = data.getId(); if (!session.isValid()) { continue; } + statement.setInt(1, id); statement.setLong(2, session.getSessionStart()); statement.setLong(3, session.getSessionEnd()); statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing session batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing session batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java index 55921e575..f2b9b86a8 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java @@ -119,12 +119,24 @@ public class TPSTable extends Table { */ public void saveTPSData(List data) throws SQLException { List> batches = DBUtils.splitIntoBatches(data); - for (List batch : batches) { - saveTPSBatch(batch); - } + batches.parallelStream() + .forEach(batch -> { + try { + saveTPSBatch(batch); + } catch (SQLException e) { + Log.toLog("UsersTable.saveUserDataInformationBatch", e); + } + }); } private void saveTPSBatch(List batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of TPS... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement("INSERT INTO " + tableName + " (" @@ -137,8 +149,6 @@ public class TPSTable extends Table { + columnChunksLoaded + ") VALUES (?, ?, ?, ?, ?, ?, ?)"); - boolean commitRequired = false; - int i = 0; for (TPS tps : batch) { statement.setLong(1, tps.getDate()); statement.setDouble(2, tps.getTps()); @@ -148,13 +158,10 @@ public class TPSTable extends Table { statement.setDouble(6, tps.getEntityCount()); statement.setDouble(7, tps.getChunksLoaded()); statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing tps batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing tps batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java index 5d31f72b6..2d20f4db3 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java @@ -680,10 +680,18 @@ public class UsersTable extends Table { try { List newUserdata = updateExistingUserData(data); Benchmark.start("Database: Insert new UserInfo multiple"); + List> batches = DBUtils.splitIntoBatches(newUserdata); - for (List batch : batches) { - insertNewUserData(batch); - } + + batches.parallelStream() + .forEach(batch -> { + try { + insertNewUserData(batch); + } catch (SQLException e) { + Log.toLog("UsersTable.saveUserDataInformationBatch", e); + } + }); + Benchmark.stop("Database: Insert new UserInfo multiple"); } finally { Benchmark.stop("Database: Save UserInfo multiple"); @@ -691,18 +699,27 @@ public class UsersTable extends Table { } private void insertNewUserData(Collection data) throws SQLException { + if (data.isEmpty()) { + return; + } + + int batchSize = data.size(); + Log.debug("Preparing insertion of new users... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement(getInsertStatement()); - boolean commitRequired = false; - int i = 0; + for (UserData uData : data) { UUID uuid = uData.getUuid(); + statement.setString(1, uuid.toString()); statement.setString(2, uData.getGeolocation()); + GMTimes gmTimes = uData.getGmTimes(); statement.setString(3, gmTimes.getState()); statement.setLong(4, gmTimes.getLastStateChange()); + statement.setLong(5, uData.getPlayTime()); statement.setInt(6, uData.getLoginTimes()); statement.setLong(7, uData.getLastPlayed()); @@ -713,17 +730,16 @@ public class UsersTable extends Table { statement.setBoolean(12, uData.isBanned()); statement.setString(13, uData.getName()); statement.setLong(14, uData.getRegistered()); + WorldTimes worldTimes = uData.getWorldTimes(); statement.setString(15, worldTimes.getState()); statement.setLong(16, worldTimes.getLastStateChange()); + statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing session batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing users batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); }