Upstream/Plan
1
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2024-12-28 12:07:35 +01:00
Code Issues Projects Releases Wiki Activity

Adds the flag "rewriteBatchedStatements=true" to the MySQL JDBC connection URL to pack batches into one Query

Browse Source 
Fixes NullPointerException at saveMultipleUserData(Collection<UserData>)
Improves the save... methods
Makes Batch Insertion parallel
Removed NoSuchFieldError where it's not going to happen
Modifies GM Times to update / insert in batches (previous: 520977ms (521s); after: 495115ms (495s)
This commit is contained in:
Fuzzlemann 2017-08-03 03:25:48 +02:00
parent dedb95766a
commit 62e73ffe1c
9 changed files with 237 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java
View File

@ -72,6 +72,7 @@ public class DBUtils {
if (wrappedBatches.size() <= j) {
wrappedBatches.add(new ArrayList<>());
}
wrappedBatches.get(j).add(new Container<>(object, entry.getKey()));
i++;
if (i % BATCH_SIZE == 0) {
@ -81,4 +82,25 @@ public class DBUtils {
}
return wrappedBatches;
}
public static <T> List<List<Container<T>>> splitIntoBatchesWithID(Map<Integer, T> objects) {
List<List<Container<T>>> wrappedBatches = new ArrayList<>();
int i = 0;
int j = 0;
for (Entry<Integer, T> entry : objects.entrySet()) {
T object = entry.getValue();
if (wrappedBatches.size() <= j) {
wrappedBatches.add(new ArrayList<>());
}
wrappedBatches.get(j).add(new Container<>(object, entry.getKey()));
i++;
if (i % BATCH_SIZE == 0) {
j++;
}
}
return wrappedBatches;
}
}

4
Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java
View File

@ -35,7 +35,9 @@ public class MySQLDB extends SQLDB {
try {
Class.forName("com.mysql.jdbc.Driver");
String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/" + config.getString("mysql.database");
String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/"
+ config.getString("mysql.database")
+ "?rewriteBatchedStatements=true";
return DriverManager.getConnection(url, config.getString("mysql.user"), config.getString("mysql.password"));
} catch (ClassNotFoundException | SQLException e) {

20
Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java
View File

@ -424,19 +424,24 @@ public abstract class SQLDB extends Database {
*/
@Override
public void saveMultipleUserData(Collection<UserData> data) throws SQLException {
Benchmark.start("Database: Save multiple Userdata");
checkConnection();
if (data.isEmpty()) {
if (data == null || data.isEmpty()) {
return;
}
Benchmark.start("Database: Save multiple Userdata");
data.removeIf(Objects::isNull);
checkConnection();
setStatus("Save userdata (multiple) for " + data.size());
usersTable.saveUserDataInformationBatch(data);
// Transform to map
Map<UUID, UserData> userDatas = data.stream().collect(Collectors.toMap(UserData::getUuid, Function.identity()));
Map<UUID, UserData> userDatas = data.stream()
.collect(Collectors.toMap(UserData::getUuid, Function.identity()));
// Get UserIDs
Map<UUID, Integer> userIds = usersTable.getAllUserIds();
// Empty dataset
// Create empty data sets
Map<Integer, Set<String>> nicknames = new HashMap<>();
Map<Integer, String> lastNicks = new HashMap<>();
Map<Integer, Set<InetAddress>> ips = new HashMap<>();
@ -445,7 +450,8 @@ public abstract class SQLDB extends Database {
Map<Integer, List<SessionData>> sessions = new HashMap<>();
Map<Integer, Map<String, Long>> gmTimes = new HashMap<>();
Map<Integer, Map<String, Long>> worldTimes = new HashMap<>();
// Put to dataset
// Put in data set
List<String> worldNames = data.stream()
.map(UserData::getWorldTimes)
.map(WorldTimes::getTimes)
@ -453,6 +459,7 @@ public abstract class SQLDB extends Database {
.flatMap(Collection::stream)
.distinct()
.collect(Collectors.toList());
for (Map.Entry<UUID, UserData> entrySet : userDatas.entrySet()) {
UUID uuid = entrySet.getKey();
UserData uData = entrySet.getValue();
@ -472,6 +479,7 @@ public abstract class SQLDB extends Database {
gmTimes.put(id, new HashMap<>(uData.getGmTimes().getTimes()));
worldTimes.put(id, new HashMap<>(uData.getWorldTimes().getTimes()));
}
// Save
nicknamesTable.saveNickLists(nicknames, lastNicks);
ipsTable.saveIPList(ips);

1
Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java
View File

@ -83,6 +83,7 @@ public class CommandUseTable extends Table {
if (data.isEmpty()) {
return;
}
Benchmark.start("Database: Save Commanduse");
Map<String, Integer> newData = new HashMap<>(data);
Map<String, Integer> saved = getCommandUse();

157
Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java
View File

@ -2,6 +2,9 @@ package main.java.com.djrapitops.plan.database.tables;
import com.djrapitops.plugin.utilities.Verify;
import main.java.com.djrapitops.plan.Log;
import main.java.com.djrapitops.plan.data.time.GMTimes;
import main.java.com.djrapitops.plan.database.Container;
import main.java.com.djrapitops.plan.database.DBUtils;
import main.java.com.djrapitops.plan.database.databases.SQLDB;
import main.java.com.djrapitops.plan.utilities.Benchmark;
@ -93,12 +96,14 @@ public class GMTimesTable extends Table {
statement.setInt(1, userId);
set = statement.executeQuery();
HashMap<String, Long> times = new HashMap<>();
while (set.next()) {
times.put("SURVIVAL", set.getLong(columnSurvivalTime));
times.put("CREATIVE", set.getLong(columnCreativeTime));
times.put("ADVENTURE", set.getLong(columnAdventureTime));
times.put("SPECTATOR", set.getLong(columnSpectatorTime));
}
return times;
} finally {
close(set);
@ -113,18 +118,21 @@ public class GMTimesTable extends Table {
try {
statement = prepareStatement("SELECT * FROM " + tableName);
set = statement.executeQuery();
while (set.next()) {
Map<String, Long> gmTimes = new HashMap<>();
int id = set.getInt(columnUserID);
if (!userIds.contains(id)) {
continue;
}
gmTimes.put("SURVIVAL", set.getLong(columnSurvivalTime));
gmTimes.put("CREATIVE", set.getLong(columnCreativeTime));
gmTimes.put("ADVENTURE", set.getLong(columnAdventureTime));
gmTimes.put("SPECTATOR", set.getLong(columnSpectatorTime));
times.put(id, gmTimes);
}
return times;
} finally {
close(set);
@ -141,8 +149,10 @@ public class GMTimesTable extends Table {
if (Verify.isEmpty(gamemodeTimes)) {
return;
}
PreparedStatement statement = null;
String[] gms = getGMKeyArray();
int update;
try {
statement = prepareStatement(
@ -153,22 +163,17 @@ public class GMTimesTable extends Table {
+ columnSpectatorTime + "=? "
+ " WHERE (" + columnUserID + "=?)");
statement.setInt(5, userId);
for (int i = 0; i < gms.length; i++) {
try {
Long time = gamemodeTimes.get(gms[i]);
if (time != null) {
statement.setLong(i + 1, time);
} else {
statement.setLong(i + 1, 0);
}
} catch (NoSuchFieldError e) {
statement.setLong(i + 1, 0);
}
statement.setLong(i + 1, time != null ? time : 0);
}
update = statement.executeUpdate();
} finally {
close(statement);
}
if (update == 0) {
addNewGMTimesRow(userId, gamemodeTimes);
}
@ -195,11 +200,52 @@ public class GMTimesTable extends Table {
if (Verify.isEmpty(gamemodeTimes)) {
return;
}
Benchmark.start("Database: Save GMTimes");
PreparedStatement statement = null;
String[] gms = getGMKeyArray();
Set<Integer> savedIDs = getSavedIDs();
Map<Integer, GMTimes> gmTimes = new HashMap<>();
for (Map.Entry<Integer, Map<String, Long>> entrySet : gamemodeTimes.entrySet()) {
int userID = entrySet.getKey();
if (!savedIDs.contains(userID)) {
continue;
}
Map<String, Long> gmTimesMap = entrySet.getValue();
gmTimes.put(userID, new GMTimes(gmTimesMap));
}
List<List<Container<GMTimes>>> batches = DBUtils.splitIntoBatchesWithID(gmTimes);
batches.parallelStream().forEach(batch -> {
try {
saveGMTimesBatch(batch);
} catch (SQLException e) {
Log.toLog("GMTimesTable.saveGMTimes", e);
}
});
gamemodeTimes.keySet().removeAll(savedIDs);
addNewGMTimesRows(gamemodeTimes);
Benchmark.stop("Database: Save GMTimes");
}
private void saveGMTimesBatch(List<Container<GMTimes>> batch) throws SQLException {
if (batch.isEmpty()) {
return;
}
int batchSize = batch.size();
Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize);
String[] gms = getGMKeyArray();
Set<Integer> savedIDs = getSavedIDs();
PreparedStatement statement = null;
try {
statement = prepareStatement(
"UPDATE " + tableName + " SET "
@ -208,48 +254,72 @@ public class GMTimesTable extends Table {
+ columnAdventureTime + "=?, "
+ columnSpectatorTime + "=? "
+ " WHERE (" + columnUserID + "=?)");
boolean commitRequired = false;
for (Map.Entry<Integer, Map<String, Long>> entrySet : gamemodeTimes.entrySet()) {
Integer id = entrySet.getKey();
for (Container<GMTimes> data : batch) {
int id = data.getId();
if (!savedIDs.contains(id)) {
continue;
}
statement.setInt(5, id);
for (int i = 0; i < gms.length; i++) {
try {
Map<String, Long> times = entrySet.getValue();
Map<String, Long> times = data.getObject().getTimes();
Long time = times.get(gms[i]);
statement.setLong(i + 1, time != null ? time : 0);
} catch (NoSuchFieldError e) {
statement.setLong(i + 1, 0);
}
}
statement.addBatch();
commitRequired = true;
}
if (commitRequired) {
Log.debug("Executing GM Times batch: " + batchSize);
statement.executeBatch();
}
gamemodeTimes.keySet().removeAll(savedIDs);
} finally {
close(statement);
}
addNewGMTimesRows(gamemodeTimes);
Benchmark.stop("Database: Save GMTimes");
}
private void addNewGMTimesRows(Map<Integer, Map<String, Long>> gamemodeTimes) throws SQLException {
if (Verify.isEmpty(gamemodeTimes)) {
return;
}
PreparedStatement statement = null;
Benchmark.start("Database: Add GMTimes Rows");
Map<Integer, GMTimes> gmTimes = new HashMap<>();
for (Map.Entry<Integer, Map<String, Long>> entrySet : gamemodeTimes.entrySet()) {
int userID = entrySet.getKey();
Map<String, Long> gmTimesMap = entrySet.getValue();
gmTimes.put(userID, new GMTimes(gmTimesMap));
}
List<List<Container<GMTimes>>> batches = DBUtils.splitIntoBatchesWithID(gmTimes);
batches.parallelStream().forEach(batch -> {
try {
addNewGMTimesBatch(batch);
} catch (SQLException e) {
e.printStackTrace();
}
});
Benchmark.stop("Database: Add GMTimes Rows");
}
private void addNewGMTimesBatch(List<Container<GMTimes>> batch) throws SQLException {
if (batch.isEmpty()) {
return;
}
int batchSize = batch.size();
Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize);
String[] gms = getGMKeyArray();
PreparedStatement statement = null;
try {
statement = prepareStatement(
"INSERT INTO " + tableName + " ("
@ -259,28 +329,22 @@ public class GMTimesTable extends Table {
+ columnAdventureTime + ", "
+ columnSpectatorTime
+ ") VALUES (?, ?, ?, ?, ?)");
boolean commitRequired = false;
for (Map.Entry<Integer, Map<String, Long>> entry : gamemodeTimes.entrySet()) {
Integer id = entry.getKey();
statement.setInt(1, id);
for (Container<GMTimes> data : batch) {
statement.setInt(1, data.getId());
for (int i = 0; i < gms.length; i++) {
try {
Map<String, Long> times = entry.getValue();
Map<String, Long> times = data.getObject().getTimes();
Long time = times.get(gms[i]);
statement.setLong(i + 2, time != null ? time : 0);
} catch (NoSuchFieldError e) {
statement.setLong(i + 2, 0);
}
}
statement.addBatch();
commitRequired = true;
}
if (commitRequired) {
statement.executeBatch();
statement.addBatch();
}
Log.debug("Executing GM Times batch: " + batchSize);
statement.executeBatch();
} finally {
close(statement);
}
@ -290,6 +354,7 @@ public class GMTimesTable extends Table {
if (Verify.isEmpty(gamemodeTimes)) {
return;
}
PreparedStatement statement = null;
String[] gms = getGMKeyArray();
try {
@ -302,14 +367,10 @@ public class GMTimesTable extends Table {
+ ") VALUES (?, ?, ?, ?, ?)");
statement.setInt(1, userId);
for (int i = 0; i < gms.length; i++) {
try {
Long time = gamemodeTimes.get(gms[i]);
for (int i = 0; i < gms.length; i++) {
Long time = gamemodeTimes.get(gms[i]);
statement.setLong(i + 2, time != null ? time : 0);
} catch (NoSuchFieldError e) {
statement.setLong(i + 2, 0);
}
}
statement.execute();

1
Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java
View File

@ -90,6 +90,7 @@ public class IPsTable extends Table {
Log.error("Host not found at getIPAddresses: " + ipAddressName); //Shouldn't ever happen
}
}
return ips;
} finally {
close(set);

50
Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java
View File

@ -107,36 +107,38 @@ public class SessionsTable extends Table {
if (sessions == null) {
return;
}
Benchmark.start("Database: Save Sessions");
sessions.removeAll(getSessionData(userId));
if (sessions.isEmpty()) {
Benchmark.stop("Database: Save Sessions");
return;
}
PreparedStatement statement = null;
try {
PreparedStatement statement = null;
try {
statement = prepareStatement("INSERT INTO " + tableName + " ("
+ columnUserID + ", "
+ columnSessionStart + ", "
+ columnSessionEnd
+ ") VALUES (?, ?, ?)");
boolean commitRequired = false;
for (SessionData session : sessions) {
long end = session.getSessionEnd();
long start = session.getSessionStart();
if (end < start) {
continue;
}
statement.setInt(1, userId);
statement.setLong(2, start);
statement.setLong(3, end);
statement.addBatch();
commitRequired = true;
}
if (commitRequired) {
statement.executeBatch();
}
} finally {
close(statement);
Benchmark.stop("Database: Save Sessions");
@ -152,25 +154,31 @@ public class SessionsTable extends Table {
if (ids == null || ids.isEmpty()) {
return new HashMap<>();
}
Benchmark.start("Database: Get Sessions multiple");
PreparedStatement statement = null;
ResultSet set = null;
try {
Map<Integer, List<SessionData>> sessions = new HashMap<>();
statement = prepareStatement("SELECT * FROM " + tableName);
set = statement.executeQuery();
for (Integer id : ids) {
sessions.put(id, new ArrayList<>());
}
while (set.next()) {
Integer id = set.getInt(columnUserID);
if (!ids.contains(id)) {
continue;
}
sessions.get(id).add(new SessionData(set.getLong(columnSessionStart), set.getLong(columnSessionEnd)));
long sessionStart = set.getLong(columnSessionStart);
long sessionEnd = set.getLong(columnSessionEnd);
sessions.get(id).add(new SessionData(sessionStart, sessionEnd));
}
set.close();
statement.close();
return sessions;
} finally {
@ -195,8 +203,8 @@ public class SessionsTable extends Table {
for (Map.Entry<Integer, List<SessionData>> entrySet : sessions.entrySet()) {
Integer id = entrySet.getKey();
List<SessionData> sessionList = entrySet.getValue();
List<SessionData> s = saved.get(id);
if (s != null) {
sessionList.removeAll(s);
}
@ -207,11 +215,17 @@ public class SessionsTable extends Table {
saved.put(id, sessionList);
}
List<List<Container<SessionData>>> batches = splitIntoBatches(sessions);
for (List<Container<SessionData>> batch : batches) {
batches.parallelStream().forEach(batch -> {
try {
saveSessionBatch(batch);
} catch (SQLException e) {
e.printStackTrace();
}
});
Benchmark.stop("Database: Save Sessions multiple");
}
@ -219,6 +233,10 @@ public class SessionsTable extends Table {
if (batch.isEmpty()) {
return;
}
int batchSize = batch.size();
Log.debug("Preparing insertion of sessions... Batch Size: " + batchSize);
PreparedStatement statement = null;
try {
statement = prepareStatement("INSERT INTO " + tableName + " ("
@ -227,25 +245,21 @@ public class SessionsTable extends Table {
+ columnSessionEnd
+ ") VALUES (?, ?, ?)");
boolean commitRequired = false;
int i = 0;
for (Container<SessionData> data : batch) {
SessionData session = data.getObject();
int id = data.getId();
if (!session.isValid()) {
continue;
}
statement.setInt(1, id);
statement.setLong(2, session.getSessionStart());
statement.setLong(3, session.getSessionEnd());
statement.addBatch();
commitRequired = true;
i++;
}
if (commitRequired) {
Log.debug("Executing session batch: " + i);
Log.debug("Executing session batch: " + batchSize);
statement.executeBatch();
}
} finally {
close(statement);
}

23
Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java
View File

@ -119,12 +119,24 @@ public class TPSTable extends Table {
*/
public void saveTPSData(List<TPS> data) throws SQLException {
List<List<TPS>> batches = DBUtils.splitIntoBatches(data);
for (List<TPS> batch : batches) {
batches.parallelStream()
.forEach(batch -> {
try {
saveTPSBatch(batch);
} catch (SQLException e) {
Log.toLog("UsersTable.saveUserDataInformationBatch", e);
}
});
}
private void saveTPSBatch(List<TPS> batch) throws SQLException {
if (batch.isEmpty()) {
return;
}
int batchSize = batch.size();
Log.debug("Preparing insertion of TPS... Batch Size: " + batchSize);
PreparedStatement statement = null;
try {
statement = prepareStatement("INSERT INTO " + tableName + " ("
@ -137,8 +149,6 @@ public class TPSTable extends Table {
+ columnChunksLoaded
+ ") VALUES (?, ?, ?, ?, ?, ?, ?)");
boolean commitRequired = false;
int i = 0;
for (TPS tps : batch) {
statement.setLong(1, tps.getDate());
statement.setDouble(2, tps.getTps());
@ -148,13 +158,10 @@ public class TPSTable extends Table {
statement.setDouble(6, tps.getEntityCount());
statement.setDouble(7, tps.getChunksLoaded());
statement.addBatch();
commitRequired = true;
i++;
}
if (commitRequired) {
Log.debug("Executing tps batch: " + i);
Log.debug("Executing tps batch: " + batchSize);
statement.executeBatch();
}
} finally {
close(statement);
}

32
Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java
View File

@ -680,10 +680,18 @@ public class UsersTable extends Table {
try {
List<UserData> newUserdata = updateExistingUserData(data);
Benchmark.start("Database: Insert new UserInfo multiple");
List<List<UserData>> batches = DBUtils.splitIntoBatches(newUserdata);
for (List<UserData> batch : batches) {
batches.parallelStream()
.forEach(batch -> {
try {
insertNewUserData(batch);
} catch (SQLException e) {
Log.toLog("UsersTable.saveUserDataInformationBatch", e);
}
});
Benchmark.stop("Database: Insert new UserInfo multiple");
} finally {
Benchmark.stop("Database: Save UserInfo multiple");
@ -691,18 +699,27 @@ public class UsersTable extends Table {
}
private void insertNewUserData(Collection<UserData> data) throws SQLException {
if (data.isEmpty()) {
return;
}
int batchSize = data.size();
Log.debug("Preparing insertion of new users... Batch Size: " + batchSize);
PreparedStatement statement = null;
try {
statement = prepareStatement(getInsertStatement());
boolean commitRequired = false;
int i = 0;
for (UserData uData : data) {
UUID uuid = uData.getUuid();
statement.setString(1, uuid.toString());
statement.setString(2, uData.getGeolocation());
GMTimes gmTimes = uData.getGmTimes();
statement.setString(3, gmTimes.getState());
statement.setLong(4, gmTimes.getLastStateChange());
statement.setLong(5, uData.getPlayTime());
statement.setInt(6, uData.getLoginTimes());
statement.setLong(7, uData.getLastPlayed());
@ -713,17 +730,16 @@ public class UsersTable extends Table {
statement.setBoolean(12, uData.isBanned());
statement.setString(13, uData.getName());
statement.setLong(14, uData.getRegistered());
WorldTimes worldTimes = uData.getWorldTimes();
statement.setString(15, worldTimes.getState());
statement.setLong(16, worldTimes.getLastStateChange());
statement.addBatch();
commitRequired = true;
i++;
}
if (commitRequired) {
Log.debug("Executing session batch: " + i);
Log.debug("Executing users batch: " + batchSize);
statement.executeBatch();
}
} finally {
close(statement);
}