Refactored SessionsTable#insertSessions to an executable:

Also affected:
- KillsTable#savePlayerKills
- WorldTimesTable#saveWorldTimes
This commit is contained in:
Rsl1122 2019-01-25 12:55:24 +02:00
parent f48b45ccb2
commit 7559ab2904
10 changed files with 204 additions and 232 deletions

View File

@ -105,6 +105,6 @@ public class BackupCopyTransaction extends RemoveEverythingTransaction {
}
private void copySessions() {
db.getSessionsTable().insertSessions(sourceDB.query(LargeFetchQueries.fetchAllSessionsWithKillAndWorldData()), true);
copy(LargeStoreQueries::storeAllSessionsWithKillAndWorldData, LargeFetchQueries.fetchAllSessionsFlatWithKillAndWorldData());
}
}

View File

@ -31,6 +31,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;
/**
* Static method class for queries that use large amount of memory.
@ -380,6 +381,20 @@ public class LargeFetchQueries {
};
}
/**
* Query the database for Session data with kill, death or world data.
*
* @return List of sessions
*/
public static Query<List<Session>> fetchAllSessionsFlatWithKillAndWorldData() {
return db -> db.query(fetchAllSessionsWithKillAndWorldData())
.values().stream()
.map(Map::values)
.flatMap(Collection::stream)
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
/**
* Query database for TPS data.
*

View File

@ -17,11 +17,10 @@
package com.djrapitops.plan.db.sql.queries;
import com.djrapitops.plan.data.WebUser;
import com.djrapitops.plan.data.container.BaseUser;
import com.djrapitops.plan.data.container.GeoInfo;
import com.djrapitops.plan.data.container.TPS;
import com.djrapitops.plan.data.container.UserInfo;
import com.djrapitops.plan.data.container.*;
import com.djrapitops.plan.data.store.keys.SessionKeys;
import com.djrapitops.plan.data.store.objects.Nickname;
import com.djrapitops.plan.data.time.GMTimes;
import com.djrapitops.plan.db.access.ExecBatchStatement;
import com.djrapitops.plan.db.access.Executable;
import com.djrapitops.plan.db.sql.tables.*;
@ -302,6 +301,12 @@ public class LargeStoreQueries {
};
}
/**
* Execute a big batch of user information insert statements.
*
* @param ofUsers Collection of BaseUsers
* @return Executable, use inside a {@link com.djrapitops.plan.db.access.transactions.Transaction}
*/
public static Executable storeAllCommonUserInformation(Collection<BaseUser> ofUsers) {
if (Verify.isEmpty(ofUsers)) {
return Executable.empty();
@ -320,4 +325,129 @@ public class LargeStoreQueries {
}
};
}
public static Executable storeAllSessionsWithoutKillOrWorldData(List<Session> sessions) {
if (Verify.isEmpty(sessions)) {
return Executable.empty();
}
return new ExecBatchStatement(SessionsTable.INSERT_STATEMENT) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
for (Session session : sessions) {
statement.setString(1, session.getUnsafe(SessionKeys.UUID).toString());
statement.setLong(2, session.getUnsafe(SessionKeys.START));
statement.setLong(3, session.getUnsafe(SessionKeys.END));
statement.setInt(4, session.getUnsafe(SessionKeys.DEATH_COUNT));
statement.setInt(5, session.getUnsafe(SessionKeys.MOB_KILL_COUNT));
statement.setLong(6, session.getUnsafe(SessionKeys.AFK_TIME));
statement.setString(7, session.getUnsafe(SessionKeys.SERVER_UUID).toString());
statement.addBatch();
}
}
};
}
public static Executable storeAllSessionsWithKillAndWorldData(List<Session> sessions) {
return connection -> {
storeAllSessionsWithoutKillOrWorldData(sessions).execute(connection);
storeSessionKillData(sessions).execute(connection);
storeSessionWorldTimeData(sessions).execute(connection);
return false;
};
}
private static Executable storeSessionKillData(List<Session> sessions) {
if (Verify.isEmpty(sessions)) {
return Executable.empty();
}
String sql = "INSERT INTO " + KillsTable.TABLE_NAME + " ("
+ KillsTable.SESSION_ID + ", "
+ KillsTable.KILLER_UUID + ", "
+ KillsTable.VICTIM_UUID + ", "
+ KillsTable.SERVER_UUID + ", "
+ KillsTable.DATE + ", "
+ KillsTable.WEAPON
+ ") VALUES (" + SessionsTable.SELECT_SESSION_ID_STATEMENT + ", ?, ?, ?, ?, ?)";
return new ExecBatchStatement(sql) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
for (Session session : sessions) {
UUID uuid = session.getUnsafe(SessionKeys.UUID);
UUID serverUUID = session.getUnsafe(SessionKeys.SERVER_UUID);
for (PlayerKill kill : session.getPlayerKills()) {
// Session ID select statement
statement.setString(1, uuid.toString());
statement.setString(2, serverUUID.toString());
statement.setLong(3, session.getUnsafe(SessionKeys.START));
statement.setLong(4, session.getUnsafe(SessionKeys.END));
statement.setString(5, uuid.toString());
statement.setString(6, kill.getVictim().toString());
statement.setString(7, serverUUID.toString());
statement.setLong(8, kill.getDate());
statement.setString(9, kill.getWeapon());
statement.addBatch();
}
}
}
};
}
private static Executable storeSessionWorldTimeData(List<Session> sessions) {
if (Verify.isEmpty(sessions)) {
return Executable.empty();
}
String sql = "INSERT INTO " + WorldTimesTable.TABLE_NAME + " (" +
WorldTimesTable.SESSION_ID + ", " +
WorldTimesTable.WORLD_ID + ", " +
WorldTimesTable.USER_UUID + ", " +
WorldTimesTable.SERVER_UUID + ", " +
WorldTimesTable.SURVIVAL + ", " +
WorldTimesTable.CREATIVE + ", " +
WorldTimesTable.ADVENTURE + ", " +
WorldTimesTable.SPECTATOR +
") VALUES ( " +
SessionsTable.SELECT_SESSION_ID_STATEMENT + ", " +
WorldTable.SELECT_WORLD_ID_STATEMENT + ", " +
"?, ?, ?, ?, ?, ?)";
return new ExecBatchStatement(sql) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
String[] gms = GMTimes.getGMKeyArray();
for (Session session : sessions) {
UUID uuid = session.getUnsafe(SessionKeys.UUID);
UUID serverUUID = session.getUnsafe(SessionKeys.SERVER_UUID);
Map<String, GMTimes> worldTimes = session.getUnsafe(SessionKeys.WORLD_TIMES).getWorldTimes();
for (Map.Entry<String, GMTimes> worldTimesEntry : worldTimes.entrySet()) {
String worldName = worldTimesEntry.getKey();
GMTimes gmTimes = worldTimesEntry.getValue();
// Session ID select statement
statement.setString(1, uuid.toString());
statement.setString(2, serverUUID.toString());
statement.setLong(3, session.getUnsafe(SessionKeys.START));
statement.setLong(4, session.getUnsafe(SessionKeys.END));
// World ID select statement
statement.setString(5, worldName);
statement.setString(6, serverUUID.toString());
statement.setString(7, uuid.toString());
statement.setString(8, serverUUID.toString());
statement.setLong(9, gmTimes.getTime(gms[0]));
statement.setLong(10, gmTimes.getTime(gms[1]));
statement.setLong(11, gmTimes.getTime(gms[2]));
statement.setLong(12, gmTimes.getTime(gms[3]));
statement.addBatch();
}
}
}
};
}
}

View File

@ -212,41 +212,4 @@ public class KillsTable extends Table {
}
}
}
public void savePlayerKills(Map<UUID, Map<UUID, List<Session>>> allSessions) {
if (Verify.isEmpty(allSessions)) {
return;
}
executeBatch(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
// Every server
for (UUID serverUUID : allSessions.keySet()) {
// Every player
for (Map.Entry<UUID, List<Session>> entry : allSessions.get(serverUUID).entrySet()) {
UUID killer = entry.getKey();
List<Session> sessions = entry.getValue();
// Every session
for (Session session : sessions) {
int sessionID = session.getUnsafe(SessionKeys.DB_ID);
// Every kill
for (PlayerKill kill : session.getPlayerKills()) {
UUID victim = kill.getVictim();
long date = kill.getDate();
String weapon = kill.getWeapon();
statement.setString(1, killer.toString());
statement.setString(2, victim.toString());
statement.setString(3, serverUUID.toString());
statement.setInt(4, sessionID);
statement.setLong(5, date);
statement.setString(6, weapon);
statement.addBatch();
}
}
}
}
}
});
}
}

View File

@ -29,8 +29,6 @@ import com.djrapitops.plan.db.patches.Version10Patch;
import com.djrapitops.plan.db.sql.parsing.CreateTableParser;
import com.djrapitops.plan.db.sql.parsing.Select;
import com.djrapitops.plan.db.sql.parsing.Sql;
import com.djrapitops.plan.db.sql.queries.LargeFetchQueries;
import com.djrapitops.plugin.utilities.Verify;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@ -62,19 +60,24 @@ public class SessionsTable extends Table {
public static final String DEATHS = "deaths";
public static final String AFK_TIME = "afk_time";
private String insertStatement;
public static final String INSERT_STATEMENT = "INSERT INTO " + TABLE_NAME + " ("
+ USER_UUID + ", "
+ SESSION_START + ", "
+ SESSION_END + ", "
+ DEATHS + ", "
+ MOB_KILLS + ", "
+ AFK_TIME + ", "
+ SERVER_UUID
+ ") VALUES (?, ?, ?, ?, ?, ?, ?)";
public static final String SELECT_SESSION_ID_STATEMENT = "(SELECT " + TABLE_NAME + "." + ID + " FROM " + TABLE_NAME +
" WHERE " + TABLE_NAME + "." + USER_UUID + "=?" +
" AND " + TABLE_NAME + "." + SERVER_UUID + "=?" +
" AND " + SESSION_START + "=?" +
" AND " + SESSION_END + "=? LIMIT 1)";
public SessionsTable(SQLDB db) {
super(TABLE_NAME, db);
insertStatement = "INSERT INTO " + tableName + " ("
+ USER_UUID + ", "
+ SESSION_START + ", "
+ SESSION_END + ", "
+ DEATHS + ", "
+ MOB_KILLS + ", "
+ AFK_TIME + ", "
+ SERVER_UUID
+ ") VALUES (?, ?, ?, ?, ?, ?, ?)";
}
public static String createTableSQL(DBType dbType) {
@ -149,7 +152,7 @@ public class SessionsTable extends Table {
* @param session Session of the player that has ended ({@code endSession} has been called)
*/
private void saveSessionInformation(UUID uuid, Session session) {
execute(new ExecStatement(insertStatement) {
execute(new ExecStatement(INSERT_STATEMENT) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, uuid.toString());
@ -445,102 +448,6 @@ public class SessionsTable extends Table {
});
}
public void insertSessions(Map<UUID, Map<UUID, List<Session>>> allSessions, boolean saveKillsAndWorldTimes) {
if (Verify.isEmpty(allSessions)) {
return;
}
executeBatch(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
for (UUID serverUUID : allSessions.keySet()) {
for (Map.Entry<UUID, List<Session>> entry : allSessions.get(serverUUID).entrySet()) {
UUID uuid = entry.getKey();
List<Session> sessions = entry.getValue();
for (Session session : sessions) {
statement.setString(1, uuid.toString());
statement.setLong(2, session.getUnsafe(SessionKeys.START));
statement.setLong(3, session.getUnsafe(SessionKeys.END));
statement.setInt(4, session.getUnsafe(SessionKeys.DEATH_COUNT));
statement.setInt(5, session.getUnsafe(SessionKeys.MOB_KILL_COUNT));
statement.setLong(6, session.getUnsafe(SessionKeys.AFK_TIME));
statement.setString(7, serverUUID.toString());
statement.addBatch();
}
}
}
}
});
if (saveKillsAndWorldTimes) {
Map<UUID, Map<UUID, List<Session>>> savedSessions = db.query(LargeFetchQueries.fetchAllSessionsWithoutKillOrWorldData());
matchSessionIDs(allSessions, savedSessions);
db.getKillsTable().savePlayerKills(allSessions);
db.getWorldTimesTable().saveWorldTimes(allSessions);
}
}
/**
* Sessions should be saved before calling this method.
*
* @param allSessions Sessions to match IDs to (contain extra data)
* @param allSavedSessions Sessions in the Database.
*/
private void matchSessionIDs(Map<UUID, Map<UUID, List<Session>>> allSessions, Map<UUID, Map<UUID, List<Session>>> allSavedSessions) {
for (UUID serverUUID : allSessions.keySet()) {
Map<UUID, List<Session>> serverSessions = allSessions.get(serverUUID);
Map<UUID, List<Session>> savedServerSessions = allSavedSessions.get(serverUUID);
for (Map.Entry<UUID, List<Session>> entry : serverSessions.entrySet()) {
UUID uuid = entry.getKey();
List<Session> sessions = entry.getValue();
List<Session> savedSessions = savedServerSessions.get(uuid);
if (savedSessions == null) {
throw new IllegalStateException("Some of the sessions being matched were not saved.");
}
matchSessions(sessions, savedSessions);
}
}
}
/**
* Used by matchSessionIDs method.
* <p>
* Matches IDs of Sessions with by sessionStart.
* Assumes that both lists are from the same user and server.
*
* @param sessions Sessions of Player in a Server.
* @param savedSessions Sessions of Player in a Server in the db.
*/
private void matchSessions(List<Session> sessions, List<Session> savedSessions) {
Map<Long, List<Session>> sessionsByStart = turnToMapByStart(sessions);
Map<Long, List<Session>> savedSessionsByStart = turnToMapByStart(savedSessions);
for (Map.Entry<Long, List<Session>> sessionEntry : sessionsByStart.entrySet()) {
long start = sessionEntry.getKey();
if (!savedSessionsByStart.containsKey(start)) {
throw new IllegalStateException("Some of the sessions being matched were not saved.");
}
Session savedSession = savedSessionsByStart.get(start).get(0);
sessionEntry.getValue().forEach(
session -> session.setSessionID(savedSession.getUnsafe(SessionKeys.DB_ID))
);
}
}
private Map<Long, List<Session>> turnToMapByStart(List<Session> sessions) {
Map<Long, List<Session>> sessionsByStart = new TreeMap<>();
for (Session session : sessions) {
long start = session.getUnsafe(SessionKeys.START);
List<Session> sorted = sessionsByStart.getOrDefault(start, new ArrayList<>());
sorted.add(session);
sessionsByStart.put(start, sorted);
}
return sessionsByStart;
}
public Map<Integer, Integer> getIDServerIDRelation() {
String sql = "SELECT " +
ID + ", " +

View File

@ -56,16 +56,16 @@ public class WorldTable extends Table {
+ SERVER_UUID
+ ") VALUES (?, ?)";
public final String statementSelectID;
public static final String SELECT_WORLD_ID_STATEMENT = "(SELECT " + TABLE_NAME + "." + ID + " FROM " + TABLE_NAME +
" WHERE (" + NAME + "=?)" +
" AND (" + TABLE_NAME + "." + SERVER_UUID + "=?)" +
" LIMIT 1)";
private final ServerTable serverTable;
public WorldTable(SQLDB db) {
super(TABLE_NAME, db);
serverTable = db.getServerTable();
statementSelectID = "(SELECT " + ID + " FROM " + tableName +
" WHERE (" + NAME + "=?)" +
" AND (" + SERVER_UUID + "=?)" +
" LIMIT 1)";
}
public static String createTableSQL(DBType dbType) {
@ -162,13 +162,13 @@ public class WorldTable extends Table {
if (!(o instanceof WorldTable)) return false;
if (!super.equals(o)) return false;
WorldTable that = (WorldTable) o;
return Objects.equals(statementSelectID, that.statementSelectID) &&
return Objects.equals(SELECT_WORLD_ID_STATEMENT, SELECT_WORLD_ID_STATEMENT) &&
Objects.equals(serverTable, that.serverTable);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), statementSelectID, serverTable);
return Objects.hash(super.hashCode(), SELECT_WORLD_ID_STATEMENT, serverTable);
}
}

View File

@ -37,7 +37,6 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;
/**
* Table that is in charge of storing playtime data for each world in each GameMode.
@ -84,7 +83,7 @@ public class WorldTimesTable extends Table {
ADVENTURE + ", " +
SPECTATOR +
") VALUES (?, " +
worldTable.statementSelectID + ", " +
WorldTable.SELECT_WORLD_ID_STATEMENT + ", " +
"?, ?, ?, ?, ?, ?)";
}
@ -324,59 +323,6 @@ public class WorldTimesTable extends Table {
}
}
public void saveWorldTimes(Map<UUID, Map<UUID, List<Session>>> allSessions) {
if (Verify.isEmpty(allSessions)) {
return;
}
List<String> worldNames = allSessions.values().stream()
.map(Map::values)
.flatMap(Collection::stream)
.flatMap(Collection::stream)
.map(s -> s.getUnsafe(SessionKeys.WORLD_TIMES))
.map(WorldTimes::getWorldTimes)
.map(Map::keySet)
.flatMap(Collection::stream)
.distinct()
.collect(Collectors.toList());
db.getWorldTable().saveWorlds(worldNames);
executeBatch(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
String[] gms = GMTimes.getGMKeyArray();
// Every Server
for (Map.Entry<UUID, Map<UUID, List<Session>>> serverSessions : allSessions.entrySet()) {
UUID serverUUID = serverSessions.getKey();
// Every User
for (Map.Entry<UUID, List<Session>> entry : serverSessions.getValue().entrySet()) {
UUID uuid = entry.getKey();
List<Session> sessions = entry.getValue();
// Every Session
for (Session session : sessions) {
int sessionID = session.getUnsafe(SessionKeys.DB_ID);
// Every WorldTimes
for (Map.Entry<String, GMTimes> worldTimesEntry : session.getUnsafe(SessionKeys.WORLD_TIMES)
.getWorldTimes().entrySet()) {
String worldName = worldTimesEntry.getKey();
GMTimes gmTimes = worldTimesEntry.getValue();
statement.setString(1, uuid.toString());
statement.setString(2, worldName);
statement.setString(3, serverUUID.toString());
statement.setString(4, serverUUID.toString());
statement.setInt(5, sessionID);
statement.setLong(6, gmTimes.getTime(gms[0]));
statement.setLong(7, gmTimes.getTime(gms[1]));
statement.setLong(8, gmTimes.getTime(gms[2]));
statement.setLong(9, gmTimes.getTime(gms[3]));
statement.addBatch();
}
}
}
}
}
});
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

View File

@ -127,8 +127,8 @@ public class SQLFetchOps extends SQLOps implements FetchOperations {
private List<PlayerContainer> getPlayerContainers(UUID serverUUID) {
List<PlayerContainer> containers = new ArrayList<>();
List<UserInfo> serverUserInfo = userInfoTable.getServerUserInfo(serverUUID);
Map<UUID, Integer> timesKicked = usersTable.getAllTimesKicked();
List<UserInfo> serverUserInfo = userInfoTable.getServerUserInfo(serverUUID); // TODO Optimize and sort out
Map<UUID, Integer> timesKicked = usersTable.getAllTimesKicked(); // TODO Optimize and sort out
Map<UUID, List<GeoInfo>> geoInfo = db.query(LargeFetchQueries.fetchAllGeoInfoData()); // TODO Optimize
Map<UUID, List<Ping>> allPings = db.query(LargeFetchQueries.fetchAllPingData()); // TODO Optimize
Map<UUID, List<Nickname>> allNicknames = db.query(LargeFetchQueries.fetchAllNicknameDataByPlayerUUIDs()); // TODO Optimize

View File

@ -78,7 +78,18 @@ public class SQLSaveOps extends SQLOps implements SaveOperations {
@Override
public void insertSessions(Map<UUID, Map<UUID, List<Session>>> ofServers, boolean containsExtraData) {
sessionsTable.insertSessions(ofServers, containsExtraData);
List<Session> sessions = ofServers.values().stream()
.map(Map::values)
.flatMap(Collection::stream)
.flatMap(Collection::stream)
.collect(Collectors.toList());
db.executeTransaction(new Transaction() {
@Override
protected void performOperations() {
execute(containsExtraData ? LargeStoreQueries.storeAllSessionsWithKillAndWorldData(sessions)
: LargeStoreQueries.storeAllSessionsWithoutKillOrWorldData(sessions));
}
});
}
@Override

View File

@ -32,8 +32,10 @@ import com.djrapitops.plan.db.access.Query;
import com.djrapitops.plan.db.access.transactions.BackupCopyTransaction;
import com.djrapitops.plan.db.access.transactions.RemoveEverythingTransaction;
import com.djrapitops.plan.db.access.transactions.RemovePlayerTransaction;
import com.djrapitops.plan.db.access.transactions.Transaction;
import com.djrapitops.plan.db.patches.Patch;
import com.djrapitops.plan.db.sql.queries.LargeFetchQueries;
import com.djrapitops.plan.db.sql.queries.LargeStoreQueries;
import com.djrapitops.plan.db.sql.tables.*;
import com.djrapitops.plan.db.tasks.CreateIndexTask;
import com.djrapitops.plan.system.PlanSystem;
@ -753,19 +755,18 @@ public abstract class CommonDBTest {
saveUserOne();
WorldTimes worldTimes = createWorldTimes();
SessionsTable sessionsTable = db.getSessionsTable();
WorldTimesTable worldTimesTable = db.getWorldTimesTable();
Session session = createSession();
session.setWorldTimes(worldTimes);
Map<UUID, Map<UUID, List<Session>>> map = new HashMap<>();
Map<UUID, List<Session>> sessionMap = new HashMap<>();
List<Session> sessions = new ArrayList<>();
sessions.add(session);
sessionMap.put(playerUUID, sessions);
map.put(serverUUID, sessionMap);
sessionsTable.insertSessions(map, true);
db.executeTransaction(new Transaction() {
@Override
protected void performOperations() {
execute(LargeStoreQueries.storeAllSessionsWithKillAndWorldData(sessions));
}
});
Map<Integer, WorldTimes> worldTimesBySessionID = worldTimesTable.getAllWorldTimesBySessionID();
System.out.println(worldTimesBySessionID);
@ -780,19 +781,18 @@ public abstract class CommonDBTest {
WorldTimes worldTimes = createWorldTimes();
Session session = createSession();
session.setWorldTimes(worldTimes);
Map<UUID, Map<UUID, List<Session>>> map = new HashMap<>();
Map<UUID, List<Session>> sessionMap = new HashMap<>();
List<Session> sessions = new ArrayList<>();
sessions.add(session);
sessionMap.put(playerUUID, sessions);
map.put(serverUUID, sessionMap);
db.executeTransaction(new Transaction() {
@Override
protected void performOperations() {
execute(LargeStoreQueries.storeAllSessionsWithKillAndWorldData(sessions));
}
});
sessionsTable.insertSessions(map, true);
List<Session> allSessions = db.query(LargeFetchQueries.fetchAllSessionsFlatWithKillAndWorldData());
Map<UUID, Map<UUID, List<Session>>> allSessions = db.query(LargeFetchQueries.fetchAllSessionsWithKillAndWorldData());
assertEquals(worldTimes, allSessions.get(serverUUID).get(playerUUID).get(0).getUnsafe(SessionKeys.WORLD_TIMES));
assertEquals(worldTimes, allSessions.get(0).getUnsafe(SessionKeys.WORLD_TIMES));
}
@Test