Upstream/Plan
1
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2024-07-09 13:15:22 +02:00
Code Issues Projects Releases Wiki Activity

Optimized (memory) /players page table query

Browse Source
This commit is contained in:
Rsl1122 2019-09-15 13:24:39 +03:00
parent 2412f5b9d5
commit e18ec134f6
6 changed files with 144 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java
View File

@ -97,7 +97,7 @@ public class JSONFactory {
Database database = dbSystem.getDatabase();
return new PlayersTableJSONParser(
Collections.emptyList(),// TODO Replace with new query
database.query(new NetworkTablePlayersQuery(System.currentTimeMillis(), playtimeThreshold, xMostRecentPlayers)),
Collections.emptyMap(),
openPlayerLinksInNewTab,
formatters

4
Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/analysis/NetworkActivityIndexQueries.java
View File

@ -76,7 +76,7 @@ public class NetworkActivityIndexQueries {
return fetchActivityGroupCount(date, playtimeThreshold, ActivityIndex.REGULAR, 5.1);
}
private static String selectActivityIndexSQL() {
public static String selectActivityIndexSQL() {
String selectActivePlaytimeSQL = SELECT +
SessionsTable.USER_UUID +
",SUM(" +
@ -96,7 +96,7 @@ public class NetworkActivityIndexQueries {
GROUP_BY + "q1." + SessionsTable.USER_UUID;
}
private static void setSelectActivityIndexSQLParameters(PreparedStatement statement, int index, long playtimeThreshold, long date) throws SQLException {
public static void setSelectActivityIndexSQLParameters(PreparedStatement statement, int index, long playtimeThreshold, long date) throws SQLException {
statement.setDouble(index, Math.PI);
statement.setLong(index + 1, playtimeThreshold);

168
Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/AllPlayerContainersQuery.java
View File

@ -1,168 +0,0 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.queries.containers;
import com.djrapitops.plan.delivery.domain.Nickname;
import com.djrapitops.plan.delivery.domain.container.DataContainer;
import com.djrapitops.plan.delivery.domain.container.PerServerContainer;
import com.djrapitops.plan.delivery.domain.container.PlayerContainer;
import com.djrapitops.plan.delivery.domain.container.SupplierDataContainer;
import com.djrapitops.plan.delivery.domain.keys.PerServerKeys;
import com.djrapitops.plan.delivery.domain.keys.PlayerKeys;
import com.djrapitops.plan.delivery.domain.mutators.PerServerMutator;
import com.djrapitops.plan.delivery.domain.mutators.SessionsMutator;
import com.djrapitops.plan.gathering.domain.*;
import com.djrapitops.plan.storage.database.SQLDB;
import com.djrapitops.plan.storage.database.queries.Query;
import com.djrapitops.plan.storage.database.queries.objects.*;
import java.util.*;
/**
* Used to get PlayerContainers of all players on the network, some limitations apply to DataContainer keys.
* <p>
* Limitations:
* - PlayerContainers do not support: PlayerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT
* - PlayerContainers PlayerKeys.PER_SERVER does not support: PerServerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT
* <p>
* Blocking methods are not called until DataContainer getter methods are called.
*
* @author Rsl1122
*/
public class AllPlayerContainersQuery implements Query<List<PlayerContainer>> {
/**
* Create PerServerContainers for each player.
*
* @param sessions Map: Server UUID - Map: Player UUID - List of Sessions
* @param allUserInfo Map: Server UUID - List of Users
* @param allPings Map: Player UUID - List of Ping data
* @return Map: Player UUID - PerServerContainer
*/
private Map<UUID, PerServerContainer> getPerServerData(
Map<UUID, Map<UUID, List<Session>>> sessions,
Map<UUID, List<UserInfo>> allUserInfo,
Map<UUID, List<Ping>> allPings
) {
Map<UUID, PerServerContainer> perServerContainers = new HashMap<>();
for (Map.Entry<UUID, List<UserInfo>> entry : allUserInfo.entrySet()) {
UUID serverUUID = entry.getKey();
List<UserInfo> serverUserInfo = entry.getValue();
for (UserInfo userInfo : serverUserInfo) {
UUID uuid = userInfo.getPlayerUuid();
if (uuid == null) {
continue;
}
PerServerContainer perServerContainer = perServerContainers.getOrDefault(uuid, new PerServerContainer());
DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer());
container.putRawData(PlayerKeys.REGISTERED, userInfo.getRegistered());
container.putRawData(PlayerKeys.BANNED, userInfo.isBanned());
container.putRawData(PlayerKeys.OPERATOR, userInfo.isOperator());
perServerContainer.put(serverUUID, container);
perServerContainers.put(uuid, perServerContainer);
}
}
for (Map.Entry<UUID, Map<UUID, List<Session>>> entry : sessions.entrySet()) {
UUID serverUUID = entry.getKey();
Map<UUID, List<Session>> serverUserSessions = entry.getValue();
for (Map.Entry<UUID, List<Session>> sessionEntry : serverUserSessions.entrySet()) {
UUID playerUUID = sessionEntry.getKey();
PerServerContainer perServerContainer = perServerContainers.getOrDefault(playerUUID, new PerServerContainer());
DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer());
List<Session> serverSessions = sessionEntry.getValue();
container.putRawData(PerServerKeys.SESSIONS, serverSessions);
container.putSupplier(PerServerKeys.LAST_SEEN, () -> SessionsMutator.forContainer(container).toLastSeen());
container.putSupplier(PerServerKeys.WORLD_TIMES, () -> SessionsMutator.forContainer(container).toTotalWorldTimes());
container.putSupplier(PerServerKeys.PLAYER_KILL_COUNT, () -> SessionsMutator.forContainer(container).toPlayerKillCount());
container.putSupplier(PerServerKeys.MOB_KILL_COUNT, () -> SessionsMutator.forContainer(container).toMobKillCount());
container.putSupplier(PerServerKeys.DEATH_COUNT, () -> SessionsMutator.forContainer(container).toDeathCount());
perServerContainer.put(serverUUID, container);
perServerContainers.put(playerUUID, perServerContainer);
}
}
for (Map.Entry<UUID, List<Ping>> entry : allPings.entrySet()) {
UUID uuid = entry.getKey();
for (Ping ping : entry.getValue()) {
UUID serverUUID = ping.getServerUUID();
PerServerContainer perServerContainer = perServerContainers.getOrDefault(uuid, new PerServerContainer());
DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer());
if (!container.supports(PerServerKeys.PING)) {
container.putRawData(PerServerKeys.PING, new ArrayList<>());
}
container.getUnsafe(PerServerKeys.PING).add(ping);
perServerContainer.put(serverUUID, container);
perServerContainers.put(uuid, perServerContainer);
}
}
return perServerContainers;
}
@Override
public List<PlayerContainer> executeQuery(SQLDB db) {
List<PlayerContainer> containers = new ArrayList<>();
Collection<BaseUser> users = db.query(BaseUserQueries.fetchAllBaseUsers());
Map<UUID, List<GeoInfo>> geoInfo = db.query(GeoInfoQueries.fetchAllGeoInformation());
Map<UUID, List<Ping>> allPings = db.query(PingQueries.fetchAllPingData());
Map<UUID, List<Nickname>> allNicknames = db.query(NicknameQueries.fetchAllNicknameDataByPlayerUUIDs());
Map<UUID, Map<UUID, List<Session>>> sessions = db.query(SessionQueries.fetchAllSessionsWithoutKillOrWorldData());
Map<UUID, List<UserInfo>> allUserInfo = db.query(UserInfoQueries.fetchAllUserInformation());
Map<UUID, PerServerContainer> perServerInfo = getPerServerData(sessions, allUserInfo, allPings);
for (BaseUser baseUser : users) {
PlayerContainer container = new PlayerContainer();
UUID uuid = baseUser.getUuid();
container.putRawData(PlayerKeys.UUID, uuid);
container.putRawData(PlayerKeys.REGISTERED, baseUser.getRegistered());
container.putRawData(PlayerKeys.NAME, baseUser.getName());
container.putRawData(PlayerKeys.KICK_COUNT, baseUser.getTimesKicked());
container.putRawData(PlayerKeys.GEO_INFO, geoInfo.get(uuid));
container.putRawData(PlayerKeys.PING, allPings.get(uuid));
container.putRawData(PlayerKeys.NICKNAMES, allNicknames.get(uuid));
container.putRawData(PlayerKeys.PER_SERVER, perServerInfo.get(uuid));
container.putCachingSupplier(PlayerKeys.SESSIONS, () -> {
List<Session> playerSessions = PerServerMutator.forContainer(container).flatMapSessions();
container.getValue(PlayerKeys.ACTIVE_SESSION).ifPresent(playerSessions::add);
return playerSessions;
}
);
// Calculating getters
container.putSupplier(PlayerKeys.LAST_SEEN, () -> SessionsMutator.forContainer(container).toLastSeen());
container.putSupplier(PlayerKeys.MOB_KILL_COUNT, () -> SessionsMutator.forContainer(container).toMobKillCount());
container.putSupplier(PlayerKeys.DEATH_COUNT, () -> SessionsMutator.forContainer(container).toDeathCount());
containers.add(container);
}
return containers;
}
}

16
Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/ContainerFetchQueries.java
View File

@ -21,7 +21,6 @@ import com.djrapitops.plan.delivery.domain.container.PlayerContainer;
import com.djrapitops.plan.delivery.domain.container.ServerContainer;
import com.djrapitops.plan.storage.database.queries.Query;
import java.util.List;
import java.util.UUID;
/**
@ -59,19 +58,4 @@ public class ContainerFetchQueries {
return new PlayerContainerQuery(playerUUID);
}
/**
* Used to get PlayerContainers of all players on the network, some limitations apply to DataContainer keys.
* <p>
* Limitations:
* - PlayerContainers do not support: PlayerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT
* - PlayerContainers PlayerKeys.PER_SERVER does not support: PerServerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT
* <p>
* Blocking methods are not called until DataContainer getter methods are called.
*
* @return a list of PlayerContainers in Plan database.
*/
public static Query<List<PlayerContainer>> fetchAllPlayerContainers() {
return new AllPlayerContainersQuery();
}
}

133
Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/NetworkTablePlayersQuery.java Normal file
View File

@ -0,0 +1,133 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.queries.objects;
import com.djrapitops.plan.delivery.domain.TablePlayer;
import com.djrapitops.plan.delivery.domain.mutators.ActivityIndex;
import com.djrapitops.plan.storage.database.SQLDB;
import com.djrapitops.plan.storage.database.queries.Query;
import com.djrapitops.plan.storage.database.queries.QueryStatement;
import com.djrapitops.plan.storage.database.queries.analysis.NetworkActivityIndexQueries;
import com.djrapitops.plan.storage.database.sql.tables.GeoInfoTable;
import com.djrapitops.plan.storage.database.sql.tables.SessionsTable;
import com.djrapitops.plan.storage.database.sql.tables.UserInfoTable;
import com.djrapitops.plan.storage.database.sql.tables.UsersTable;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static com.djrapitops.plan.storage.database.sql.parsing.Sql.*;
/**
* Query for displaying players on /players page.
*
* @author Rsl1122
*/
public class NetworkTablePlayersQuery implements Query<List<TablePlayer>> {
private final long date;
private final long activeMsThreshold;
private final int xMostRecentPlayers;
public NetworkTablePlayersQuery(long date, long activeMsThreshold, int xMostRecentPlayers) {
this.date = date;
this.activeMsThreshold = activeMsThreshold;
this.xMostRecentPlayers = xMostRecentPlayers;
}
@Override
public List<TablePlayer> executeQuery(SQLDB db) {
String selectGeolocations = SELECT +
GeoInfoTable.USER_UUID + ", " +
GeoInfoTable.GEOLOCATION + ", " +
GeoInfoTable.LAST_USED +
FROM + GeoInfoTable.TABLE_NAME;
String selectLatestGeolocationDate = SELECT +
GeoInfoTable.USER_UUID + ", " +
"MAX(" + GeoInfoTable.LAST_USED + ") as last_used_g" +
FROM + GeoInfoTable.TABLE_NAME +
GROUP_BY + GeoInfoTable.USER_UUID;
String selectLatestGeolocations = SELECT +
"g1." + GeoInfoTable.GEOLOCATION + ',' +
"g1." + GeoInfoTable.USER_UUID +
FROM + "(" + selectGeolocations + ") AS g1" +
INNER_JOIN + "(" + selectLatestGeolocationDate + ") AS g2 ON g1.uuid = g2.uuid" +
WHERE + GeoInfoTable.LAST_USED + "=last_used_g";
String selectSessionData = SELECT + "s." + SessionsTable.USER_UUID + ',' +
"MAX(" + SessionsTable.SESSION_END + ") as last_seen," +
"COUNT(1) as count," +
"SUM(" + SessionsTable.SESSION_END + '-' + SessionsTable.SESSION_START + ") as playtime" +
FROM + SessionsTable.TABLE_NAME + " s" +
GROUP_BY + "s." + SessionsTable.USER_UUID;
String selectBanned = SELECT + DISTINCT + "ub." + UserInfoTable.USER_UUID +
FROM + UserInfoTable.TABLE_NAME + " ub" +
WHERE + UserInfoTable.BANNED + "=?";
String selectBaseUsers = SELECT +
"u." + UsersTable.USER_UUID + ',' +
"u." + UsersTable.USER_NAME + ',' +
"u." + UsersTable.REGISTERED + ',' +
"ban." + UserInfoTable.USER_UUID + " as banned," +
"geoloc." + GeoInfoTable.GEOLOCATION + ',' +
"ses.last_seen," +
"ses.count," +
"ses.playtime," +
"act.activity_index" +
FROM + UsersTable.TABLE_NAME + " u" +
LEFT_JOIN + '(' + selectBanned + ") ban on ban." + UserInfoTable.USER_UUID + "=u." + UsersTable.USER_UUID +
LEFT_JOIN + '(' + selectLatestGeolocations + ") geoloc on geoloc." + GeoInfoTable.USER_UUID + "=u." + UsersTable.USER_UUID +
LEFT_JOIN + '(' + selectSessionData + ") ses on ses." + SessionsTable.USER_UUID + "=u." + UsersTable.USER_UUID +
LEFT_JOIN + '(' + NetworkActivityIndexQueries.selectActivityIndexSQL() + ") act on u." + UsersTable.USER_UUID + "=act." + UserInfoTable.USER_UUID +
ORDER_BY + "ses.last_seen DESC LIMIT ?";
return db.query(new QueryStatement<List<TablePlayer>>(selectBaseUsers, 1000) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setBoolean(1, true);
NetworkActivityIndexQueries.setSelectActivityIndexSQLParameters(statement, 2, activeMsThreshold, date);
statement.setInt(10, xMostRecentPlayers);
}
@Override
public List<TablePlayer> processResults(ResultSet set) throws SQLException {
List<TablePlayer> players = new ArrayList<>();
while (set.next()) {
TablePlayer.Builder player = TablePlayer.builder()
.uuid(UUID.fromString(set.getString(UsersTable.USER_UUID)))
.name(set.getString(UsersTable.USER_NAME))
.geolocation(set.getString(GeoInfoTable.GEOLOCATION))
.registered(set.getLong(UsersTable.REGISTERED))
.lastSeen(set.getLong("last_seen"))
.sessionCount(set.getInt("count"))
.playtime(set.getLong("playtime"))
.activityIndex(new ActivityIndex(set.getDouble("activity_index"), date));
if (set.getString("banned") != null) {
player.banned();
}
players.add(player.build());
}
return players;
}
});
}
}

26
Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java
View File

@ -51,7 +51,6 @@ import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.locale.Locale;
import com.djrapitops.plan.storage.database.queries.*;
import com.djrapitops.plan.storage.database.queries.analysis.ActivityIndexQueries;
import com.djrapitops.plan.storage.database.queries.containers.AllPlayerContainersQuery;
import com.djrapitops.plan.storage.database.queries.containers.ContainerFetchQueries;
import com.djrapitops.plan.storage.database.queries.containers.ServerPlayerContainersQuery;
import com.djrapitops.plan.storage.database.queries.objects.*;
@ -1080,23 +1079,6 @@ public interface DatabaseTest {
assertEquals(expected, result);
}
@Test
default void allPlayerContainersQueryDoesNotReturnDuplicatePlayers() {
db().executeTransaction(TestData.storeServers());
executeTransactions(TestData.storePlayerOneData());
executeTransactions(TestData.storePlayerTwoData());
List<UUID> expected = Arrays.asList(playerUUID, player2UUID);
Collections.sort(expected);
Collection<UUID> result = db().query(new AllPlayerContainersQuery())
.stream().map(player -> player.getUnsafe(PlayerKeys.UUID))
.sorted()
.collect(Collectors.toList());
assertEquals(expected, result);
}
@Test
default void sqlDateConversionSanityCheck() {
Database db = db();
@ -1423,6 +1405,14 @@ public interface DatabaseTest {
assertNotEquals(Collections.emptyList(), result);
}
@Test
default void networkTablePlayersQueryQueriesAtLeastOnePlayer() {
sessionsAreStoredWithAllData();
List<TablePlayer> result = db().query(new NetworkTablePlayersQuery(System.currentTimeMillis(), 10L, 1));
assertNotEquals(Collections.emptyList(), result);
}
@PluginInfo(name = "ConditionalExtension")
class ConditionalExtension implements DataExtension {