From e18ec134f66434eb1ca6f6753ffc7956c19779ff Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Sun, 15 Sep 2019 13:24:39 +0300 Subject: [PATCH] Optimized (memory) /players page table query --- .../delivery/rendering/json/JSONFactory.java | 2 +- .../analysis/NetworkActivityIndexQueries.java | 4 +- .../containers/AllPlayerContainersQuery.java | 168 ------------------ .../containers/ContainerFetchQueries.java | 16 -- .../objects/NetworkTablePlayersQuery.java | 133 ++++++++++++++ .../plan/storage/database/DatabaseTest.java | 26 +-- 6 files changed, 144 insertions(+), 205 deletions(-) delete mode 100644 Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/AllPlayerContainersQuery.java create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/NetworkTablePlayersQuery.java diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java index 6ec9d9094..74875870a 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java @@ -97,7 +97,7 @@ public class JSONFactory { Database database = dbSystem.getDatabase(); return new PlayersTableJSONParser( - Collections.emptyList(),// TODO Replace with new query + database.query(new NetworkTablePlayersQuery(System.currentTimeMillis(), playtimeThreshold, xMostRecentPlayers)), Collections.emptyMap(), openPlayerLinksInNewTab, formatters diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/analysis/NetworkActivityIndexQueries.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/analysis/NetworkActivityIndexQueries.java index 735ebf189..a149c9730 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/analysis/NetworkActivityIndexQueries.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/analysis/NetworkActivityIndexQueries.java @@ -76,7 +76,7 @@ public class NetworkActivityIndexQueries { return fetchActivityGroupCount(date, playtimeThreshold, ActivityIndex.REGULAR, 5.1); } - private static String selectActivityIndexSQL() { + public static String selectActivityIndexSQL() { String selectActivePlaytimeSQL = SELECT + SessionsTable.USER_UUID + ",SUM(" + @@ -96,7 +96,7 @@ public class NetworkActivityIndexQueries { GROUP_BY + "q1." + SessionsTable.USER_UUID; } - private static void setSelectActivityIndexSQLParameters(PreparedStatement statement, int index, long playtimeThreshold, long date) throws SQLException { + public static void setSelectActivityIndexSQLParameters(PreparedStatement statement, int index, long playtimeThreshold, long date) throws SQLException { statement.setDouble(index, Math.PI); statement.setLong(index + 1, playtimeThreshold); diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/AllPlayerContainersQuery.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/AllPlayerContainersQuery.java deleted file mode 100644 index 5aeeacbf1..000000000 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/AllPlayerContainersQuery.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * This file is part of Player Analytics (Plan). - * - * Plan is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License v3 as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Plan is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Plan. If not, see . - */ -package com.djrapitops.plan.storage.database.queries.containers; - -import com.djrapitops.plan.delivery.domain.Nickname; -import com.djrapitops.plan.delivery.domain.container.DataContainer; -import com.djrapitops.plan.delivery.domain.container.PerServerContainer; -import com.djrapitops.plan.delivery.domain.container.PlayerContainer; -import com.djrapitops.plan.delivery.domain.container.SupplierDataContainer; -import com.djrapitops.plan.delivery.domain.keys.PerServerKeys; -import com.djrapitops.plan.delivery.domain.keys.PlayerKeys; -import com.djrapitops.plan.delivery.domain.mutators.PerServerMutator; -import com.djrapitops.plan.delivery.domain.mutators.SessionsMutator; -import com.djrapitops.plan.gathering.domain.*; -import com.djrapitops.plan.storage.database.SQLDB; -import com.djrapitops.plan.storage.database.queries.Query; -import com.djrapitops.plan.storage.database.queries.objects.*; - -import java.util.*; - -/** - * Used to get PlayerContainers of all players on the network, some limitations apply to DataContainer keys. - *

- * Limitations: - * - PlayerContainers do not support: PlayerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT - * - PlayerContainers PlayerKeys.PER_SERVER does not support: PerServerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT - *

- * Blocking methods are not called until DataContainer getter methods are called. - * - * @author Rsl1122 - */ -public class AllPlayerContainersQuery implements Query> { - - /** - * Create PerServerContainers for each player. - * - * @param sessions Map: Server UUID - Map: Player UUID - List of Sessions - * @param allUserInfo Map: Server UUID - List of Users - * @param allPings Map: Player UUID - List of Ping data - * @return Map: Player UUID - PerServerContainer - */ - private Map getPerServerData( - Map>> sessions, - Map> allUserInfo, - Map> allPings - ) { - Map perServerContainers = new HashMap<>(); - - for (Map.Entry> entry : allUserInfo.entrySet()) { - UUID serverUUID = entry.getKey(); - List serverUserInfo = entry.getValue(); - - for (UserInfo userInfo : serverUserInfo) { - UUID uuid = userInfo.getPlayerUuid(); - if (uuid == null) { - continue; - } - PerServerContainer perServerContainer = perServerContainers.getOrDefault(uuid, new PerServerContainer()); - DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer()); - container.putRawData(PlayerKeys.REGISTERED, userInfo.getRegistered()); - container.putRawData(PlayerKeys.BANNED, userInfo.isBanned()); - container.putRawData(PlayerKeys.OPERATOR, userInfo.isOperator()); - perServerContainer.put(serverUUID, container); - perServerContainers.put(uuid, perServerContainer); - } - } - - for (Map.Entry>> entry : sessions.entrySet()) { - UUID serverUUID = entry.getKey(); - Map> serverUserSessions = entry.getValue(); - - for (Map.Entry> sessionEntry : serverUserSessions.entrySet()) { - UUID playerUUID = sessionEntry.getKey(); - PerServerContainer perServerContainer = perServerContainers.getOrDefault(playerUUID, new PerServerContainer()); - DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer()); - - List serverSessions = sessionEntry.getValue(); - container.putRawData(PerServerKeys.SESSIONS, serverSessions); - - container.putSupplier(PerServerKeys.LAST_SEEN, () -> SessionsMutator.forContainer(container).toLastSeen()); - - container.putSupplier(PerServerKeys.WORLD_TIMES, () -> SessionsMutator.forContainer(container).toTotalWorldTimes()); - container.putSupplier(PerServerKeys.PLAYER_KILL_COUNT, () -> SessionsMutator.forContainer(container).toPlayerKillCount()); - container.putSupplier(PerServerKeys.MOB_KILL_COUNT, () -> SessionsMutator.forContainer(container).toMobKillCount()); - container.putSupplier(PerServerKeys.DEATH_COUNT, () -> SessionsMutator.forContainer(container).toDeathCount()); - perServerContainer.put(serverUUID, container); - perServerContainers.put(playerUUID, perServerContainer); - } - } - - for (Map.Entry> entry : allPings.entrySet()) { - UUID uuid = entry.getKey(); - for (Ping ping : entry.getValue()) { - UUID serverUUID = ping.getServerUUID(); - PerServerContainer perServerContainer = perServerContainers.getOrDefault(uuid, new PerServerContainer()); - DataContainer container = perServerContainer.getOrDefault(serverUUID, new SupplierDataContainer()); - - if (!container.supports(PerServerKeys.PING)) { - container.putRawData(PerServerKeys.PING, new ArrayList<>()); - } - container.getUnsafe(PerServerKeys.PING).add(ping); - - perServerContainer.put(serverUUID, container); - perServerContainers.put(uuid, perServerContainer); - } - } - - return perServerContainers; - } - - @Override - public List executeQuery(SQLDB db) { - List containers = new ArrayList<>(); - - Collection users = db.query(BaseUserQueries.fetchAllBaseUsers()); - Map> geoInfo = db.query(GeoInfoQueries.fetchAllGeoInformation()); - Map> allPings = db.query(PingQueries.fetchAllPingData()); - Map> allNicknames = db.query(NicknameQueries.fetchAllNicknameDataByPlayerUUIDs()); - - Map>> sessions = db.query(SessionQueries.fetchAllSessionsWithoutKillOrWorldData()); - Map> allUserInfo = db.query(UserInfoQueries.fetchAllUserInformation()); - Map perServerInfo = getPerServerData(sessions, allUserInfo, allPings); - - for (BaseUser baseUser : users) { - PlayerContainer container = new PlayerContainer(); - UUID uuid = baseUser.getUuid(); - container.putRawData(PlayerKeys.UUID, uuid); - - container.putRawData(PlayerKeys.REGISTERED, baseUser.getRegistered()); - container.putRawData(PlayerKeys.NAME, baseUser.getName()); - container.putRawData(PlayerKeys.KICK_COUNT, baseUser.getTimesKicked()); - container.putRawData(PlayerKeys.GEO_INFO, geoInfo.get(uuid)); - container.putRawData(PlayerKeys.PING, allPings.get(uuid)); - container.putRawData(PlayerKeys.NICKNAMES, allNicknames.get(uuid)); - container.putRawData(PlayerKeys.PER_SERVER, perServerInfo.get(uuid)); - - container.putCachingSupplier(PlayerKeys.SESSIONS, () -> { - List playerSessions = PerServerMutator.forContainer(container).flatMapSessions(); - container.getValue(PlayerKeys.ACTIVE_SESSION).ifPresent(playerSessions::add); - return playerSessions; - } - ); - - // Calculating getters - container.putSupplier(PlayerKeys.LAST_SEEN, () -> SessionsMutator.forContainer(container).toLastSeen()); - - container.putSupplier(PlayerKeys.MOB_KILL_COUNT, () -> SessionsMutator.forContainer(container).toMobKillCount()); - container.putSupplier(PlayerKeys.DEATH_COUNT, () -> SessionsMutator.forContainer(container).toDeathCount()); - - containers.add(container); - } - return containers; - } -} \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/ContainerFetchQueries.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/ContainerFetchQueries.java index 7a7781c05..599e493f8 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/ContainerFetchQueries.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/containers/ContainerFetchQueries.java @@ -21,7 +21,6 @@ import com.djrapitops.plan.delivery.domain.container.PlayerContainer; import com.djrapitops.plan.delivery.domain.container.ServerContainer; import com.djrapitops.plan.storage.database.queries.Query; -import java.util.List; import java.util.UUID; /** @@ -59,19 +58,4 @@ public class ContainerFetchQueries { return new PlayerContainerQuery(playerUUID); } - /** - * Used to get PlayerContainers of all players on the network, some limitations apply to DataContainer keys. - *

- * Limitations: - * - PlayerContainers do not support: PlayerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT - * - PlayerContainers PlayerKeys.PER_SERVER does not support: PerServerKeys WORLD_TIMES, PLAYER_KILLS, PLAYER_KILL_COUNT - *

- * Blocking methods are not called until DataContainer getter methods are called. - * - * @return a list of PlayerContainers in Plan database. - */ - public static Query> fetchAllPlayerContainers() { - return new AllPlayerContainersQuery(); - } - } \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/NetworkTablePlayersQuery.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/NetworkTablePlayersQuery.java new file mode 100644 index 000000000..e65d6aa1e --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/NetworkTablePlayersQuery.java @@ -0,0 +1,133 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.storage.database.queries.objects; + +import com.djrapitops.plan.delivery.domain.TablePlayer; +import com.djrapitops.plan.delivery.domain.mutators.ActivityIndex; +import com.djrapitops.plan.storage.database.SQLDB; +import com.djrapitops.plan.storage.database.queries.Query; +import com.djrapitops.plan.storage.database.queries.QueryStatement; +import com.djrapitops.plan.storage.database.queries.analysis.NetworkActivityIndexQueries; +import com.djrapitops.plan.storage.database.sql.tables.GeoInfoTable; +import com.djrapitops.plan.storage.database.sql.tables.SessionsTable; +import com.djrapitops.plan.storage.database.sql.tables.UserInfoTable; +import com.djrapitops.plan.storage.database.sql.tables.UsersTable; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static com.djrapitops.plan.storage.database.sql.parsing.Sql.*; + +/** + * Query for displaying players on /players page. + * + * @author Rsl1122 + */ +public class NetworkTablePlayersQuery implements Query> { + + private final long date; + private final long activeMsThreshold; + private final int xMostRecentPlayers; + + public NetworkTablePlayersQuery(long date, long activeMsThreshold, int xMostRecentPlayers) { + this.date = date; + this.activeMsThreshold = activeMsThreshold; + this.xMostRecentPlayers = xMostRecentPlayers; + } + + @Override + public List executeQuery(SQLDB db) { + String selectGeolocations = SELECT + + GeoInfoTable.USER_UUID + ", " + + GeoInfoTable.GEOLOCATION + ", " + + GeoInfoTable.LAST_USED + + FROM + GeoInfoTable.TABLE_NAME; + String selectLatestGeolocationDate = SELECT + + GeoInfoTable.USER_UUID + ", " + + "MAX(" + GeoInfoTable.LAST_USED + ") as last_used_g" + + FROM + GeoInfoTable.TABLE_NAME + + GROUP_BY + GeoInfoTable.USER_UUID; + String selectLatestGeolocations = SELECT + + "g1." + GeoInfoTable.GEOLOCATION + ',' + + "g1." + GeoInfoTable.USER_UUID + + FROM + "(" + selectGeolocations + ") AS g1" + + INNER_JOIN + "(" + selectLatestGeolocationDate + ") AS g2 ON g1.uuid = g2.uuid" + + WHERE + GeoInfoTable.LAST_USED + "=last_used_g"; + + String selectSessionData = SELECT + "s." + SessionsTable.USER_UUID + ',' + + "MAX(" + SessionsTable.SESSION_END + ") as last_seen," + + "COUNT(1) as count," + + "SUM(" + SessionsTable.SESSION_END + '-' + SessionsTable.SESSION_START + ") as playtime" + + FROM + SessionsTable.TABLE_NAME + " s" + + GROUP_BY + "s." + SessionsTable.USER_UUID; + + String selectBanned = SELECT + DISTINCT + "ub." + UserInfoTable.USER_UUID + + FROM + UserInfoTable.TABLE_NAME + " ub" + + WHERE + UserInfoTable.BANNED + "=?"; + + String selectBaseUsers = SELECT + + "u." + UsersTable.USER_UUID + ',' + + "u." + UsersTable.USER_NAME + ',' + + "u." + UsersTable.REGISTERED + ',' + + "ban." + UserInfoTable.USER_UUID + " as banned," + + "geoloc." + GeoInfoTable.GEOLOCATION + ',' + + "ses.last_seen," + + "ses.count," + + "ses.playtime," + + "act.activity_index" + + FROM + UsersTable.TABLE_NAME + " u" + + LEFT_JOIN + '(' + selectBanned + ") ban on ban." + UserInfoTable.USER_UUID + "=u." + UsersTable.USER_UUID + + LEFT_JOIN + '(' + selectLatestGeolocations + ") geoloc on geoloc." + GeoInfoTable.USER_UUID + "=u." + UsersTable.USER_UUID + + LEFT_JOIN + '(' + selectSessionData + ") ses on ses." + SessionsTable.USER_UUID + "=u." + UsersTable.USER_UUID + + LEFT_JOIN + '(' + NetworkActivityIndexQueries.selectActivityIndexSQL() + ") act on u." + UsersTable.USER_UUID + "=act." + UserInfoTable.USER_UUID + + ORDER_BY + "ses.last_seen DESC LIMIT ?"; + + return db.query(new QueryStatement>(selectBaseUsers, 1000) { + @Override + public void prepare(PreparedStatement statement) throws SQLException { + statement.setBoolean(1, true); + NetworkActivityIndexQueries.setSelectActivityIndexSQLParameters(statement, 2, activeMsThreshold, date); + statement.setInt(10, xMostRecentPlayers); + } + + @Override + public List processResults(ResultSet set) throws SQLException { + List players = new ArrayList<>(); + while (set.next()) { + TablePlayer.Builder player = TablePlayer.builder() + .uuid(UUID.fromString(set.getString(UsersTable.USER_UUID))) + .name(set.getString(UsersTable.USER_NAME)) + .geolocation(set.getString(GeoInfoTable.GEOLOCATION)) + .registered(set.getLong(UsersTable.REGISTERED)) + .lastSeen(set.getLong("last_seen")) + .sessionCount(set.getInt("count")) + .playtime(set.getLong("playtime")) + .activityIndex(new ActivityIndex(set.getDouble("activity_index"), date)); + if (set.getString("banned") != null) { + player.banned(); + } + players.add(player.build()); + } + return players; + } + }); + } +} \ No newline at end of file diff --git a/Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java b/Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java index 5b75f7462..ad577efc1 100644 --- a/Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java +++ b/Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java @@ -51,7 +51,6 @@ import com.djrapitops.plan.settings.config.PlanConfig; import com.djrapitops.plan.settings.locale.Locale; import com.djrapitops.plan.storage.database.queries.*; import com.djrapitops.plan.storage.database.queries.analysis.ActivityIndexQueries; -import com.djrapitops.plan.storage.database.queries.containers.AllPlayerContainersQuery; import com.djrapitops.plan.storage.database.queries.containers.ContainerFetchQueries; import com.djrapitops.plan.storage.database.queries.containers.ServerPlayerContainersQuery; import com.djrapitops.plan.storage.database.queries.objects.*; @@ -1080,23 +1079,6 @@ public interface DatabaseTest { assertEquals(expected, result); } - @Test - default void allPlayerContainersQueryDoesNotReturnDuplicatePlayers() { - db().executeTransaction(TestData.storeServers()); - executeTransactions(TestData.storePlayerOneData()); - executeTransactions(TestData.storePlayerTwoData()); - - List expected = Arrays.asList(playerUUID, player2UUID); - Collections.sort(expected); - - Collection result = db().query(new AllPlayerContainersQuery()) - .stream().map(player -> player.getUnsafe(PlayerKeys.UUID)) - .sorted() - .collect(Collectors.toList()); - - assertEquals(expected, result); - } - @Test default void sqlDateConversionSanityCheck() { Database db = db(); @@ -1423,6 +1405,14 @@ public interface DatabaseTest { assertNotEquals(Collections.emptyList(), result); } + @Test + default void networkTablePlayersQueryQueriesAtLeastOnePlayer() { + sessionsAreStoredWithAllData(); + + List result = db().query(new NetworkTablePlayersQuery(System.currentTimeMillis(), 10L, 1)); + assertNotEquals(Collections.emptyList(), result); + } + @PluginInfo(name = "ConditionalExtension") class ConditionalExtension implements DataExtension {