Upstream/LuckPerms
1
0
Fork 0
mirror of https://github.com/LuckPerms/LuckPerms.git synced 2024-11-24 03:25:19 +01:00
Code Issues Projects Releases Wiki Activity

Optimize bulk user loading for editor command (#3273)

Browse Source
This commit is contained in:
Luck 2022-02-08 22:45:00 +00:00
parent 1b3b9b5c62
commit bc15e348f5
No known key found for this signature in database
GPG Key ID: EFA9B3EC5FD90F8B
7 changed files with 138 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
common/src/main/java/me/lucko/luckperms/common/storage/Storage.java
View File

@ -154,6 +154,16 @@ public class Storage {
});
}
public CompletableFuture<Map<UUID, User>> loadUsers(Set<UUID> uniqueIds) {
return future(() -> {
Map<UUID, User> users = this.implementation.loadUsers(uniqueIds);
for (User user : users.values()) {
this.plugin.getEventDispatcher().dispatchUserLoad(user);
}
return users;
});
}
public CompletableFuture<Void> saveUser(User user) {
return future(() -> this.implementation.saveUser(user));
}

2
common/src/main/java/me/lucko/luckperms/common/storage/implementation/StorageImplementation.java
View File

@ -69,6 +69,8 @@ public interface StorageImplementation {
User loadUser(UUID uniqueId, String username) throws Exception;
Map<UUID, User> loadUsers(Set<UUID> uniqueIds) throws Exception;
void saveUser(User user) throws Exception;
Set<UUID> getUniqueUsers() throws Exception;

11
common/src/main/java/me/lucko/luckperms/common/storage/implementation/file/AbstractConfigurateStorage.java
View File

@ -67,6 +67,7 @@ import java.nio.file.Path;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
@ -200,6 +201,16 @@ public abstract class AbstractConfigurateStorage implements StorageImplementatio
return user;
}
@Override
public Map<UUID, User> loadUsers(Set<UUID> uniqueIds) throws Exception {
// add multithreading here?
Map<UUID, User> map = new HashMap<>();
for (UUID uniqueId : uniqueIds) {
map.put(uniqueId, loadUser(uniqueId, null));
}
return map;
}
@Override
public void saveUser(User user) throws IOException {
user.normalData().discardChanges();

11
common/src/main/java/me/lucko/luckperms/common/storage/implementation/mongodb/MongoStorage.java
View File

@ -72,6 +72,7 @@ import org.bson.Document;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@ -328,6 +329,16 @@ public class MongoStorage implements StorageImplementation {
return user;
}
@Override
public Map<UUID, User> loadUsers(Set<UUID> uniqueIds) throws Exception {
// make this a bulk search?
Map<UUID, User> map = new HashMap<>();
for (UUID uniqueId : uniqueIds) {
map.put(uniqueId, loadUser(uniqueId, null));
}
return map;
}
@Override
public void saveUser(User user) {
MongoCollection<Document> c = this.database.getCollection(this.prefix + "users");

5
common/src/main/java/me/lucko/luckperms/common/storage/implementation/split/SplitStorage.java
View File

@ -152,6 +152,11 @@ public class SplitStorage implements StorageImplementation {
return implFor(SplitStorageType.USER).loadUser(uniqueId, username);
}
@Override
public Map<UUID, User> loadUsers(Set<UUID> uniqueIds) throws Exception {
return implFor(SplitStorageType.USER).loadUsers(uniqueIds);
}
@Override
public void saveUser(User user) throws Exception {
implFor(SplitStorageType.USER).saveUser(user);

86
common/src/main/java/me/lucko/luckperms/common/storage/implementation/sql/SqlStorage.java
View File

@ -84,6 +84,7 @@ public class SqlStorage implements StorageImplementation {
private static final Type LIST_STRING_TYPE = new TypeToken<List<String>>(){}.getType();
private static final String USER_PERMISSIONS_SELECT = "SELECT id, permission, value, server, world, expiry, contexts FROM '{prefix}user_permissions' WHERE uuid=?";
private static final String USER_PERMISSIONS_SELECT_MULTIPLE = "SELECT uuid, id, permission, value, server, world, expiry, contexts FROM '{prefix}user_permissions' WHERE ";
private static final String USER_PERMISSIONS_DELETE_SPECIFIC = "DELETE FROM '{prefix}user_permissions' WHERE id=?";
private static final String USER_PERMISSIONS_DELETE_SPECIFIC_PROPS = "DELETE FROM '{prefix}user_permissions' WHERE uuid=? AND permission=? AND value=? AND server=? AND world=? AND expiry=? AND contexts=?";
private static final String USER_PERMISSIONS_DELETE = "DELETE FROM '{prefix}user_permissions' WHERE uuid=?";
@ -99,6 +100,7 @@ public class SqlStorage implements StorageImplementation {
private static final String PLAYER_SELECT_ALL_UUIDS_BY_USERNAME = "SELECT uuid FROM '{prefix}players' WHERE username=? AND NOT uuid=?";
private static final String PLAYER_DELETE_ALL_UUIDS_BY_USERNAME = "DELETE FROM '{prefix}players' WHERE username=? AND NOT uuid=?";
private static final String PLAYER_SELECT_BY_UUID = "SELECT username, primary_group FROM '{prefix}players' WHERE uuid=? LIMIT 1";
private static final String PLAYER_SELECT_BY_UUID_MULTIPLE = "SELECT uuid, username, primary_group FROM '{prefix}players' WHERE ";
private static final String PLAYER_SELECT_PRIMARY_GROUP_BY_UUID = "SELECT primary_group FROM '{prefix}players' WHERE uuid=? LIMIT 1";
private static final String PLAYER_UPDATE_PRIMARY_GROUP_BY_UUID = "UPDATE '{prefix}players' SET primary_group=? WHERE uuid=?";
@ -319,16 +321,38 @@ public class SqlStorage implements StorageImplementation {
@Override
public User loadUser(UUID uniqueId, String username) throws SQLException {
User user = this.plugin.getUserManager().getOrMake(uniqueId, username);
List<Node> nodes;
SqlPlayerData playerData;
try (Connection c = this.connectionFactory.getConnection()) {
nodes = selectUserPermissions(c, user.getUniqueId());
playerData = selectPlayerData(c, user.getUniqueId());
nodes = selectUserPermissions(c, uniqueId);
playerData = selectPlayerData(c, uniqueId);
}
return createUser(uniqueId, username, playerData, nodes, true);
}
@Override
public Map<UUID, User> loadUsers(Set<UUID> uniqueIds) throws Exception {
Map<UUID, List<Node>> nodesMap;
Map<UUID, SqlPlayerData> playerDataMap;
try (Connection c = this.connectionFactory.getConnection()) {
nodesMap = selectUserPermissions(c, uniqueIds);
playerDataMap = selectPlayerData(c, uniqueIds);
}
Map<UUID, User> users = new HashMap<>();
for (UUID uniqueId : uniqueIds) {
SqlPlayerData playerData = playerDataMap.get(uniqueId);
List<Node> nodes = nodesMap.get(uniqueId);
users.put(uniqueId, createUser(uniqueId, null, playerData, nodes, false));
}
return users;
}
private User createUser(UUID uniqueId, String username, SqlPlayerData playerData, List<Node> nodes, boolean saveAfterAudit) throws SQLException {
User user = this.plugin.getUserManager().getOrMake(uniqueId, username);
if (playerData != null) {
if (playerData.primaryGroup != null) {
user.getPrimaryGroup().setStoredValue(playerData.primaryGroup);
@ -342,7 +366,7 @@ public class SqlStorage implements StorageImplementation {
user.loadNodesFromStorage(nodes);
this.plugin.getUserManager().giveDefaultIfNeeded(user);
if (user.auditTemporaryNodes()) {
if (user.auditTemporaryNodes() && saveAfterAudit) {
saveUser(user);
}
@ -866,6 +890,58 @@ public class SqlStorage implements StorageImplementation {
}
}
private Map<UUID, List<Node>> selectUserPermissions(Connection c, Set<UUID> users) throws SQLException {
Map<UUID, List<Node>> map = new HashMap<>();
for (UUID uuid : users) {
map.put(uuid, new ArrayList<>());
}
try (Statement s = c.createStatement()) {
String sql = createUserSelectWhereClause(USER_PERMISSIONS_SELECT_MULTIPLE, users);
try (ResultSet rs = s.executeQuery(sql)) {
while (rs.next()) {
UUID uuid = UUID.fromString(rs.getString("uuid"));
Node node = readNode(rs);
if (node != null) {
map.get(uuid).add(readNode(rs));
}
}
}
}
return map;
}
private Map<UUID, SqlPlayerData> selectPlayerData(Connection c, Set<UUID> users) throws SQLException {
Map<UUID, SqlPlayerData> map = new HashMap<>();
try (Statement s = c.createStatement()) {
String sql = createUserSelectWhereClause(PLAYER_SELECT_BY_UUID_MULTIPLE, users);
try (ResultSet rs = s.executeQuery(sql)) {
while (rs.next()) {
UUID uuid = UUID.fromString(rs.getString("uuid"));
SqlPlayerData data = new SqlPlayerData(
rs.getString("primary_group"),
rs.getString("username")
);
map.put(uuid, data);
}
}
}
return map;
}
private String createUserSelectWhereClause(String baseQuery, Set<UUID> users) {
String param = users.stream()
.map(uuid -> "'" + uuid + "'")
.collect(Collectors.joining(",", "uuid IN (", ")"));
// we don't want to use preparedstatements because the parameter length is variable
// safe to do string concat/replacement because the UUID.toString value isn't injectable
return this.statementProcessor.apply(baseQuery) + param;
}
private void deleteUser(Connection c, UUID user) throws SQLException {
try (PreparedStatement ps = c.prepareStatement(this.statementProcessor.apply(USER_PERMISSIONS_DELETE))) {
ps.setString(1, user.toString());

27
common/src/main/java/me/lucko/luckperms/common/webeditor/WebEditorRequest.java
View File

@ -61,9 +61,11 @@ import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;
@ -257,17 +259,24 @@ public class WebEditorRequest {
.distinct();
}
stream.filter(uuid -> !users.containsKey(uuid))
Set<UUID> uuids = stream
.filter(uuid -> !users.containsKey(uuid))
.sorted()
.limit(MAX_USERS - users.size())
.map(uuid -> plugin.getStorage().loadUser(uuid, null))
.forEach(fut -> {
User user = fut.join();
if (user != null) {
users.put(user.getUniqueId(), user);
plugin.getUserManager().getHouseKeeper().cleanup(user.getUniqueId());
}
});
.collect(Collectors.toSet());
if (uuids.isEmpty()) {
return;
}
// load users in bulk from storage
Map<UUID, User> loadedUsers = plugin.getStorage().loadUsers(uuids).join();
users.putAll(loadedUsers);
// schedule cleanup
for (UUID uniqueId : loadedUsers.keySet()) {
plugin.getUserManager().getHouseKeeper().cleanup(uniqueId);
}
}
}