Upstream/LuckPerms
1
0
Fork 0
mirror of https://github.com/LuckPerms/LuckPerms.git synced 2024-11-30 22:53:27 +01:00
Code Issues Projects Releases Wiki Activity

Run exporter in own thread and process user exports concurrently

Browse Source
This commit is contained in:
Luck 2017-03-10 20:44:19 +00:00
parent bb818c8942
commit 2462eac497
No known key found for this signature in database
GPG Key ID: EFA9B3EC5FD90F8B
14 changed files with 360 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bukkit/src/main/java/me/lucko/luckperms/bukkit/LPBukkitPlugin.java
View File

@ -49,7 +49,6 @@ import me.lucko.luckperms.common.contexts.ContextManager;
import me.lucko.luckperms.common.contexts.ServerCalculator; import me.lucko.luckperms.common.contexts.ServerCalculator;
import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.UuidCache;
import me.lucko.luckperms.common.core.model.User; import me.lucko.luckperms.common.core.model.User;
import me.lucko.luckperms.common.data.Importer;
import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.debug.DebugHandler;
import me.lucko.luckperms.common.dependencies.DependencyManager; import me.lucko.luckperms.common.dependencies.DependencyManager;
import me.lucko.luckperms.common.locale.LocaleManager; import me.lucko.luckperms.common.locale.LocaleManager;
@ -112,7 +111,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin {
private BukkitListener listener; private BukkitListener listener;
private ApiProvider apiProvider; private ApiProvider apiProvider;
private Logger log; private Logger log;
private Importer importer;
private DefaultsProvider defaultsProvider; private DefaultsProvider defaultsProvider;
private ChildPermissionProvider childPermissionProvider; private ChildPermissionProvider childPermissionProvider;
private LocaleManager localeManager; private LocaleManager localeManager;
@ -247,7 +245,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin {
userManager = new GenericUserManager(this); userManager = new GenericUserManager(this);
groupManager = new GenericGroupManager(this); groupManager = new GenericGroupManager(this);
trackManager = new GenericTrackManager(this); trackManager = new GenericTrackManager(this);
importer = new Importer(commandManager);
calculatorFactory = new BukkitCalculatorFactory(this); calculatorFactory = new BukkitCalculatorFactory(this);
cachedStateManager = new CachedStateManager(this); cachedStateManager = new CachedStateManager(this);
@ -371,7 +368,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin {
listener = null; listener = null;
apiProvider = null; apiProvider = null;
log = null; log = null;
importer = null;
defaultsProvider = null; defaultsProvider = null;
childPermissionProvider = null; childPermissionProvider = null;
localeManager = null; localeManager = null;

2
bukkit/src/main/java/me/lucko/luckperms/bukkit/migration/MigrationPowerfulPerms.java
View File

@ -111,7 +111,7 @@ public class MigrationPowerfulPerms extends SubCommand<Object> {
Set<UUID> uuids = new HashSet<>(); Set<UUID> uuids = new HashSet<>();
try (HikariSupplier hikari = new HikariSupplier(address, database, username, password)) { try (HikariSupplier hikari = new HikariSupplier(address, database, username, password)) {
hikari.setup(); hikari.setup("powerfulperms-migrator-pool");
try (Connection c = hikari.getConnection()) { try (Connection c = hikari.getConnection()) {
DatabaseMetaData meta = c.getMetaData(); DatabaseMetaData meta = c.getMetaData();

6
bungee/src/main/java/me/lucko/luckperms/bungee/LPBungeePlugin.java
View File

@ -42,7 +42,6 @@ import me.lucko.luckperms.common.contexts.ContextManager;
import me.lucko.luckperms.common.contexts.ServerCalculator; import me.lucko.luckperms.common.contexts.ServerCalculator;
import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.UuidCache;
import me.lucko.luckperms.common.core.model.User; import me.lucko.luckperms.common.core.model.User;
import me.lucko.luckperms.common.data.Importer;
import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.debug.DebugHandler;
import me.lucko.luckperms.common.dependencies.DependencyManager; import me.lucko.luckperms.common.dependencies.DependencyManager;
import me.lucko.luckperms.common.locale.LocaleManager; import me.lucko.luckperms.common.locale.LocaleManager;
@ -85,6 +84,7 @@ import java.util.stream.Collectors;
public class LPBungeePlugin extends Plugin implements LuckPermsPlugin { public class LPBungeePlugin extends Plugin implements LuckPermsPlugin {
private final Set<UUID> ignoringLogs = ConcurrentHashMap.newKeySet(); private final Set<UUID> ignoringLogs = ConcurrentHashMap.newKeySet();
private LuckPermsScheduler scheduler; private LuckPermsScheduler scheduler;
private CommandManager commandManager;
private LuckPermsConfiguration configuration; private LuckPermsConfiguration configuration;
private UserManager userManager; private UserManager userManager;
private GroupManager groupManager; private GroupManager groupManager;
@ -94,7 +94,6 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin {
private UuidCache uuidCache; private UuidCache uuidCache;
private ApiProvider apiProvider; private ApiProvider apiProvider;
private Logger log; private Logger log;
private Importer importer;
private LocaleManager localeManager; private LocaleManager localeManager;
private CachedStateManager cachedStateManager; private CachedStateManager cachedStateManager;
private ContextManager<ProxiedPlayer> contextManager; private ContextManager<ProxiedPlayer> contextManager;
@ -181,7 +180,7 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin {
// register commands // register commands
getLog().info("Registering commands..."); getLog().info("Registering commands...");
CommandManager commandManager = new CommandManager(this); commandManager = new CommandManager(this);
getProxy().getPluginManager().registerCommand(this, new BungeeCommand(this, commandManager)); getProxy().getPluginManager().registerCommand(this, new BungeeCommand(this, commandManager));
// disable the default Bungee /perms command so it gets handled by the Bukkit plugin // disable the default Bungee /perms command so it gets handled by the Bukkit plugin
@ -193,7 +192,6 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin {
userManager = new GenericUserManager(this); userManager = new GenericUserManager(this);
groupManager = new GenericGroupManager(this); groupManager = new GenericGroupManager(this);
trackManager = new GenericTrackManager(this); trackManager = new GenericTrackManager(this);
importer = new Importer(commandManager);
calculatorFactory = new BungeeCalculatorFactory(this); calculatorFactory = new BungeeCalculatorFactory(this);
cachedStateManager = new CachedStateManager(this); cachedStateManager = new CachedStateManager(this);

146
common/src/main/java/me/lucko/luckperms/common/commands/misc/ExportCommand.java
View File

@ -22,47 +22,25 @@
package me.lucko.luckperms.common.commands.misc; package me.lucko.luckperms.common.commands.misc;
import me.lucko.luckperms.api.Node;
import me.lucko.luckperms.common.commands.Arg; import me.lucko.luckperms.common.commands.Arg;
import me.lucko.luckperms.common.commands.CommandResult; import me.lucko.luckperms.common.commands.CommandResult;
import me.lucko.luckperms.common.commands.SingleCommand; import me.lucko.luckperms.common.commands.SingleCommand;
import me.lucko.luckperms.common.commands.sender.Sender; import me.lucko.luckperms.common.commands.sender.Sender;
import me.lucko.luckperms.common.constants.Message; import me.lucko.luckperms.common.constants.Message;
import me.lucko.luckperms.common.constants.Permission; import me.lucko.luckperms.common.constants.Permission;
import me.lucko.luckperms.common.core.NodeFactory; import me.lucko.luckperms.common.data.Exporter;
import me.lucko.luckperms.common.core.model.Group;
import me.lucko.luckperms.common.core.model.Track;
import me.lucko.luckperms.common.core.model.User;
import me.lucko.luckperms.common.plugin.LuckPermsPlugin; import me.lucko.luckperms.common.plugin.LuckPermsPlugin;
import me.lucko.luckperms.common.storage.Storage;
import me.lucko.luckperms.common.utils.Predicates; import me.lucko.luckperms.common.utils.Predicates;
import me.lucko.luckperms.common.utils.ProgressLogger;
import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
public class ExportCommand extends SingleCommand { public class ExportCommand extends SingleCommand {
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); private AtomicBoolean running = new AtomicBoolean(false);
private static void write(BufferedWriter writer, String s) {
try {
writer.write(s);
writer.newLine();
} catch (IOException e) {
e.printStackTrace();
}
}
public ExportCommand() { public ExportCommand() {
super("Export", "Export data to a file", "/%s export <file>", Permission.EXPORT, Predicates.not(1), super("Export", "Export data to a file", "/%s export <file>", Permission.EXPORT, Predicates.not(1),
@ -74,9 +52,10 @@ public class ExportCommand extends SingleCommand {
@Override @Override
public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List<String> args, String label) { public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List<String> args, String label) {
ProgressLogger log = new ProgressLogger(null, Message.EXPORT_LOG, Message.EXPORT_LOG_PROGRESS); if (running.get()) {
log.addListener(plugin.getConsoleSender()); Message.EXPORT_ALREADY_RUNNING.send(sender);
log.addListener(sender); return CommandResult.STATE_ERROR;
}
File f = new File(plugin.getDataDirectory(), args.get(0)); File f = new File(plugin.getDataDirectory(), args.get(0));
if (f.exists()) { if (f.exists()) {
@ -99,112 +78,23 @@ public class ExportCommand extends SingleCommand {
return CommandResult.FAILURE; return CommandResult.FAILURE;
} }
try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { if (!running.compareAndSet(false, true)) {
log.log("Starting."); Message.EXPORT_ALREADY_RUNNING.send(sender);
return CommandResult.STATE_ERROR;
write(writer, "# LuckPerms Export File");
write(writer, "# Generated by " + sender.getName() + " at " + DATE_FORMAT.format(new Date(System.currentTimeMillis())));
write(writer, "");
// Export Groups
log.log("Starting group export.");
// Create the actual groups first
write(writer, "# Create groups");
for (Group group : plugin.getGroupManager().getAll().values()) {
write(writer, "/luckperms creategroup " + group.getName());
}
write(writer, "");
AtomicInteger groupCount = new AtomicInteger(0);
for (Group group : plugin.getGroupManager().getAll().values()) {
write(writer, "# Export group: " + group.getName());
for (Node node : group.getNodes()) {
write(writer, NodeFactory.nodeAsCommand(node, group.getName(), true));
}
write(writer, "");
log.logAllProgress("Exported {} groups so far.", groupCount.incrementAndGet());
}
log.log("Exported " + groupCount.get() + " groups.");
write(writer, "");
write(writer, "");
// Export tracks
log.log("Starting track export.");
Collection<? extends Track> tracks = plugin.getTrackManager().getAll().values();
if (!tracks.isEmpty()) {
// Create the actual tracks first
write(writer, "# Create tracks");
for (Track track : tracks) {
write(writer, "/luckperms createtrack " + track.getName());
} }
write(writer, ""); Exporter exporter = new Exporter(plugin, sender, path);
AtomicInteger trackCount = new AtomicInteger(0); // Run the exporter in its own thread.
for (Track track : plugin.getTrackManager().getAll().values()) { plugin.doAsync(() -> {
write(writer, "# Export track: " + track.getName()); try {
for (String group : track.getGroups()) { exporter.run();
write(writer, "/luckperms track " + track.getName() + " append " + group); } finally {
} running.set(false);
write(writer, "");
log.logAllProgress("Exported {} tracks so far.", trackCount.incrementAndGet());
} }
});
write(writer, "");
write(writer, "");
}
log.log("Exported " + tracks.size() + " tracks.");
// Export users
log.log("Starting user export. Finding a list of unique users to export.");
Storage ds = plugin.getStorage();
Set<UUID> users = ds.getUniqueUsers().join();
log.log("Found " + users.size() + " unique users to export.");
write(writer, "# Export users");
AtomicInteger userCount = new AtomicInteger(0);
for (UUID uuid : users) {
plugin.getStorage().loadUser(uuid, "null").join();
User user = plugin.getUserManager().get(uuid);
write(writer, "# Export user: " + user.getUuid().toString() + " - " + user.getName());
boolean inDefault = false;
for (Node node : user.getNodes()) {
if (node.isGroupNode() && node.getGroupName().equalsIgnoreCase("default")) {
inDefault = true;
continue;
}
write(writer, NodeFactory.nodeAsCommand(node, user.getUuid().toString(), false));
}
if (!user.getPrimaryGroup().equalsIgnoreCase("default")) {
write(writer, "/luckperms user " + user.getUuid().toString() + " switchprimarygroup " + user.getPrimaryGroup());
}
if (!inDefault) {
write(writer, "/luckperms user " + user.getUuid().toString() + " parent remove default");
}
plugin.getUserManager().cleanup(user);
log.logProgress("Exported {} users so far.", userCount.incrementAndGet());
}
log.log("Exported " + userCount.get() + " users.");
writer.flush();
log.getListeners().forEach(l -> Message.LOG_EXPORT_SUCCESS.send(l, f.getAbsolutePath()));
return CommandResult.SUCCESS; return CommandResult.SUCCESS;
} catch (Throwable t) {
t.printStackTrace();
return CommandResult.FAILURE;
}
} }
} }

24
common/src/main/java/me/lucko/luckperms/common/commands/misc/ImportCommand.java
View File

@ -38,8 +38,11 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class ImportCommand extends SingleCommand { public class ImportCommand extends SingleCommand {
private AtomicBoolean running = new AtomicBoolean(false);
public ImportCommand() { public ImportCommand() {
super("Import", "Import data from a file", "/%s import <file>", Permission.IMPORT, Predicates.not(1), super("Import", "Import data from a file", "/%s import <file>", Permission.IMPORT, Predicates.not(1),
Arg.list( Arg.list(
@ -50,13 +53,11 @@ public class ImportCommand extends SingleCommand {
@Override @Override
public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List<String> args, String label) { public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List<String> args, String label) {
if (args.size() == 0) { if (running.get()) {
sendUsage(sender, label); Message.IMPORT_ALREADY_RUNNING.send(sender);
return CommandResult.INVALID_ARGS; return CommandResult.STATE_ERROR;
} }
Importer importer = plugin.getImporter();
File f = new File(plugin.getDataDirectory(), args.get(0)); File f = new File(plugin.getDataDirectory(), args.get(0));
if (!f.exists()) { if (!f.exists()) {
Message.IMPORT_LOG_DOESNT_EXIST.send(sender, f.getAbsolutePath()); Message.IMPORT_LOG_DOESNT_EXIST.send(sender, f.getAbsolutePath());
@ -80,13 +81,22 @@ public class ImportCommand extends SingleCommand {
return CommandResult.FAILURE; return CommandResult.FAILURE;
} }
if (!importer.startRun()) { if (!running.compareAndSet(false, true)) {
Message.IMPORT_ALREADY_RUNNING.send(sender); Message.IMPORT_ALREADY_RUNNING.send(sender);
return CommandResult.STATE_ERROR; return CommandResult.STATE_ERROR;
} }
Importer importer = new Importer(plugin.getCommandManager(), sender, commands);
// Run the importer in its own thread. // Run the importer in its own thread.
plugin.doAsync(() -> importer.start(sender, commands)); plugin.doAsync(() -> {
try {
importer.run();
} finally {
running.set(false);
}
});
return CommandResult.SUCCESS; return CommandResult.SUCCESS;
} }
} }

2
common/src/main/java/me/lucko/luckperms/common/commands/utils/Util.java
View File

@ -341,7 +341,7 @@ public class Util {
return Maps.immutableEntry(message, title); return Maps.immutableEntry(message, title);
} }
public static <T> List<List<T>> divideList(List<T> source, int size) { public static <T> List<List<T>> divideList(Iterable<T> source, int size) {
List<List<T>> lists = new ArrayList<>(); List<List<T>> lists = new ArrayList<>();
Iterator<T> it = source.iterator(); Iterator<T> it = source.iterator();
while (it.hasNext()) { while (it.hasNext()) {

1
common/src/main/java/me/lucko/luckperms/common/constants/Message.java
View File

@ -410,6 +410,7 @@ public enum Message {
LOG_EXPORT_SUCCESS("&aSuccessfully exported the log to &b{0}&a.", true), LOG_EXPORT_SUCCESS("&aSuccessfully exported the log to &b{0}&a.", true),
IMPORT_ALREADY_RUNNING("Another import process is already running. Please wait for it to finish and try again.", true), IMPORT_ALREADY_RUNNING("Another import process is already running. Please wait for it to finish and try again.", true),
EXPORT_ALREADY_RUNNING("Another export process is already running. Please wait for it to finish and try again.", true),
IMPORT_LOG_DOESNT_EXIST("Error: File {0} does not exist.", true), IMPORT_LOG_DOESNT_EXIST("Error: File {0} does not exist.", true),
IMPORT_LOG_NOT_READABLE("Error: File {0} is not readable.", true), IMPORT_LOG_NOT_READABLE("Error: File {0} is not readable.", true),
IMPORT_LOG_FAILURE("An unexpected error occured whilst reading from the log file.", true), IMPORT_LOG_FAILURE("An unexpected error occured whilst reading from the log file.", true),

2
common/src/main/java/me/lucko/luckperms/common/core/model/User.java
View File

@ -143,7 +143,7 @@ public class User extends PermissionHolder implements Identifiable<UserIdentifie
ud.recalculatePermissions(); ud.recalculatePermissions();
ud.recalculateMeta(); ud.recalculateMeta();
getPlugin().getApiProvider().getEventFactory().handleUserDataRecalculate(this, userData); getPlugin().getApiProvider().getEventFactory().handleUserDataRecalculate(this, ud);
getPlugin().onUserRefresh(this); getPlugin().onUserRefresh(this);
} }

261
common/src/main/java/me/lucko/luckperms/common/data/Exporter.java Normal file
View File

@ -0,0 +1,261 @@
/*
* Copyright (c) 2016 Lucko (Luck) <luck@lucko.me>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package me.lucko.luckperms.common.data;
import me.lucko.luckperms.api.Node;
import me.lucko.luckperms.common.commands.sender.Sender;
import me.lucko.luckperms.common.commands.utils.Util;
import me.lucko.luckperms.common.constants.Message;
import me.lucko.luckperms.common.core.NodeFactory;
import me.lucko.luckperms.common.core.model.Group;
import me.lucko.luckperms.common.core.model.Track;
import me.lucko.luckperms.common.core.model.User;
import me.lucko.luckperms.common.plugin.LuckPermsPlugin;
import me.lucko.luckperms.common.storage.Storage;
import me.lucko.luckperms.common.utils.ProgressLogger;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Function;
/**
* Handles export operations
*/
public class Exporter implements Runnable {
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z");
// number of users --> the value to divide the list by. base value of 1000 per thread, with a max of 10 threads. so, 3000 = 3 threads, 10000+ = 10 threads
private static final Function<Integer, Integer> THREAD_COUNT_FUNCTION = usersCount -> {
// how many threads to make. must be 1 <= x <= 15
int i = Math.max(1, Math.min(15, usersCount / 1000));
// then work out the value to split at. e.g. if we have 1,000 users to export and 2 threads, we should split at every 500 users.
return Math.max(1, usersCount / i);
};
private static void write(BufferedWriter writer, String s) {
try {
writer.write(s);
writer.newLine();
} catch (IOException e) {
e.printStackTrace();
}
}
private final LuckPermsPlugin plugin;
private final Sender executor;
private final Path filePath;
private final ProgressLogger log;
public Exporter(LuckPermsPlugin plugin, Sender executor, Path filePath) {
this.plugin = plugin;
this.executor = executor;
this.filePath = filePath;
log = new ProgressLogger(null, Message.EXPORT_LOG, Message.EXPORT_LOG_PROGRESS);
log.addListener(plugin.getConsoleSender());
log.addListener(executor);
}
@Override
public void run() {
try (BufferedWriter writer = Files.newBufferedWriter(filePath, StandardCharsets.UTF_8)) {
log.log("Starting.");
write(writer, "# LuckPerms Export File");
write(writer, "# Generated by " + executor.getName() + " at " + DATE_FORMAT.format(new Date(System.currentTimeMillis())));
write(writer, "");
// Export Groups
log.log("Starting group export.");
// Create the actual groups first
write(writer, "# Create groups");
for (Group group : plugin.getGroupManager().getAll().values()) {
write(writer, "/luckperms creategroup " + group.getName());
}
write(writer, "");
AtomicInteger groupCount = new AtomicInteger(0);
for (Group group : plugin.getGroupManager().getAll().values()) {
write(writer, "# Export group: " + group.getName());
for (Node node : group.getNodes()) {
write(writer, NodeFactory.nodeAsCommand(node, group.getName(), true));
}
write(writer, "");
log.logAllProgress("Exported {} groups so far.", groupCount.incrementAndGet());
}
log.log("Exported " + groupCount.get() + " groups.");
write(writer, "");
write(writer, "");
// Export tracks
log.log("Starting track export.");
Collection<? extends Track> tracks = plugin.getTrackManager().getAll().values();
if (!tracks.isEmpty()) {
// Create the actual tracks first
write(writer, "# Create tracks");
for (Track track : tracks) {
write(writer, "/luckperms createtrack " + track.getName());
}
write(writer, "");
AtomicInteger trackCount = new AtomicInteger(0);
for (Track track : plugin.getTrackManager().getAll().values()) {
write(writer, "# Export track: " + track.getName());
for (String group : track.getGroups()) {
write(writer, "/luckperms track " + track.getName() + " append " + group);
}
write(writer, "");
log.logAllProgress("Exported {} tracks so far.", trackCount.incrementAndGet());
}
write(writer, "");
write(writer, "");
}
log.log("Exported " + tracks.size() + " tracks.");
// Users are migrated in separate threads.
// This is because there are likely to be a lot of them, and because we can.
// It's a big speed improvement, since the database/files are split up and can handle concurrent reads.
log.log("Starting user export. Finding a list of unique users to export.");
// Find all of the unique users we need to export
Storage ds = plugin.getStorage();
Set<UUID> users = ds.getUniqueUsers().join();
log.log("Found " + users.size() + " unique users to export.");
write(writer, "# Export users");
List<List<UUID>> subUsers;
AtomicInteger userCount = new AtomicInteger(0);
// not really that many users, so it's not really worth spreading the load.
if (users.size() < 1500) {
subUsers = Collections.singletonList(new ArrayList<>(users));
} else {
subUsers = Util.divideList(users, THREAD_COUNT_FUNCTION.apply(users.size()));
}
log.log("Split users into " + subUsers.size() + " threads for export.");
// Setup a file writing lock. We don't want multiple threads writing at the same time.
// The write function accepts a list of strings, as we want a user's data to be grouped together.
// This means it can be processed and added in one go.
ReentrantLock lock = new ReentrantLock();
Consumer<List<String>> writeFunction = strings -> {
lock.lock();
try {
for (String s : strings) {
write(writer, s);
}
} finally {
lock.unlock();
}
};
// A set of futures, which are really just the threads we need to wait for.
Set<CompletableFuture<Void>> futures = new HashSet<>();
// iterate through each user sublist.
for (List<UUID> subList : subUsers) {
// register and start a new thread to process the sublist
futures.add(CompletableFuture.runAsync(() -> {
// iterate through each user in the sublist, and grab their data.
for (UUID uuid : subList) {
try {
// actually export the user. this output will be fed to the writing function when we have all of the user's data.
List<String> output = new ArrayList<>();
plugin.getStorage().loadUser(uuid, "null").join();
User user = plugin.getUserManager().get(uuid);
output.add("# Export user: " + user.getUuid().toString() + " - " + user.getName());
boolean inDefault = false;
for (Node node : user.getNodes()) {
if (node.isGroupNode() && node.getGroupName().equalsIgnoreCase("default")) {
inDefault = true;
continue;
}
output.add(NodeFactory.nodeAsCommand(node, user.getUuid().toString(), false));
}
if (!user.getPrimaryGroup().equalsIgnoreCase("default")) {
output.add("/luckperms user " + user.getUuid().toString() + " switchprimarygroup " + user.getPrimaryGroup());
}
if (!inDefault) {
output.add("/luckperms user " + user.getUuid().toString() + " parent remove default");
}
plugin.getUserManager().cleanup(user);
writeFunction.accept(output);
log.logProgress("Exported {} users so far.", userCount.incrementAndGet());
} catch (Exception e) {
e.printStackTrace();
}
}
}, plugin.getScheduler().getAsyncExecutor()));
}
// all of the threads have been scheduled now and are running. we just need to wait for them all to complete
CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).join();
log.log("Exported " + userCount.get() + " users.");
writer.flush();
log.getListeners().forEach(l -> Message.LOG_EXPORT_SUCCESS.send(l, filePath.toFile().getAbsolutePath()));
} catch (Exception e) {
e.printStackTrace();
}
}
}

130
common/src/main/java/me/lucko/luckperms/common/data/Importer.java
View File

@ -24,7 +24,6 @@ package me.lucko.luckperms.common.data;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.Getter; import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter; import lombok.Setter;
import com.google.common.base.Splitter; import com.google.common.base.Splitter;
@ -48,33 +47,27 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
* Class to handle import operations * Handles import operations
*/ */
@RequiredArgsConstructor public class Importer implements Runnable {
public class Importer { private static final int PROGRESS_REPORT_SECONDS = 10;
private final CommandManager commandManager;
private boolean running = false; private final CommandManager commandManager;
private Set<Sender> notify = ImmutableSet.of(); private final Set<Sender> notify;
private List<String> commands = null; private final List<String> commands;
private Map<Integer, Result> cmdResult = null; private final Map<Integer, Result> cmdResult;
private final FakeSender fake;
private long lastMsg = 0; private long lastMsg = 0;
private int executing = -1; private int executing = -1;
public synchronized boolean startRun() { public Importer(CommandManager commandManager, Sender executor, List<String> commands) {
if (running) { this.commandManager = commandManager;
return false;
}
running = true;
return true;
}
public void start(Sender executor, List<String> commands) {
if (executor.isConsole()) { if (executor.isConsole()) {
this.notify = ImmutableSet.of(executor); this.notify = ImmutableSet.of(executor);
} else { } else {
@ -90,29 +83,20 @@ public class Importer {
.map(s -> s.startsWith("luckperms ") ? s.substring("luckperms ".length()) : s) .map(s -> s.startsWith("luckperms ") ? s.substring("luckperms ".length()) : s)
.collect(Collectors.toList()); .collect(Collectors.toList());
cmdResult = new HashMap<>(); this.cmdResult = new HashMap<>();
this.fake = new FakeSender();
run();
}
private void cleanup() {
notify = ImmutableSet.of();
commands = null;
cmdResult = null;
lastMsg = 0;
executing = -1;
running = false;
} }
@Override
public void run() { public void run() {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
notify.forEach(s -> Message.IMPORT_START.send(s)); notify.forEach(s -> Message.IMPORT_START.send(s));
final Sender fake = new FakeSender(this);
int index = 1; int index = 1;
for (String command : commands) { for (String command : commands) {
if (lastMsg < (System.currentTimeMillis() / 1000) - 5) { long time = System.currentTimeMillis() / 1000;
lastMsg = System.currentTimeMillis() / 1000; if (lastMsg < (time - PROGRESS_REPORT_SECONDS)) {
lastMsg = time;
sendProgress(index); sendProgress(index);
} }
@ -134,80 +118,57 @@ public class Importer {
} }
long endTime = System.currentTimeMillis(); long endTime = System.currentTimeMillis();
double seconds = (endTime - startTime) / 1000.0; double seconds = (endTime - startTime) / 1000;
int errors = 0; int errors = (int) cmdResult.values().stream().filter(v -> v.getResult() != null && !v.getResult().asBoolean()).count();
for (Map.Entry<Integer, Result> e : cmdResult.entrySet()) {
if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) {
errors++;
}
}
if (errors == 0) { if (errors == 0) {
for (Sender s : notify) { notify.forEach(s -> Message.IMPORT_END_COMPLETE.send(s, seconds));
Message.IMPORT_END_COMPLETE.send(s, seconds);
}
} else if (errors == 1) { } else if (errors == 1) {
for (Sender s : notify) { notify.forEach(s -> Message.IMPORT_END_COMPLETE_ERR_SIN.send(s, seconds, errors));
Message.IMPORT_END_COMPLETE_ERR_SIN.send(s, seconds, errors);
}
} else { } else {
for (Sender s : notify) { notify.forEach(s -> Message.IMPORT_END_COMPLETE_ERR.send(s, seconds, errors));
Message.IMPORT_END_COMPLETE_ERR.send(s, seconds, errors);
}
} }
int errIndex = 1; AtomicInteger errIndex = new AtomicInteger(1);
for (Map.Entry<Integer, Result> e : cmdResult.entrySet()) { for (Map.Entry<Integer, Result> e : cmdResult.entrySet()) {
if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) { if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) {
for (Sender s : notify) { notify.forEach(s -> {
Message.IMPORT_END_ERROR_HEADER.send(s, errIndex, e.getKey(), e.getValue().getCommand(), e.getValue().getResult().toString()); Message.IMPORT_END_ERROR_HEADER.send(s, errIndex.get(), e.getKey(), e.getValue().getCommand(), e.getValue().getResult().toString());
for (String out : e.getValue().getOutput()) { for (String out : e.getValue().getOutput()) {
Message.IMPORT_END_ERROR_CONTENT.send(s, out); Message.IMPORT_END_ERROR_CONTENT.send(s, out);
} }
Message.IMPORT_END_ERROR_FOOTER.send(s); Message.IMPORT_END_ERROR_FOOTER.send(s);
} });
errIndex++; errIndex.incrementAndGet();
} }
} }
cleanup();
} }
private void sendProgress(int executing) { private void sendProgress(int executing) {
int percent = (executing * 100) / commands.size(); int percent = (executing * 100) / commands.size();
int errors = 0; int errors = (int) cmdResult.values().stream().filter(v -> v.getResult() != null && !v.getResult().asBoolean()).count();
for (Map.Entry<Integer, Result> e : cmdResult.entrySet()) {
if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) {
errors++;
}
}
if (errors == 1) { if (errors == 1) {
for (Sender s : notify) { notify.forEach(s -> Message.IMPORT_PROGRESS_SIN.send(s, percent, executing, commands.size(), errors));
Message.IMPORT_PROGRESS_SIN.send(s, percent, executing, commands.size(), errors);
}
} else { } else {
for (Sender s : notify) { notify.forEach(s -> Message.IMPORT_PROGRESS.send(s, percent, executing, commands.size(), errors));
Message.IMPORT_PROGRESS.send(s, percent, executing, commands.size(), errors);
}
} }
} }
private Result getResult(int executing, String command) { private Result getResult(int executing, String command) {
if (!cmdResult.containsKey(executing)) { return cmdResult.compute(executing, (i, r) -> {
cmdResult.put(executing, new Result(command)); if (r == null) {
r = new Result(command);
} else {
if (!command.equals("") && r.getCommand().equals("")) {
r.setCommand(command);
}
} }
Result existing = cmdResult.get(executing); return r;
});
if (!command.equals("") && existing.getCommand().equals("")) {
existing.setCommand(command);
}
return existing;
} }
private void logMessage(String msg) { private void logMessage(String msg) {
@ -216,16 +177,11 @@ public class Importer {
} }
} }
private static class FakeSender implements Sender { private class FakeSender implements Sender {
private final Importer instance;
private FakeSender(Importer instance) {
this.instance = instance;
}
@Override @Override
public LuckPermsPlugin getPlatform() { public LuckPermsPlugin getPlatform() {
return instance.commandManager.getPlugin(); return commandManager.getPlugin();
} }
@Override @Override
@ -240,12 +196,12 @@ public class Importer {
@Override @Override
public void sendMessage(String s) { public void sendMessage(String s) {
instance.logMessage(s); logMessage(s);
} }
@Override @Override
public void sendMessage(FancyMessage message) { public void sendMessage(FancyMessage message) {
instance.logMessage(message.toOldMessageFormat()); logMessage(message.toOldMessageFormat());
} }
@Override @Override

8
common/src/main/java/me/lucko/luckperms/common/plugin/LuckPermsPlugin.java
View File

@ -29,6 +29,7 @@ import me.lucko.luckperms.common.api.ApiProvider;
import me.lucko.luckperms.common.caching.handlers.CachedStateManager; import me.lucko.luckperms.common.caching.handlers.CachedStateManager;
import me.lucko.luckperms.common.calculators.CalculatorFactory; import me.lucko.luckperms.common.calculators.CalculatorFactory;
import me.lucko.luckperms.common.commands.BaseCommand; import me.lucko.luckperms.common.commands.BaseCommand;
import me.lucko.luckperms.common.commands.CommandManager;
import me.lucko.luckperms.common.commands.sender.Sender; import me.lucko.luckperms.common.commands.sender.Sender;
import me.lucko.luckperms.common.commands.utils.Util; import me.lucko.luckperms.common.commands.utils.Util;
import me.lucko.luckperms.common.config.LuckPermsConfiguration; import me.lucko.luckperms.common.config.LuckPermsConfiguration;
@ -36,7 +37,6 @@ import me.lucko.luckperms.common.constants.Message;
import me.lucko.luckperms.common.contexts.ContextManager; import me.lucko.luckperms.common.contexts.ContextManager;
import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.UuidCache;
import me.lucko.luckperms.common.core.model.User; import me.lucko.luckperms.common.core.model.User;
import me.lucko.luckperms.common.data.Importer;
import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.debug.DebugHandler;
import me.lucko.luckperms.common.locale.LocaleManager; import me.lucko.luckperms.common.locale.LocaleManager;
import me.lucko.luckperms.common.managers.GroupManager; import me.lucko.luckperms.common.managers.GroupManager;
@ -126,11 +126,11 @@ public interface LuckPermsPlugin {
ApiProvider getApiProvider(); ApiProvider getApiProvider();
/** /**
* Gets the importer instance * Gets the command manager
* *
* @return the importer * @return the command manager
*/ */
Importer getImporter(); CommandManager getCommandManager();
/** /**
* Gets the instance providing locale translations for the plugin * Gets the instance providing locale translations for the plugin

4
common/src/main/java/me/lucko/luckperms/common/utils/HikariSupplier.java
View File

@ -39,9 +39,9 @@ public class HikariSupplier implements AutoCloseable {
private HikariDataSource hikari; private HikariDataSource hikari;
public void setup() { public void setup(String poolName) {
hikari = new HikariDataSource(); hikari = new HikariDataSource();
hikari.setPoolName("powerfulperms-migrator-pool"); hikari.setPoolName(poolName);
hikari.setMaximumPoolSize(2); hikari.setMaximumPoolSize(2);
hikari.setDataSourceClassName("com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); hikari.setDataSourceClassName("com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
hikari.addDataSourceProperty("serverName", address.split(":")[0]); hikari.addDataSourceProperty("serverName", address.split(":")[0]);

1
default-lang.yml
View File

@ -346,6 +346,7 @@ log-export-failure: "An unexpected error occured whilst writing to the file."
log-export-success: "&aSuccessfully exported the log to &b{0}&a." log-export-success: "&aSuccessfully exported the log to &b{0}&a."
import-already-running: "Another import process is already running. Please wait for it to finish and try again." import-already-running: "Another import process is already running. Please wait for it to finish and try again."
export-already-running: "Another export process is already running. Please wait for it to finish and try again."
import-log-doesnt-exist: "Error: File {0} does not exist." import-log-doesnt-exist: "Error: File {0} does not exist."
import-log-not-readable: "Error: File {0} is not readable." import-log-not-readable: "Error: File {0} is not readable."
import-log-failure: "An unexpected error occured whilst reading from the log file." import-log-failure: "An unexpected error occured whilst reading from the log file."

4
sponge/src/main/java/me/lucko/luckperms/sponge/LPSpongePlugin.java
View File

@ -140,6 +140,7 @@ public class LPSpongePlugin implements LuckPermsPlugin {
private boolean lateLoad = false; private boolean lateLoad = false;
private LuckPermsScheduler scheduler; private LuckPermsScheduler scheduler;
private SpongeCommand commandManager;
private LuckPermsConfiguration configuration; private LuckPermsConfiguration configuration;
private SpongeUserManager userManager; private SpongeUserManager userManager;
private SpongeGroupManager groupManager; private SpongeGroupManager groupManager;
@ -239,7 +240,7 @@ public class LPSpongePlugin implements LuckPermsPlugin {
// register commands // register commands
getLog().info("Registering commands..."); getLog().info("Registering commands...");
CommandManager cmdService = game.getCommandManager(); CommandManager cmdService = game.getCommandManager();
SpongeCommand commandManager = new SpongeCommand(this); commandManager = new SpongeCommand(this);
cmdService.register(this, commandManager, "luckperms", "perms", "lp", "permissions", "perm"); cmdService.register(this, commandManager, "luckperms", "perms", "lp", "permissions", "perm");
// load internal managers // load internal managers
@ -248,7 +249,6 @@ public class LPSpongePlugin implements LuckPermsPlugin {
userManager = new SpongeUserManager(this); userManager = new SpongeUserManager(this);
groupManager = new SpongeGroupManager(this); groupManager = new SpongeGroupManager(this);
trackManager = new GenericTrackManager(this); trackManager = new GenericTrackManager(this);
importer = new Importer(commandManager);
calculatorFactory = new SpongeCalculatorFactory(this); calculatorFactory = new SpongeCalculatorFactory(this);
cachedStateManager = new CachedStateManager(this); cachedStateManager = new CachedStateManager(this);