Merge branch '1147-limbo-improve-persistence' of https://github.com/AuthMe/AuthMeReloaded

This commit is contained in:
ljacqu 2017-03-31 19:57:23 +02:00
commit 20ab161406
12 changed files with 64 additions and 162 deletions

View File

@ -1,5 +1,5 @@
<!-- AUTO-GENERATED FILE! Do not edit this directly -->
<!-- File auto-generated on Wed Mar 22 23:10:33 CET 2017. See docs/config/config.tpl.md -->
<!-- File auto-generated on Tue Mar 28 21:48:52 CEST 2017. See docs/config/config.tpl.md -->
## AuthMe Configuration
The first time you run AuthMe it will create a config.yml file in the plugins/AuthMe folder,
@ -163,9 +163,6 @@ settings:
teleportUnAuthedToSpawn: false
# Can unregistered players walk around?
allowMovement: false
# Should not authenticated players have speed = 0?
# This will reset the fly/walk speed to default value after the login.
removeSpeed: true
# After how many seconds should players who fail to login or register
# be kicked? Set to 0 to disable.
timeout: 30
@ -451,27 +448,26 @@ Security:
# such as OP status, ability to fly, and walk/fly speed.
# Once the user is logged in, we add back the properties we previously saved.
# In this section, you may define how these properties should be handled.
# Read more at https://github.com/AuthMe/AuthMeReloaded/wiki/Limbo-players
limbo:
persistence:
# Besides storing the data in memory, you can define if/how the data should be persisted
# on disk. This is useful in case of a server crash, so next time the server starts we can
# properly restore things like OP status, ability to fly, and walk/fly speed.
# DISABLED: no disk storage, INDIVIDUAL_FILES: each player data in its own file,
# SINGLE_FILE: all data in one single file (only if you have a small server!)
# SEGMENT_FILES: distributes players into different buckets based on their UUID. See below.
# DISABLED: no disk storage,
# INDIVIDUAL_FILES: each player data in its own file,
# DISTRIBUTED_FILES: distributes players into different files based on their UUID, see below
type: 'INDIVIDUAL_FILES'
# This setting only affects SEGMENT_FILES persistence. The segment file
# This setting only affects DISTRIBUTED_FILES persistence. The distributed file
# persistence attempts to reduce the number of files by distributing players into various
# buckets based on their UUID. This setting defines into how many files the players should
# be distributed. Possible values: ONE, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,
# ONE_TWENTY for 128, TWO_FIFTY for 256.
# For example, if you expect 100 non-logged in players, setting to SIXTEEN will average
# 6.25 players per file (100 / 16). If you set to ONE, like persistence SINGLE_FILE, only
# one file will be used. Contrary to SINGLE_FILE, it won't keep the entries in cache, which
# may deliver different results in terms of performance.
# 6.25 players per file (100 / 16).
# Note: if you change this setting all data will be migrated. If you have a lot of data,
# change this setting only on server restart, not with /authme reload.
segmentDistribution: 'SIXTEEN'
distributionSize: 'SIXTEEN'
# Whether the player is allowed to fly: RESTORE, ENABLE, DISABLE.
# RESTORE sets back the old property from the player.
restoreAllowFlight: 'RESTORE'
@ -511,4 +507,4 @@ To change settings on a running server, save your changes to config.yml and use
---
This page was automatically generated on the [AuthMe/AuthMeReloaded repository](https://github.com/AuthMe/AuthMeReloaded/tree/master/docs/) on Wed Mar 22 23:10:33 CET 2017
This page was automatically generated on the [AuthMe/AuthMeReloaded repository](https://github.com/AuthMe/AuthMeReloaded/tree/master/docs/) on Tue Mar 28 21:48:52 CEST 2017

View File

@ -30,7 +30,7 @@ import java.util.Optional;
* Persistence handler for LimboPlayer objects by distributing the objects to store
* in various segments (buckets) based on the start of the player's UUID.
*/
class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
class DistributedFilesPersistenceHandler implements LimboPersistenceHandler {
private static final Type LIMBO_MAP_TYPE = new TypeToken<Map<String, LimboPlayer>>(){}.getType();
@ -39,7 +39,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
private final SegmentNameBuilder segmentNameBuilder;
@Inject
SegmentFilesPersistenceHolder(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings) {
DistributedFilesPersistenceHandler(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings) {
cacheFolder = new File(dataFolder, "playerdata");
if (!cacheFolder.exists()) {
// TODO ljacqu 20170313: Create FileUtils#mkdirs
@ -52,7 +52,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
.setPrettyPrinting()
.create();
segmentNameBuilder = new SegmentNameBuilder(settings.getProperty(LimboSettings.SEGMENT_DISTRIBUTION));
segmentNameBuilder = new SegmentNameBuilder(settings.getProperty(LimboSettings.DISTRIBUTION_SIZE));
convertOldDataToCurrentSegmentScheme();
deleteEmptyFiles();
@ -100,7 +100,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
@Override
public LimboPersistenceType getType() {
return LimboPersistenceType.SEGMENT_FILES;
return LimboPersistenceType.DISTRIBUTED_FILES;
}
private void saveEntries(Map<String, LimboPlayer> entries, File file) {
@ -126,7 +126,11 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
private File getPlayerSegmentFile(String uuid) {
String segment = segmentNameBuilder.createSegmentName(uuid);
return new File(cacheFolder, segment + "-limbo.json");
return getSegmentFile(segment);
}
private File getSegmentFile(String segmentId) {
return new File(cacheFolder, segmentId + "-limbo.json");
}
/**
@ -167,7 +171,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
ConsoleLogger.info("Saving " + limbosFromOldSegments.size() + " LimboPlayers from old segments into "
+ limboBySegment.size() + " current segments");
for (Map.Entry<String, Map<String, LimboPlayer>> entry : limboBySegment.entrySet()) {
File file = new File(cacheFolder, entry.getKey() + "-limbo.json");
File file = getSegmentFile(entry.getKey());
Map<String, LimboPlayer> limbosToSave = Optional.ofNullable(readLimboPlayers(file))
.orElseGet(HashMap::new);
limbosToSave.putAll(entry.getValue());
@ -193,7 +197,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
}
/**
* Deletes files from the current segmenting scheme that are empty.
* Deletes segment files that are empty.
*/
private void deleteEmptyFiles() {
File[] files = listFiles(cacheFolder);

View File

@ -19,13 +19,13 @@ import java.nio.charset.StandardCharsets;
/**
* Saves LimboPlayer objects as JSON into individual files.
*/
class SeparateFilePersistenceHandler implements LimboPersistenceHandler {
class IndividualFilesPersistenceHandler implements LimboPersistenceHandler {
private final Gson gson;
private final File cacheDir;
@Inject
SeparateFilePersistenceHandler(@DataFolder File dataFolder, BukkitService bukkitService) {
IndividualFilesPersistenceHandler(@DataFolder File dataFolder, BukkitService bukkitService) {
cacheDir = new File(dataFolder, "playerdata");
if (!cacheDir.exists() && !cacheDir.isDirectory() && !cacheDir.mkdir()) {
ConsoleLogger.warning("Failed to create playerdata directory '" + cacheDir + "'");

View File

@ -6,13 +6,10 @@ package fr.xephi.authme.data.limbo.persistence;
public enum LimboPersistenceType {
/** Store each LimboPlayer in a separate file. */
INDIVIDUAL_FILES(SeparateFilePersistenceHandler.class),
INDIVIDUAL_FILES(IndividualFilesPersistenceHandler.class),
/** Store all LimboPlayers in the same file. */
SINGLE_FILE(SingleFilePersistenceHandler.class),
/** Distribute LimboPlayers by segments into a set number of files. */
SEGMENT_FILES(SegmentFilesPersistenceHolder.class),
/** Store LimboPlayers distributed in a configured number of files. */
DISTRIBUTED_FILES(DistributedFilesPersistenceHandler.class),
/** No persistence to disk. */
DISABLED(NoOpPersistenceHandler.class);

View File

@ -4,7 +4,7 @@ import java.util.HashMap;
import java.util.Map;
/**
* Creates segment names for {@link SegmentFilesPersistenceHolder}.
* Creates segment names for {@link DistributedFilesPersistenceHandler}.
*/
class SegmentNameBuilder {
@ -18,7 +18,7 @@ class SegmentNameBuilder {
*
* @param partition the segment configuration
*/
SegmentNameBuilder(SegmentConfiguration partition) {
SegmentNameBuilder(SegmentSize partition) {
this.length = partition.getLength();
this.distribution = partition.getDistribution();
this.prefix = "seg" + partition.getTotalSegments() + "-";

View File

@ -3,7 +3,7 @@ package fr.xephi.authme.data.limbo.persistence;
/**
* Configuration for the total number of segments to use.
* <p>
* The {@link SegmentFilesPersistenceHolder} reduces the number of files by assigning each UUID
* The {@link DistributedFilesPersistenceHandler} reduces the number of files by assigning each UUID
* to a segment. This enum allows to define how many segments the UUIDs should be distributed in.
* <p>
* Segments are defined by a <b>distribution</b> and a <b>length.</b> The distribution defines
@ -33,7 +33,7 @@ package fr.xephi.authme.data.limbo.persistence;
* Where possible, prefer a length of 1 (no string concatenation required) or a distribution of
* 16 (no remapping of the characters required).
*/
public enum SegmentConfiguration {
public enum SegmentSize {
/** 1. */
ONE(1, 1),
@ -65,7 +65,7 @@ public enum SegmentConfiguration {
private final int distribution;
private final int length;
SegmentConfiguration(int distribution, int length) {
SegmentSize(int distribution, int length) {
this.distribution = distribution;
this.length = length;
}
@ -86,7 +86,7 @@ public enum SegmentConfiguration {
}
/**
* @return number of segments to which this configuration will distribute UUIDs
* @return number of segments to which this configuration will distribute all UUIDs
*/
public int getTotalSegments() {
return (int) Math.pow(distribution, length);

View File

@ -1,94 +0,0 @@
package fr.xephi.authme.data.limbo.persistence;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.data.limbo.LimboPlayer;
import fr.xephi.authme.initialization.DataFolder;
import fr.xephi.authme.service.BukkitService;
import fr.xephi.authme.util.FileUtils;
import fr.xephi.authme.util.PlayerUtils;
import org.bukkit.entity.Player;
import javax.inject.Inject;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
/**
* Saves all LimboPlayers in one JSON file and keeps the entries in memory.
*/
class SingleFilePersistenceHandler implements LimboPersistenceHandler {
private final File cacheFile;
private final Gson gson;
private Map<String, LimboPlayer> entries;
@Inject
SingleFilePersistenceHandler(@DataFolder File dataFolder, BukkitService bukkitService) {
cacheFile = new File(dataFolder, "limbo.json");
if (!cacheFile.exists()) {
FileUtils.create(cacheFile);
}
gson = new GsonBuilder()
.registerTypeAdapter(LimboPlayer.class, new LimboPlayerSerializer())
.registerTypeAdapter(LimboPlayer.class, new LimboPlayerDeserializer(bukkitService))
.setPrettyPrinting()
.create();
Type type = new TypeToken<ConcurrentMap<String, LimboPlayer>>(){}.getType();
try (FileReader fr = new FileReader(cacheFile)) {
entries = gson.fromJson(fr, type);
} catch (IOException e) {
ConsoleLogger.logException("Failed to read from '" + cacheFile + "':", e);
}
if (entries == null) {
entries = new ConcurrentHashMap<>();
}
}
@Override
public LimboPlayer getLimboPlayer(Player player) {
return entries.get(PlayerUtils.getUUIDorName(player));
}
@Override
public void saveLimboPlayer(Player player, LimboPlayer limbo) {
entries.put(PlayerUtils.getUUIDorName(player), limbo);
saveEntries("adding '" + player.getName() + "'");
}
@Override
public void removeLimboPlayer(Player player) {
LimboPlayer entry = entries.remove(PlayerUtils.getUUIDorName(player));
if (entry != null) {
saveEntries("removing '" + player.getName() + "'");
}
}
/**
* Saves the entries to the disk.
*
* @param action the reason for saving (for logging purposes)
*/
private void saveEntries(String action) {
try (FileWriter fw = new FileWriter(cacheFile)) {
gson.toJson(entries, fw);
} catch (IOException e) {
ConsoleLogger.logException("Failed saving JSON limbo cache after " + action, e);
}
}
@Override
public LimboPersistenceType getType() {
return LimboPersistenceType.SINGLE_FILE;
}
}

View File

@ -8,7 +8,7 @@ import com.google.common.collect.ImmutableMap;
import fr.xephi.authme.data.limbo.AllowFlightRestoreType;
import fr.xephi.authme.data.limbo.WalkFlySpeedRestoreType;
import fr.xephi.authme.data.limbo.persistence.LimboPersistenceType;
import fr.xephi.authme.data.limbo.persistence.SegmentConfiguration;
import fr.xephi.authme.data.limbo.persistence.SegmentSize;
import java.util.Map;
@ -23,28 +23,26 @@ public final class LimboSettings implements SettingsHolder {
"Besides storing the data in memory, you can define if/how the data should be persisted",
"on disk. This is useful in case of a server crash, so next time the server starts we can",
"properly restore things like OP status, ability to fly, and walk/fly speed.",
"DISABLED: no disk storage, INDIVIDUAL_FILES: each player data in its own file,",
"SINGLE_FILE: all data in one single file (only if you have a small server!)",
"SEGMENT_FILES: distributes players into different buckets based on their UUID. See below."
"DISABLED: no disk storage,",
"INDIVIDUAL_FILES: each player data in its own file,",
"DISTRIBUTED_FILES: distributes players into different files based on their UUID, see below"
})
public static final Property<LimboPersistenceType> LIMBO_PERSISTENCE_TYPE =
newProperty(LimboPersistenceType.class, "limbo.persistence.type", LimboPersistenceType.INDIVIDUAL_FILES);
@Comment({
"This setting only affects SEGMENT_FILES persistence. The segment file",
"This setting only affects DISTRIBUTED_FILES persistence. The distributed file",
"persistence attempts to reduce the number of files by distributing players into various",
"buckets based on their UUID. This setting defines into how many files the players should",
"be distributed. Possible values: ONE, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,",
"ONE_TWENTY for 128, TWO_FIFTY for 256.",
"For example, if you expect 100 non-logged in players, setting to SIXTEEN will average",
"6.25 players per file (100 / 16). If you set to ONE, like persistence SINGLE_FILE, only",
"one file will be used. Contrary to SINGLE_FILE, it won't keep the entries in cache, which",
"may deliver different results in terms of performance.",
"6.25 players per file (100 / 16).",
"Note: if you change this setting all data will be migrated. If you have a lot of data,",
"change this setting only on server restart, not with /authme reload."
})
public static final Property<SegmentConfiguration> SEGMENT_DISTRIBUTION =
newProperty(SegmentConfiguration.class, "limbo.persistence.segmentDistribution", SegmentConfiguration.SIXTEEN);
public static final Property<SegmentSize> DISTRIBUTION_SIZE =
newProperty(SegmentSize.class, "limbo.persistence.distributionSize", SegmentSize.SIXTEEN);
@Comment({
"Whether the player is allowed to fly: RESTORE, ENABLE, DISABLE.",
@ -79,7 +77,8 @@ public final class LimboSettings implements SettingsHolder {
"Before a user logs in, various properties are temporarily removed from the player,",
"such as OP status, ability to fly, and walk/fly speed.",
"Once the user is logged in, we add back the properties we previously saved.",
"In this section, you may define how these properties should be handled."
"In this section, you may define how these properties should be handled.",
"Read more at https://github.com/AuthMe/AuthMeReloaded/wiki/Limbo-players"
};
return ImmutableMap.of("limbo", limboExplanation);
}

View File

@ -37,10 +37,10 @@ import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.mock;
/**
* Test for {@link SegmentFilesPersistenceHolder}.
* Test for {@link DistributedFilesPersistenceHandler}.
*/
@RunWith(DelayedInjectionRunner.class)
public class SegmentFilesPersistenceHolderTest {
public class DistributedFilesPersistenceHandlerTest {
/** Player is in seg32-10110 and should be migrated into seg16-f. */
private static final UUID MIGRATED_UUID = fromString("f6a97c88-7c8f-c12e-4931-6206d4ca067d");
@ -70,7 +70,7 @@ public class SegmentFilesPersistenceHolderTest {
@InjectDelayed
private SegmentFilesPersistenceHolder persistenceHandler;
private DistributedFilesPersistenceHandler persistenceHandler;
@Mock
private Settings settings;
@ -90,7 +90,7 @@ public class SegmentFilesPersistenceHolderTest {
@BeforeInjecting
public void setUpClasses() throws IOException {
given(settings.getProperty(LimboSettings.SEGMENT_DISTRIBUTION)).willReturn(SegmentConfiguration.SIXTEEN);
given(settings.getProperty(LimboSettings.DISTRIBUTION_SIZE)).willReturn(SegmentSize.SIXTEEN);
dataFolder = temporaryFolder.newFolder();
playerDataFolder = new File(dataFolder, "playerdata");
playerDataFolder.mkdir();

View File

@ -30,16 +30,16 @@ import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.mock;
/**
* Test for {@link SeparateFilePersistenceHandler}.
* Test for {@link IndividualFilesPersistenceHandler}.
*/
@RunWith(DelayedInjectionRunner.class)
public class SeparateFilePersistenceHandlerTest {
public class IndividualFilesPersistenceHandlerTest {
private static final UUID SAMPLE_UUID = UUID.nameUUIDFromBytes("PersistenceTest".getBytes());
private static final String SOURCE_FOLDER = TestHelper.PROJECT_ROOT + "data/backup/";
@InjectDelayed
private SeparateFilePersistenceHandler handler;
private IndividualFilesPersistenceHandler handler;
@Mock
private BukkitService bukkitService;

View File

@ -5,13 +5,13 @@ import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.EIGHT;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.ONE;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTEEN;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTY_FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.THIRTY_TWO;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.TWO_FIFTY;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.EIGHT;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.ONE;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.SIXTEEN;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.SIXTY_FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.THIRTY_TWO;
import static fr.xephi.authme.data.limbo.persistence.SegmentSize.TWO_FIFTY;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertThat;
@ -23,11 +23,11 @@ public class SegmentNameBuilderTest {
/**
* Checks that using a given segment size really produces as many segments as defined.
* E.g. if we partition with {@link SegmentConfiguration#EIGHT} we expect eight different buckets.
* E.g. if we partition with {@link SegmentSize#EIGHT} we expect eight different buckets.
*/
@Test
public void shouldCreatePromisedSizeOfSegments() {
for (SegmentConfiguration part : SegmentConfiguration.values()) {
for (SegmentSize part : SegmentSize.values()) {
// Perform this check only for `length` <= 5 because the test creates all hex numbers with `length` digits.
if (part.getLength() <= 5) {
checkTotalSegmentsProduced(part);
@ -35,7 +35,7 @@ public class SegmentNameBuilderTest {
}
}
private void checkTotalSegmentsProduced(SegmentConfiguration part) {
private void checkTotalSegmentsProduced(SegmentSize part) {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(part);
Set<String> encounteredSegments = new HashSet<>();

View File

@ -12,9 +12,9 @@ import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.fail;
/**
* Test for {@link SegmentConfiguration}.
* Test for {@link SegmentSize}.
*/
public class SegmentConfigurationTest {
public class SegmentSizeTest {
@Test
public void shouldHaveDistributionThatIsPowerOf2() {
@ -22,7 +22,7 @@ public class SegmentConfigurationTest {
Set<Integer> allowedDistributions = ImmutableSet.of(1, 2, 4, 8, 16);
// when / then
for (SegmentConfiguration entry : SegmentConfiguration.values()) {
for (SegmentSize entry : SegmentSize.values()) {
if (!allowedDistributions.contains(entry.getDistribution())) {
fail("Distribution must be a power of 2 and within [1, 16]. Offending item: " + entry);
}
@ -35,7 +35,7 @@ public class SegmentConfigurationTest {
Set<Integer> segmentTotals = new HashSet<>();
// when / then
for (SegmentConfiguration entry : SegmentConfiguration.values()) {
for (SegmentSize entry : SegmentSize.values()) {
int totalSegments = entry.getTotalSegments();
assertThat(entry + " must have a positive segment size",
totalSegments, greaterThan(0));