#457 Improve ForceFlatToSqlite conversion

- Change ForceFlatToSqlite converter to use a generic datasource destination (i.e. can be used for Flat2MySQL later)
- Add tests, including for FlatFile
- Check that user is not present in destination datasource before adding
- Persist last location from flatfile as well

src/main/java/fr/xephi/authme/converter/ForceFlatToSqlite.java

@@ -3,39 +3,51 @@ package fr.xephi.authme.converter;
 import fr.xephi.authme.ConsoleLogger;
 import fr.xephi.authme.cache.auth.PlayerAuth;
 import fr.xephi.authme.datasource.DataSource;
-import fr.xephi.authme.datasource.DataSourceType;
+import fr.xephi.authme.datasource.FlatFile;
-import fr.xephi.authme.datasource.SQLite;
+import fr.xephi.authme.util.StringUtils;
-import fr.xephi.authme.settings.NewSetting;
+
-import fr.xephi.authme.settings.properties.DatabaseSettings;
+import java.util.ArrayList;
-import java.sql.SQLException;
+import java.util.List;
 
 /**
  * Mandatory migration from the deprecated flat file datasource to SQLite.
  */
-public class ForceFlatToSqlite {
+public class ForceFlatToSqlite implements Converter {
 
-    private final DataSource database;
+    private final DataSource source;
-    private final NewSetting settings;
+    private final DataSource destination;
 
-    public ForceFlatToSqlite(DataSource database, NewSetting settings) {
+    /**
-        this.database = database;
+     * Constructor.
-        this.settings = settings;
+     *
+     * @param source The datasource to convert (flatfile)
+     * @param destination The datasource to copy the data to (sqlite)
+     */
+    public ForceFlatToSqlite(FlatFile source, DataSource destination) {
+        this.source = source;
+        this.destination = destination;
     }
 
-    public DataSource run() {
+    /**
-        try {
+     * Perform the conversion.
-            DataSource sqlite = new SQLite(settings);
+     */
-            for (PlayerAuth auth : database.getAllAuths()) {
+    @Override
-                auth.setRealName("Player");
+    public void run() {
-                sqlite.saveAuth(auth);
+        List<String> skippedPlayers = new ArrayList<>();
+        for (PlayerAuth auth : source.getAllAuths()) {
+            if (destination.isAuthAvailable(auth.getNickname())) {
+                skippedPlayers.add(auth.getNickname());
+            } else {
+                destination.saveAuth(auth);
+                destination.updateQuitLoc(auth);
             }
-            settings.setProperty(DatabaseSettings.BACKEND, DataSourceType.SQLITE);
-            settings.save();
-            ConsoleLogger.info("Database successfully converted to sqlite!");
-            return sqlite;
-        } catch (SQLException | ClassNotFoundException e) {
-            ConsoleLogger.logException("Could not convert from Flatfile to SQLite:", e);
         }
-        return null;
+
+        if (!skippedPlayers.isEmpty()) {
+            ConsoleLogger.showError("Warning: skipped conversion for players which were already in SQLite: "
+                + StringUtils.join(", ", skippedPlayers));
+        }
+        ConsoleLogger.info("Database successfully converted from " + source.getClass().getSimpleName()
+            + " to " + destination.getClass().getSimpleName());
     }
 }

src/main/java/fr/xephi/authme/datasource/FlatFile.java

@@ -1,5 +1,6 @@
 package fr.xephi.authme.datasource;
 
+import com.google.common.annotations.VisibleForTesting;
 import fr.xephi.authme.AuthMe;
 import fr.xephi.authme.ConsoleLogger;
 import fr.xephi.authme.cache.auth.PlayerAuth;
@@ -42,7 +43,7 @@ public class FlatFile implements DataSource {
         try {
             source.createNewFile();
         } catch (IOException e) {
-            ConsoleLogger.showError(e.getMessage());
+            ConsoleLogger.logException("Cannot open flatfile", e);
             if (Settings.isStopEnabled) {
                 ConsoleLogger.showError("Can't use FLAT FILE... SHUTDOWN...");
                 instance.getServer().shutdown();
@@ -50,10 +51,14 @@ public class FlatFile implements DataSource {
             if (!Settings.isStopEnabled) {
                 instance.getServer().getPluginManager().disablePlugin(instance);
             }
-            e.printStackTrace();
         }
     }
 
+    @VisibleForTesting
+    public FlatFile(File source) {
+        this.source = source;
+    }
+
     @Override
     public synchronized boolean isAuthAvailable(String user) {
         BufferedReader br = null;
@@ -601,7 +606,7 @@ public class FlatFile implements DataSource {
 
     @Override
     public boolean updateRealName(String user, String realName) {
-        return false;
+        throw new UnsupportedOperationException("Flat file no longer supported");
     }
 
     @Override
@@ -618,33 +623,25 @@ public class FlatFile implements DataSource {
             String line;
             while ((line = br.readLine()) != null) {
                 String[] args = line.split(":");
-                switch (args.length) {
+                // We expect to encounter 2, 3, 4, 7, 8 or 9 fields. Ignore the line otherwise
-                    case 2:
+                if (args.length >= 2 && args.length != 5 && args.length != 6 && args.length <= 9) {
-                        auths.add(new PlayerAuth(args[0], args[1], "192.168.0.1", 0, "your@email.com", args[0]));
+                    PlayerAuth.Builder builder = PlayerAuth.builder()
-                        break;
+                        .name(args[0]).realName(args[0])
-                    case 3:
+                        .password(args[1], null);
-                        auths.add(new PlayerAuth(args[0], args[1], args[2], 0, "your@email.com", args[0]));
+                    if (args.length >= 3)   builder.ip(args[2]);
-                        break;
+                    if (args.length >= 4)   builder.lastLogin(Long.parseLong(args[3]));
-                    case 4:
+                    if (args.length >= 7) {
-                        auths.add(new PlayerAuth(args[0], args[1], args[2], Long.parseLong(args[3]), "your@email.com", args[0]));
+                        builder.locX(Double.parseDouble(args[4]))
-                        break;
+                            .locY(Double.parseDouble(args[5]))
-                    case 7:
+                            .locZ(Double.parseDouble(args[6]));
-                        auths.add(new PlayerAuth(args[0], args[1], args[2], Long.parseLong(args[3]), Double.parseDouble(args[4]), Double.parseDouble(args[5]), Double.parseDouble(args[6]), "unavailableworld", "your@email.com", args[0]));
+                    }
-                        break;
+                    if (args.length >= 8)   builder.locWorld(args[7]);
-                    case 8:
+                    if (args.length >= 9)   builder.email(args[8]);
-                        auths.add(new PlayerAuth(args[0], args[1], args[2], Long.parseLong(args[3]), Double.parseDouble(args[4]), Double.parseDouble(args[5]), Double.parseDouble(args[6]), args[7], "your@email.com", args[0]));
+                    auths.add(builder.build());
-                        break;
-                    case 9:
-                        auths.add(new PlayerAuth(args[0], args[1], args[2], Long.parseLong(args[3]), Double.parseDouble(args[4]), Double.parseDouble(args[5]), Double.parseDouble(args[6]), args[7], args[8], args[0]));
-                        break;
                 }
             }
-        } catch (FileNotFoundException ex) {
-            ConsoleLogger.showError(ex.getMessage());
-            return auths;
         } catch (IOException ex) {
-            ConsoleLogger.showError(ex.getMessage());
+            ConsoleLogger.logException("Error while getting auths from flatfile:", ex);
-            return auths;
         } finally {
             if (br != null) {
                 try {
@@ -658,7 +655,7 @@ public class FlatFile implements DataSource {
 
     @Override
     public List<PlayerAuth> getLoggedPlayers() {
-        return new ArrayList<>();
+        throw new UnsupportedOperationException("Flat file no longer supported");
     }
 
     @Override

src/main/java/fr/xephi/authme/util/MigrationService.java

@@ -5,6 +5,8 @@ import fr.xephi.authme.cache.auth.PlayerAuth;
 import fr.xephi.authme.converter.ForceFlatToSqlite;
 import fr.xephi.authme.datasource.DataSource;
 import fr.xephi.authme.datasource.DataSourceType;
+import fr.xephi.authme.datasource.FlatFile;
+import fr.xephi.authme.datasource.SQLite;
 import fr.xephi.authme.security.HashAlgorithm;
 import fr.xephi.authme.security.crypts.HashedPassword;
 import fr.xephi.authme.security.crypts.SHA256;
@@ -58,12 +60,17 @@ public final class MigrationService {
         if (DataSourceType.FILE == settings.getProperty(DatabaseSettings.BACKEND)) {
             ConsoleLogger.showError("FlatFile backend has been detected and is now deprecated; it will be changed "
                 + "to SQLite... Connection will be impossible until conversion is done!");
-            ForceFlatToSqlite converter = new ForceFlatToSqlite(dataSource, settings);
+            FlatFile flatFile = (FlatFile) dataSource;
-            DataSource result = converter.run();
+            try {
-            if (result == null) {
+                SQLite sqlite = new SQLite(settings);
-                throw new IllegalStateException("Error during conversion from flatfile to SQLite");
+                ForceFlatToSqlite converter = new ForceFlatToSqlite(flatFile, sqlite);
-            } else {
+                converter.run();
-                return result;
+                settings.setProperty(DatabaseSettings.BACKEND, DataSourceType.SQLITE);
+                settings.save();
+                return sqlite;
+            } catch (Exception e) {
+                ConsoleLogger.logException("Error during conversion from Flatfile to SQLite", e);
+                throw new IllegalStateException(e);
             }
         }
         return null;

src/test/java/fr/xephi/authme/AuthMeMatchers.java

@@ -1,4 +1,4 @@
-package fr.xephi.authme.datasource;
+package fr.xephi.authme;
 
 import fr.xephi.authme.cache.auth.PlayerAuth;
 import fr.xephi.authme.security.crypts.HashedPassword;

src/test/java/fr/xephi/authme/converter/ForceFlatToSqliteTest.java

@@ -0,0 +1,72 @@
+package fr.xephi.authme.converter;
+
+import com.google.common.io.Files;
+import fr.xephi.authme.ConsoleLoggerTestInitializer;
+import fr.xephi.authme.TestHelper;
+import fr.xephi.authme.cache.auth.PlayerAuth;
+import fr.xephi.authme.datasource.DataSource;
+import fr.xephi.authme.datasource.FlatFile;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static fr.xephi.authme.AuthMeMatchers.hasAuthBasicData;
+import static fr.xephi.authme.AuthMeMatchers.hasAuthLocation;
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Test for {@link ForceFlatToSqlite}.
+ */
+public class ForceFlatToSqliteTest {
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private FlatFile flatFile;
+
+    @BeforeClass
+    public static void setup() {
+        ConsoleLoggerTestInitializer.setupLogger();
+    }
+
+    @Before
+    public void copyFile() throws IOException {
+        File source = TestHelper.getJarFile("/datasource-integration/flatfile-test.txt");
+        File destination = temporaryFolder.newFile();
+        Files.copy(source, destination);
+        flatFile = new FlatFile(destination);
+    }
+
+    @Test
+    public void shouldConvertToSqlite() {
+        // given
+        DataSource dataSource = mock(DataSource.class);
+        ForceFlatToSqlite converter = new ForceFlatToSqlite(flatFile, dataSource);
+
+        // when
+        converter.run();
+
+        // then
+        ArgumentCaptor<PlayerAuth> authCaptor = ArgumentCaptor.forClass(PlayerAuth.class);
+        verify(dataSource, times(7)).saveAuth(authCaptor.capture());
+        List<PlayerAuth> auths = authCaptor.getAllValues();
+        assertThat(auths, hasItem(hasAuthBasicData("bobby", "Bobby", "your@email.com", "123.45.67.89")));
+        assertThat(auths, hasItem(hasAuthLocation(1.05, 2.1, 4.2, "world")));
+        assertThat(auths, hasItem(hasAuthBasicData("user", "user", "user@example.org", "34.56.78.90")));
+        assertThat(auths, hasItem(hasAuthLocation(124.1, 76.3, -127.8, "nether")));
+        assertThat(auths, hasItem(hasAuthBasicData("eightfields", "eightFields", "your@email.com", "6.6.6.66")));
+        assertThat(auths, hasItem(hasAuthLocation(8.8, 17.6, 26.4, "eightworld")));
+    }
+
+}

src/test/java/fr/xephi/authme/datasource/AbstractDataSourceIntegrationTest.java

@@ -7,9 +7,9 @@ import org.junit.Test;
 import java.util.Arrays;
 import java.util.List;
 
-import static fr.xephi.authme.datasource.AuthMeMatchers.equalToHash;
+import static fr.xephi.authme.AuthMeMatchers.equalToHash;
-import static fr.xephi.authme.datasource.AuthMeMatchers.hasAuthBasicData;
+import static fr.xephi.authme.AuthMeMatchers.hasAuthBasicData;
-import static fr.xephi.authme.datasource.AuthMeMatchers.hasAuthLocation;
+import static fr.xephi.authme.AuthMeMatchers.hasAuthLocation;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
@@ -129,15 +129,9 @@ public abstract class AbstractDataSourceIntegrationTest {
         // then
         assertThat(response, equalTo(true));
         assertThat(authList, hasSize(2));
+        assertThat(authList, hasItem(hasAuthBasicData("bobby", "Bobby", "your@email.com", "123.45.67.89")));
         assertThat(newAuthList, hasSize(3));
-        boolean hasBobby = false;
+        assertThat(newAuthList, hasItem(hasAuthBasicData("bobby", "Bobby", "your@email.com", "123.45.67.89")));
-        for (PlayerAuth auth : authList) {
-            if (auth.getNickname().equals("bobby")) {
-                hasBobby = true;
-                break;
-            }
-        }
-        assertThat(hasBobby, equalTo(true));
     }
 
     @Test

src/test/java/fr/xephi/authme/datasource/FlatFileIntegrationTest.java

@@ -0,0 +1,100 @@
+package fr.xephi.authme.datasource;
+
+import com.google.common.io.Files;
+import fr.xephi.authme.TestHelper;
+import fr.xephi.authme.cache.auth.PlayerAuth;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import static fr.xephi.authme.AuthMeMatchers.equalToHash;
+import static fr.xephi.authme.AuthMeMatchers.hasAuthBasicData;
+import static fr.xephi.authme.AuthMeMatchers.hasAuthLocation;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Integration test for the deprecated {@link FlatFile} datasource. The flatfile datasource is no longer used.
+ * Essentially, the only time we use it is in {@link fr.xephi.authme.converter.ForceFlatToSqlite},
+ * which requires {@link FlatFile#getAllAuths()}.
+ */
+public class FlatFileIntegrationTest {
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private DataSource dataSource;
+
+    @Before
+    public void copyFileToTemporaryFolder() throws IOException {
+        File originalFile = TestHelper.getJarFile("/datasource-integration/flatfile-test.txt");
+        File copy = temporaryFolder.newFile();
+        Files.copy(originalFile, copy);
+        dataSource = new FlatFile(copy);
+    }
+
+    @Test
+    public void shouldReturnIfAuthIsAvailableOrNot() {
+        // given / when
+        boolean isBobbyAvailable = dataSource.isAuthAvailable("bobby");
+        boolean isChrisAvailable = dataSource.isAuthAvailable("chris");
+        boolean isUserAvailable = dataSource.isAuthAvailable("USER");
+
+        // then
+        assertThat(isBobbyAvailable, equalTo(true));
+        assertThat(isChrisAvailable, equalTo(false));
+        assertThat(isUserAvailable, equalTo(true));
+    }
+
+    @Test
+    public void shouldReturnAllAuths() {
+        // given / when
+        List<PlayerAuth> authList = dataSource.getAllAuths();
+
+        // then
+        assertThat(authList, hasSize(7));
+        assertThat(getName("bobby", authList), hasAuthBasicData("bobby", "Bobby", "your@email.com", "123.45.67.89"));
+        assertThat(getName("bobby", authList), hasAuthLocation(1.05, 2.1, 4.2, "world"));
+        assertThat(getName("bobby", authList).getPassword(), equalToHash("$SHA$11aa0706173d7272$dbba966"));
+        assertThat(getName("twofields", authList), hasAuthBasicData("twofields", "twoFields", "your@email.com", "127.0.0.1"));
+        assertThat(getName("twofields", authList).getPassword(), equalToHash("hash1234"));
+        assertThat(getName("threefields", authList), hasAuthBasicData("threefields", "threeFields", "your@email.com", "33.33.33.33"));
+        assertThat(getName("fourfields", authList), hasAuthBasicData("fourfields", "fourFields", "your@email.com", "4.4.4.4"));
+        assertThat(getName("fourfields", authList).getLastLogin(), equalTo(404040404L));
+        assertThat(getName("sevenfields", authList), hasAuthLocation(7.7, 14.14, 21.21, "world"));
+        assertThat(getName("eightfields", authList), hasAuthLocation(8.8, 17.6, 26.4, "eightworld"));
+        assertThat(getName("eightfields", authList).getLastLogin(), equalTo(1234567888L));
+        assertThat(getName("eightfields", authList).getPassword(), equalToHash("hash8168"));
+    }
+
+    @Test
+    public void shouldAddAuth() {
+        // given / when
+        boolean response = dataSource.saveAuth(
+            PlayerAuth.builder().name("Test").email("user@EXAMPLE.org").ip("123.45.67.77").build());
+        List<PlayerAuth> authList = dataSource.getAllAuths();
+
+        // then
+        assertThat(response, equalTo(true));
+        assertThat(authList, hasSize(8));
+        assertThat(authList, hasItem(hasAuthBasicData("test", "test", "user@EXAMPLE.org", "123.45.67.77")));
+    }
+
+    private static PlayerAuth getName(String name, Collection<PlayerAuth> auths) {
+        for (PlayerAuth auth : auths) {
+            if (name.equals(auth.getNickname())) {
+                return auth;
+            }
+        }
+        throw new IllegalStateException("Did not find auth with name '" + name + "'");
+    }
+
+}

src/test/resources/datasource-integration/flatfile-test.txt

@@ -0,0 +1,7 @@
+Bobby:$SHA$11aa0706173d7272$dbba966:123.45.67.89:1449136800:1.05:2.1:4.2:world:your@email.com
+user:b28c32f624a4eb161d6adc9acb5bfc5b:34.56.78.90:1453242857:124.1:76.3:-127.8:nether:user@example.org
+twoFields:hash1234
+threeFields:hash369:33.33.33.33
+fourFields:$hash$4444:4.4.4.4:404040404
+sevenFields:hash7749:5.5.5.55:1414141414:7.7:14.14:21.21
+eightFields:hash8168:6.6.6.66:1234567888:8.8:17.6:26.4:eightworld