#514 Messages tool adds missing messages as comments

- Add the English text as a TODO comment for all missing messages in the translations
- Add a TODO comment for missing tags
- Change the task and messages verifier to use YamlConfiguration

src/tools/messages/MessageFileVerifier.java

@@ -1,16 +1,21 @@
 package messages;
 
+import com.google.common.base.Predicate;
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
 import fr.xephi.authme.output.MessageKey;
 import fr.xephi.authme.util.StringUtils;
+import org.bukkit.configuration.file.FileConfiguration;
+import org.bukkit.configuration.file.YamlConfiguration;
 import utils.FileUtils;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -21,8 +26,6 @@ import java.util.Set;
  */
 public class MessageFileVerifier {
 
-    private static final String NEW_LINE = "\n";
-
     private final String messagesFile;
     private final Set<String> unknownKeys = new HashSet<>();
     // Map with the missing key and a boolean indicating whether or not it was added to the file by this object
@@ -69,47 +72,30 @@ public class MessageFileVerifier {
     }
 
     private void verifyKeys() {
-        List<MessageKey> messageKeys = getAllMessageKeys();
+        FileConfiguration configuration = YamlConfiguration.loadConfiguration(new File(messagesFile));
-        List<String> fileLines = readFileLines();
+
-        for (String line : fileLines) {
+        // Check known keys (their existence + presence of all tags)
-            // Skip comments and empty lines
+        for (MessageKey messageKey : MessageKey.values()) {
-            if (!line.startsWith("#") && !line.trim().isEmpty()) {
+            final String key = messageKey.getKey();
-                processKeyInFile(line, messageKeys);
+            if (configuration.isString(key)) {
+                checkTagsInMessage(messageKey, configuration.getString(key));
+            } else {
+                missingKeys.put(key, false);
             }
         }
 
-        // All keys that remain are keys that are absent in the file
+        // Check FileConfiguration for all of its keys to find unknown keys
-        for (MessageKey missingKey : messageKeys) {
+        for (String key : configuration.getValues(true).keySet()) {
-            missingKeys.put(missingKey.getKey(), false);
+            if (!messageKeyExists(key)) {
+                unknownKeys.add(key);
+            }
         }
     }
 
-    private void processKeyInFile(String line, List<MessageKey> messageKeys) {
+    private void checkTagsInMessage(MessageKey messageKey, String message) {
-        if (line.indexOf(':') == -1) {
+        for (String tag : messageKey.getTags()) {
-            System.out.println("Skipping line in unknown format: '" + line + "'");
-            return;
-        }
-
-        final String readKey = line.substring(0, line.indexOf(':'));
-        boolean foundKey = false;
-        for (Iterator<MessageKey> it = messageKeys.iterator(); it.hasNext(); ) {
-            MessageKey messageKey = it.next();
-            if (messageKey.getKey().equals(readKey)) {
-                checkTagsInMessage(readKey, line.substring(line.indexOf(':')), messageKey.getTags());
-                it.remove();
-                foundKey = true;
-                break;
-            }
-        }
-        if (!foundKey) {
-            unknownKeys.add(readKey);
-        }
-    }
-
-    private void checkTagsInMessage(String key, String message, String[] tags) {
-        for (String tag : tags) {
             if (!message.contains(tag)) {
-                missingTags.put(key, tag);
+                missingTags.put(messageKey.getKey(), tag);
             }
         }
     }
@@ -119,54 +105,90 @@ public class MessageFileVerifier {
      *
      * @param defaultMessages The collection of default messages
      */
-    public void addMissingKeys(Map<String, String> defaultMessages) {
+    public void addMissingKeys(FileConfiguration defaultMessages) {
+        final List<String> fileLines = new ArrayList<>(
+            Arrays.asList(FileUtils.readFromFile(messagesFile).split("\\n")));
+
         List<String> keysToAdd = new ArrayList<>();
         for (Map.Entry<String, Boolean> entry : missingKeys.entrySet()) {
-            if (Boolean.FALSE.equals(entry.getValue()) && defaultMessages.get(entry.getKey()) != null) {
+            final String key = entry.getKey();
-                keysToAdd.add(entry.getKey());
+
+            if (Boolean.FALSE.equals(entry.getValue()) && defaultMessages.get(key) != null) {
+                keysToAdd.add(key);
             }
         }
 
-        // We know that all keys in keysToAdd are safe to retrieve and write
+        // Add missing keys as comments to the bottom of the file
-        StringBuilder sb = new StringBuilder(NEW_LINE);
         for (String keyToAdd : keysToAdd) {
-            sb.append(keyToAdd).append(":").append(defaultMessages.get(keyToAdd)).append(NEW_LINE);
+            int indexOfComment = Iterables.indexOf(fileLines, isCommentFor(keyToAdd));
-            missingKeys.put(keyToAdd, true);
+            if (indexOfComment != -1) {
+                // Comment for keyToAdd already exists, so remove it since we're going to add it
+                fileLines.remove(indexOfComment);
             }
-        FileUtils.appendToFile(messagesFile, sb.toString());
+            String comment = commentForKey(keyToAdd) + "'" +
+                defaultMessages.getString(keyToAdd).replace("'", "''") + "'";
+            fileLines.add(comment);
+            missingKeys.put(keyToAdd, Boolean.TRUE);
         }
 
-    private static List<MessageKey> getAllMessageKeys() {
+        // Add a comment above messages missing a tag
-        return new ArrayList<>(Arrays.asList(MessageKey.values()));
+        for (Map.Entry<String, Collection<String>> entry : missingTags.asMap().entrySet()) {
+            final String key = entry.getKey();
+            addCommentForMissingTags(fileLines, key, entry.getValue());
+        }
+
+        FileUtils.writeToFile(messagesFile, StringUtils.join("\n", fileLines));
     }
 
     /**
-     * Read all lines from the messages file and skip empty lines and comment lines.
+     * Add a comment above a message to note the tags the message is missing. Removes
-     * This method appends lines starting with two spaces to the previously read line,
+     * any similar comment that may already be above the message.
-     * akin to a YAML parser.
+     *
+     * @param fileLines The lines of the file (to modify)
+     * @param key The key of the message
+     * @param tags The missing tags
      */
-    private List<String> readFileLines() {
+    private void addCommentForMissingTags(List<String> fileLines, final String key, Collection<String> tags) {
-        String[] rawLines = FileUtils.readFromFile(messagesFile).split("\\n");
+        int indexForComment = Iterables.indexOf(fileLines, isCommentFor(key));
-        List<String> lines = new ArrayList<>();
+        if (indexForComment == -1) {
-        for (String line : rawLines) {
+            indexForComment = Iterables.indexOf(fileLines, new Predicate<String>() {
-            // Skip comments and empty lines
+                @Override
-            if (!line.startsWith("#") && !StringUtils.isEmpty(line)) {
+                public boolean apply(String input) {
-                // Line is indented, i.e. it needs to be appended to the previous line
+                    return input.startsWith(key + ": ");
-                if (line.startsWith("  ")) {
+                }
-                    appendToLastElement(lines, line.substring(1));
+            });
+            if (indexForComment == -1) {
+                System.err.println("Error adding comment for key '" + key + "': couldn't find entry in file lines");
+                return;
+            }
         } else {
-                    lines.add(line);
+            fileLines.remove(indexForComment);
-                }
-            }
-        }
-        return lines;
         }
 
-    private static void appendToLastElement(List<String> list, String text) {
+        String tagWord = tags.size() > 1 ? "tags" : "tag";
-        if (list.isEmpty()) {
+        fileLines.add(indexForComment, commentForKey(key)
-            throw new IllegalStateException("List cannot be empty!");
+            + String.format("Missing %s %s", tagWord, StringUtils.join(", ", tags)));
     }
-        int lastIndex = list.size() - 1;
+
-        list.set(lastIndex, list.get(lastIndex).concat(text));
+    private static String commentForKey(String key) {
+        return String.format("# TODO %s: ", key);
+    }
+
+    private static Predicate<String> isCommentFor(final String key) {
+        return new Predicate<String>() {
+            @Override
+            public boolean apply(String input) {
+                return input.startsWith(commentForKey(key));
+            }
+        };
+    }
+
+    private static boolean messageKeyExists(String key) {
+        for (MessageKey messageKey : MessageKey.values()) {
+            if (messageKey.getKey().equals(key)) {
+                return true;
+            }
+        }
+        return false;
     }
 }

src/tools/messages/VerifyMessagesTask.java

@@ -2,14 +2,14 @@ package messages;
 
 import com.google.common.collect.Multimap;
 import fr.xephi.authme.util.StringUtils;
-import utils.FileUtils;
+import org.bukkit.configuration.file.FileConfiguration;
+import org.bukkit.configuration.file.YamlConfiguration;
 import utils.ToolTask;
 import utils.ToolsConstants;
 
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
@@ -29,6 +29,8 @@ public final class VerifyMessagesTask implements ToolTask {
     private static final Pattern MESSAGE_FILE_PATTERN = Pattern.compile("messages_[a-z]{2,7}\\.yml");
     /** Tag that is replaced to the messages folder in user input. */
     private static final String SOURCES_TAG = "{msgdir}";
+    /** File to get default messages from (assumes that it is complete). */
+    private static final String DEFAULT_MESSAGES_FILE = MESSAGES_FOLDER + "messages_en.yml";
 
     @Override
     public String getTaskName() {
@@ -43,13 +45,7 @@ public final class VerifyMessagesTask implements ToolTask {
         String inputFile = scanner.nextLine();
 
         System.out.println("Add any missing keys to files? ['y' = yes]");
-        boolean addMissingKeys = "y".equals(scanner.nextLine());
+        boolean addMissingKeys = "y".equalsIgnoreCase(scanner.nextLine());
-
-        // Set up needed objects
-        Map<String, String> defaultMessages = null;
-        if (addMissingKeys) {
-            defaultMessages = constructDefaultMessages();
-        }
 
         List<File> messageFiles;
         if (StringUtils.isEmpty(inputFile)) {
@@ -59,6 +55,11 @@ public final class VerifyMessagesTask implements ToolTask {
             messageFiles = Collections.singletonList(customFile);
         }
 
+        FileConfiguration defaultMessages = null;
+        if (addMissingKeys) {
+            defaultMessages = YamlConfiguration.loadConfiguration(new File(DEFAULT_MESSAGES_FILE));
+        }
+
         // Verify the given files
         for (File file : messageFiles) {
             System.out.println("Verifying '" + file.getName() + "'");
@@ -92,19 +93,18 @@ public final class VerifyMessagesTask implements ToolTask {
         }
     }
 
-    private static void verifyFileAndAddKeys(MessageFileVerifier verifier, Map<String, String> defaultMessages) {
+    private static void verifyFileAndAddKeys(MessageFileVerifier verifier, FileConfiguration defaultMessages) {
         Map<String, Boolean> missingKeys = verifier.getMissingKeys();
-        if (!missingKeys.isEmpty()) {
+        if (!missingKeys.isEmpty() || !verifier.getMissingTags().isEmpty()) {
             verifier.addMissingKeys(defaultMessages);
-            missingKeys = verifier.getMissingKeys();
             List<String> addedKeys = getKeysWithValue(Boolean.TRUE, missingKeys);
             System.out.println("  Added missing keys " + addedKeys);
 
             List<String> unsuccessfulKeys = getKeysWithValue(Boolean.FALSE, missingKeys);
             if (!unsuccessfulKeys.isEmpty()) {
-                System.out.println("  Warning! Could not add all missing keys (problem with loading " +
+                System.err.println("  Warning! Could not add all missing keys (problem with loading " +
                     "default messages?)");
-                System.out.println("  Could not add keys " + unsuccessfulKeys);
+                System.err.println("  Could not add keys " + unsuccessfulKeys);
             }
         }
 
@@ -119,24 +119,6 @@ public final class VerifyMessagesTask implements ToolTask {
         }
     }
 
-    private static Map<String, String> constructDefaultMessages() {
-        String defaultMessagesFile = MESSAGES_FOLDER + "messages_en.yml";
-        List<String> lines = FileUtils.readLinesFromFile(defaultMessagesFile);
-        Map<String, String> messages = new HashMap<>(lines.size());
-        for (String line : lines) {
-            if (line.startsWith("#") || line.trim().isEmpty()) {
-                continue;
-            }
-            if (line.indexOf(':') == -1 || line.indexOf(':') == line.length() - 1) {
-                System.out.println("Warning! Unknown format in default messages file for line '" + line + "'");
-            } else {
-                String key = line.substring(0, line.indexOf(':'));
-                messages.put(key, line.substring(line.indexOf(':') + 1));
-            }
-        }
-        return messages;
-    }
-
     private static <K, V> List<K> getKeysWithValue(V value, Map<K, V> map) {
         List<K> result = new ArrayList<>();
         for (Map.Entry<K, V> entry : map.entrySet()) {

src/tools/utils/FileUtils.java

@@ -24,7 +24,7 @@ public final class FileUtils {
         writeToFile(destinationFile, result);
     }
 
-    private static void writeToFile(String outputFile, String contents) {
+    public static void writeToFile(String outputFile, String contents) {
         try {
             Files.write(Paths.get(outputFile), contents.getBytes());
         } catch (IOException e) {