Improved DB connection handling

Support for aggressive connection timeouts, with exponential backoff
for multiple failures.

src/main/java/com/gmail/nossr50/runnables/SQLReconnect.java

@@ -15,9 +15,7 @@ public class SQLReconnect implements Runnable {
 
     @Override
     public void run() {
-        if (!Database.isConnected()) {
+        if (Database.checkConnection()) {
-            Database.connect();
-            if (Database.isConnected()) {
             Users.saveAll(); //Save all profiles
             Users.clearAll(); //Clear the profiles
 
@@ -26,5 +24,4 @@ public class SQLReconnect implements Runnable {
             }
         }
     }
-    }
 }

src/main/java/com/gmail/nossr50/util/Database.java

@@ -22,24 +22,28 @@ public class Database {
     private static String tablePrefix = configInstance.getMySQLTablePrefix();
     private static Connection connection = null;
     private static mcMMO plugin = null;
-    private static long reconnectTimestamp = 0;
+
+    // Scale waiting time by this much per failed attempt
+    private static final double SCALING_FACTOR = 5;
+    
+    // Minimum wait in nanoseconds (default 500ms)
+    private static final long MIN_WAIT = 500*100000L;
+
+    // Maximum time to wait between reconnects (default 5 minutes)
+    private static final long MAX_WAIT = 5*60000000000L;
+
+    // How long to wait when checking if connection is valid (default 3 seconds)
+    private static final int VALID_TIMEOUT = 3;
+
+    // When next to try connecting to Database in nanoseconds
+    private static long nextReconnectTimestamp = 0L;
+    
+    // How many connection attemtps have failed
+    private static int reconnectAttempt = 0;
 
     public Database(mcMMO instance) {
         plugin = instance;
-        connect(); //Connect to MySQL
+        checkConnected(); //Connect to MySQL
-
-        // Load the driver instance
-        try {
-            Class.forName("com.mysql.jdbc.Driver");
-            DriverManager.getConnection(connectionString);
-        }
-        catch (ClassNotFoundException e) {
-            plugin.getLogger().warning(e.getLocalizedMessage());
-        }
-        catch (SQLException ex) {
-            plugin.getLogger().warning(ex.getLocalizedMessage());
-            printErrors(ex);
-        }
     }
 
     /**
@@ -49,6 +53,8 @@ public class Database {
         try {
             System.out.println("[mcMMO] Attempting connection to MySQL...");
 
+            // Force driver to load if not yet loaded
+            Class.forName("com.mysql.jdbc.Driver");
             Properties connectionProperties = new Properties();
             connectionProperties.put("autoReconnect", "false");
             connectionProperties.put("maxReconnects", "0");
@@ -57,9 +63,14 @@ public class Database {
             System.out.println("[mcMMO] Connection to MySQL was a success!");
         }
         catch (SQLException ex) {
+        	connection = null;
             System.out.println("[mcMMO] Connection to MySQL failed!");
             ex.printStackTrace();
             printErrors(ex);
+        } catch (ClassNotFoundException ex) {
+        	connection = null;
+            System.out.println("[mcMMO] MySQL database driver not found!");
+            ex.printStackTrace();
 		}
     }
 
@@ -185,7 +196,7 @@ public class Database {
      * @return true if the query was successfully written, false otherwise.
      */
     public boolean write(String sql) {
-        if (isConnected()) {
+        if (checkConnected()) {
             try {
                 PreparedStatement statement = connection.prepareStatement(sql);
                 statement.executeUpdate();
@@ -197,9 +208,6 @@ public class Database {
                 return false;
             }
         }
-        else {
-            attemptReconnect();
-        }
 
         return false;
     }
@@ -214,7 +222,7 @@ public class Database {
         ResultSet resultSet;
         int result = 0;
 
-        if (isConnected()) {
+        if (checkConnected()) {
             try {
                 PreparedStatement statement = connection.prepareStatement(sql);
                 resultSet = statement.executeQuery();
@@ -232,45 +240,101 @@ public class Database {
                 printErrors(ex);
             }
         }
-        else {
-            attemptReconnect();
-        }
 
         return result;
     }
 
     /**
-     * Get connection status
+     * Check connection status and re-establish if dead or stale.
+     *
+     * If the very first immediate attempt fails, further attempts 
+     * will be made in progressively larger intervals up to MAX_WAIT 
+     * intervals. 
+     *
+     * This allows for MySQL to time out idle connections as needed by 
+     * server operator, without affecting McMMO, while still providing 
+     * protection against a database outage taking down Bukkit's tick  
+     * processing loop due to attemping a database connection each
+     * time McMMO needs the database.
      *
      * @return the boolean value for whether or not we are connected
      */
-    public static boolean isConnected() {
+    public static boolean checkConnected() {
-        if (connection == null) {
+    	boolean isClosed = true;
+    	boolean isValid = false;
+    	boolean exists = (connection != null);
+
+    	// Initialized as needed later
+    	long timestamp=0;
+    	
+    	// If we're waiting for server to recover then leave early
+    	if (nextReconnectTimestamp > 0 && nextReconnectTimestamp > System.nanoTime()) {
     		return false;
     	}
     	
+    	if (exists) {
             try {
-            return connection.isValid(3);
+            	isClosed = connection.isClosed();
+    		} catch (SQLException e) {
+    			isClosed = true;
+    			e.printStackTrace();
+    			printErrors(e);
     		}
-        catch (SQLException e) {
+            
+            if (!isClosed) {
+                try {
+                	isValid = connection.isValid(VALID_TIMEOUT);
+        		} catch (SQLException e) {
+        			// Don't print stack trace because it's valid to lose idle connections 
+        			// to the server and have to restart them.
+        			isValid = false;
+        		}
+            }
+    	}
+
+    	// Leave if all ok
+    	if (exists && !isClosed && isValid) {
+    		// Housekeeping
+    		nextReconnectTimestamp = 0;
+			reconnectAttempt = 0;
+    		return true;
+    	}
+ 
+    	// Cleanup after ourselves for GC and MySQL's sake
+    	if (exists && !isClosed) {
+    		try { 
+    			connection.close();
+    		} catch (SQLException ex) {
+    			// This is a housekeeping exercise, ignore errors
+    		}
+    	}
+
+    	// Try to connect again
+    	connect();
+
+    	// Leave if connection is good
+    	try {
+			if (connection != null && !connection.isClosed()) {
+				// Schedule a database save if we really had an outage
+				if (reconnectAttempt > 1) {
+		            plugin.getServer().getScheduler().scheduleSyncDelayedTask(plugin, new SQLReconnect(plugin), 5);
+				}
+				nextReconnectTimestamp = 0;
+				reconnectAttempt = 0;
+				return true;
+			}
+		} catch (SQLException e) {
+			// Failed to check isClosed, so presume connection is bad and attempt later
+			e.printStackTrace();
+			printErrors(e);
+		}
+
+    	reconnectAttempt++;
+    		
+   		nextReconnectTimestamp = (long)(System.nanoTime() + Math.min(MAX_WAIT, (reconnectAttempt*SCALING_FACTOR*MIN_WAIT)));
+   		
    		return false;
     }
-    }
-
-    /**
-     * Schedules a Sync Delayed Task with the Bukkit Scheduler to attempt reconnection after a minute has elapsed
-     * This will check for a connection being present or not to prevent unneeded reconnection attempts
-     */
-    public static void attemptReconnect() {
-        final int RECONNECT_WAIT_TICKS = 60000;
-        final int RECONNECT_DELAY_TICKS = 1200;
-
-        if (reconnectTimestamp + RECONNECT_WAIT_TICKS < System.currentTimeMillis()) {
-            System.out.println("[mcMMO] Connection to MySQL was lost! Attempting to reconnect in 60 seconds..."); //Only reconnect if another attempt hasn't been made recently
-            reconnectTimestamp = System.currentTimeMillis();
-            plugin.getServer().getScheduler().scheduleSyncDelayedTask(plugin, new SQLReconnect(plugin), RECONNECT_DELAY_TICKS);
-        }
-    }
 
     /**
      * Read SQL query.
@@ -282,7 +346,7 @@ public class Database {
         ResultSet resultSet;
         HashMap<Integer, ArrayList<String>> rows = new HashMap<Integer, ArrayList<String>>();
 
-        if (isConnected()) {
+        if (checkConnected()) {
             try {
                 PreparedStatement statement = connection.prepareStatement(sql);
                 resultSet = statement.executeQuery();
@@ -303,9 +367,6 @@ public class Database {
                 printErrors(ex);
             }
         }
-        else {
-            attemptReconnect();
-        }
 
         return rows;
     }