Attempt to reduce load when lock wait timeout is exceeded

- Delay is dynamically adjusted if the exception occurs again
- The transaction is attempted again

Affects issues:
- Possibly fixed #1546

Plan/common/src/main/java/com/djrapitops/plan/storage/database/AbstractDatabase.java

@@ -16,6 +16,8 @@
  */
 package com.djrapitops.plan.storage.database;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * Abstract class representing a Database.
  * <p>
@@ -27,6 +29,7 @@ public abstract class AbstractDatabase implements Database {
 
     protected final DBAccessLock accessLock;
     private State state;
+    private final AtomicInteger heavyLoadDelayMs = new AtomicInteger(0);
 
     public AbstractDatabase() {
         state = State.CLOSED;
@@ -42,4 +45,20 @@ public abstract class AbstractDatabase implements Database {
         this.state = state;
         accessLock.operabilityChanged();
     }
+
+    public boolean isUnderHeavyLoad() {
+        return heavyLoadDelayMs.get() != 0;
+    }
+
+    public void increaseHeavyLoadDelay() {
+        heavyLoadDelayMs.incrementAndGet();
+    }
+
+    public void assumeNoMoreHeavyLoad() {
+        this.heavyLoadDelayMs.set(0);
+    }
+
+    public int getHeavyLoadDelayMs() {
+        return heavyLoadDelayMs.get();
+    }
 }

Plan/common/src/main/java/com/djrapitops/plan/storage/database/SQLDB.java

@@ -301,4 +301,12 @@ public abstract class SQLDB extends AbstractDatabase {
     public void setTransactionExecutorServiceProvider(Supplier<ExecutorService> transactionExecutorServiceProvider) {
         this.transactionExecutorServiceProvider = transactionExecutorServiceProvider;
     }
+
+    public RunnableFactory getRunnableFactory() {
+        return runnableFactory;
+    }
+
+    public PluginLogger getLogger() {
+        return logger;
+    }
 }

Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/ThrowawayTransaction.java

@@ -29,6 +29,6 @@ public abstract class ThrowawayTransaction extends Transaction {
 
     @Override
     protected boolean shouldBeExecuted() {
-        return getDBState() != Database.State.CLOSING;
+        return getDBState() != Database.State.CLOSING && dbIsNotUnderHeavyLoad();
     }
 }

Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/Transaction.java

@@ -22,10 +22,13 @@ import com.djrapitops.plan.storage.database.Database;
 import com.djrapitops.plan.storage.database.SQLDB;
 import com.djrapitops.plan.storage.database.queries.Query;
 import com.djrapitops.plan.utilities.logging.ErrorContext;
+import com.djrapitops.plugin.api.TimeAmount;
+import com.djrapitops.plugin.task.AbsRunnable;
 import com.djrapitops.plugin.utilities.Verify;
 
 import java.sql.*;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -68,6 +71,15 @@ public abstract class Transaction {
 
         attempts++; // Keeps track how many attempts have been made to avoid infinite recursion.
 
+        if (db.isUnderHeavyLoad()) {
+            try {
+                Thread.sleep(db.getHeavyLoadDelayMs());
+                Thread.yield();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+
         try {
             initializeTransaction(db);
             performOperations();
@@ -94,6 +106,22 @@ public abstract class Transaction {
             executeTransaction(db); // Recurse to attempt again.
             return;
         }
+
+        if (dbType == DBType.MYSQL && errorCode == 1205) {
+            if (!db.isUnderHeavyLoad()) {
+                db.getLogger().warn("Database appears to be under heavy load. Dropping some unimportant transactions and adding short pauses for next 10 minutes.");
+                db.getRunnableFactory().create("Increase load", new AbsRunnable() {
+                    @Override
+                    public void run() {
+                        db.assumeNoMoreHeavyLoad();
+                    }
+                }).runTaskLaterAsynchronously(TimeAmount.toTicks(10, TimeUnit.MINUTES));
+            }
+            db.increaseHeavyLoadDelay();
+            executeTransaction(db); // Recurse to attempt again.
+            return;
+        }
+
         if (attempts >= ATTEMPT_LIMIT) {
             failMsg += " (Attempted " + attempts + " times)";
         }
@@ -220,4 +248,8 @@ public abstract class Transaction {
     public boolean wasSuccessful() {
         return success;
     }
+
+    public boolean dbIsNotUnderHeavyLoad() {
+        return !db.isUnderHeavyLoad();
+    }
 }