From 181310dc7f9744c6e7af24009e89a143eb46eacb Mon Sep 17 00:00:00 2001 From: Aurora Lahtela <24460436+AuroraLS3@users.noreply.github.com> Date: Thu, 20 Jul 2023 09:01:26 +0300 Subject: [PATCH] 3091/mariadb driver support (#3122) * Fix MariaDB 11 driver issue * Update some more logging messages * Throw DBInitException if using MariaDB 11.0.2 * Fix mariadb container health check * Use 11.1-rc for mariadb image Affects issues: - Fixed #3091 for MariaDB 11.1.1 or newer --- .github/workflows/ci.yml | 4 +- Plan/build.gradle | 6 +- Plan/common/build.gradle | 16 ++++- .../database/MariaDB11Exception.java | 29 +++++++++ .../plan/storage/database/MySQLDB.java | 59 ++++++++++++++++--- .../queries/schema/MySQLSchemaQueries.java | 8 +++ .../database/sql/tables/WorldTable.java | 4 +- 7 files changed, 108 insertions(+), 18 deletions(-) create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/exceptions/database/MariaDB11Exception.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac36d77fe..94cb03b49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: services: mariadb: - image: mariadb:10.6.14 + image: mariadb:11.1-rc ports: - 3306 env: @@ -19,7 +19,7 @@ jobs: MYSQL_PASSWORD: password MYSQL_DATABASE: test MYSQL_ROOT_PASSWORD: password - options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3 steps: - name: 📥 Checkout git repository diff --git a/Plan/build.gradle b/Plan/build.gradle index e9879c628..cad5cbb99 100644 --- a/Plan/build.gradle +++ b/Plan/build.gradle @@ -88,6 +88,7 @@ subprojects { jettyVersion = "11.0.15" caffeineVersion = "2.9.2" mysqlVersion = "8.0.33" + mariadbVersion = "3.1.4" sqliteVersion = "3.41.2.1" adventureVersion = "4.14.0" hikariVersion = "5.0.1" @@ -134,8 +135,9 @@ subprojects { // Awaitility (Concurrent wait conditions) // Testing dependencies required by Plan - testImplementation "org.xerial:sqlite-jdbc:$sqliteVersion" // SQLite - testImplementation "mysql:mysql-connector-java:$mysqlVersion" // MySQL + testImplementation "org.xerial:sqlite-jdbc:$sqliteVersion" // SQLite + testImplementation "com.mysql:mysql-connector-j:$mysqlVersion" // MySQL + testImplementation "org.mariadb.jdbc:mariadb-java-client:$mariadbVersion" // MariaDB } configurations { diff --git a/Plan/common/build.gradle b/Plan/common/build.gradle index 32cdbf568..6432c34a3 100644 --- a/Plan/common/build.gradle +++ b/Plan/common/build.gradle @@ -10,10 +10,11 @@ plugins { configurations { // Runtime downloading scopes mysqlDriver + mariadbDriver sqliteDriver ipAddressMatcher - testImplementation.extendsFrom mysqlDriver, sqliteDriver, ipAddressMatcher - compileOnly.extendsFrom mysqlDriver, sqliteDriver, ipAddressMatcher + testImplementation.extendsFrom mysqlDriver, mariadbDriver, sqliteDriver, ipAddressMatcher + compileOnly.extendsFrom mysqlDriver, mariadbDriver, sqliteDriver, ipAddressMatcher swaggerJson // swagger.json configuration } @@ -26,6 +27,14 @@ task generateResourceForMySQLDriver(type: GenerateDependencyDownloadResourceTask includeShadowJarRelocations = false } +task generateResourceForMariaDBDriver(type: GenerateDependencyDownloadResourceTask) { + var conf = configurations.mariadbDriver + configuration = conf + file = "assets/plan/dependencies/" + conf.name + ".txt" + // Not necessary to include in the resource + includeShadowJarRelocations = false +} + task generateResourceForSQLiteDriver(type: GenerateDependencyDownloadResourceTask) { var conf = configurations.sqliteDriver configuration = conf @@ -52,7 +61,8 @@ dependencies { // Effectively disables relocating exclude module: "jar-relocator" } - mysqlDriver "mysql:mysql-connector-java:$mysqlVersion" + mysqlDriver "com.mysql:mysql-connector-j:$mysqlVersion" + mariadbDriver "org.mariadb.jdbc:mariadb-java-client:$mariadbVersion" sqliteDriver "org.xerial:sqlite-jdbc:$sqliteVersion" ipAddressMatcher "com.github.seancfoley:ipaddress:$ipAddressMatcherVersion" diff --git a/Plan/common/src/main/java/com/djrapitops/plan/exceptions/database/MariaDB11Exception.java b/Plan/common/src/main/java/com/djrapitops/plan/exceptions/database/MariaDB11Exception.java new file mode 100644 index 000000000..f227c8a64 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/exceptions/database/MariaDB11Exception.java @@ -0,0 +1,29 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.exceptions.database; + +/** + * Exception thrown when MySQL driver can't connect to MariaDB. + * + * @author AuroraLS3 + */ +public class MariaDB11Exception extends DBInitException { + + public MariaDB11Exception(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/MySQLDB.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/MySQLDB.java index 03221aad3..8c091c8c9 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/MySQLDB.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/MySQLDB.java @@ -18,11 +18,14 @@ package com.djrapitops.plan.storage.database; import com.djrapitops.plan.exceptions.database.DBInitException; import com.djrapitops.plan.exceptions.database.DBOpException; +import com.djrapitops.plan.exceptions.database.MariaDB11Exception; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.settings.config.PlanConfig; import com.djrapitops.plan.settings.config.paths.DatabaseSettings; import com.djrapitops.plan.settings.locale.Locale; import com.djrapitops.plan.settings.locale.lang.PluginLang; +import com.djrapitops.plan.storage.database.queries.schema.MySQLSchemaQueries; +import com.djrapitops.plan.storage.database.transactions.init.OperationCriticalTransaction; import com.djrapitops.plan.storage.file.PlanFiles; import com.djrapitops.plan.utilities.logging.ErrorContext; import com.djrapitops.plan.utilities.logging.ErrorLogger; @@ -50,6 +53,8 @@ public class MySQLDB extends SQLDB { private static int increment = 1; + private static boolean useMariaDbDriver = false; + protected HikariDataSource dataSource; @Inject @@ -77,9 +82,10 @@ public class MySQLDB extends SQLDB { @Override protected List getDependencyResource() { try { - return files.getResourceFromJar("dependencies/mysqlDriver.txt").asLines(); + String driverFile = useMariaDbDriver ? "dependencies/mariadbDriver.txt" : "dependencies/mysqlDriver.txt"; + return files.getResourceFromJar(driverFile).asLines(); } catch (IOException e) { - throw new DBInitException("Failed to get MySQL dependency information", e); + throw new DBInitException("Failed to get " + (useMariaDbDriver ? "MariaDB" : "MySQL") + " dependency information", e); } } @@ -89,7 +95,7 @@ public class MySQLDB extends SQLDB { @Override public void setupDataSource() { if (driverClassLoader == null) { - logger.info("Downloading MySQL Driver, this may take a while..."); + logger.info("Downloading " + (useMariaDbDriver ? "MariaDB" : "MySQL") + " Driver, this may take a while..."); downloadDriver(); } @@ -99,6 +105,21 @@ public class MySQLDB extends SQLDB { // Set the context class loader to the driver class loader for Hikari to use for finding the Driver currentThread.setContextClassLoader(driverClassLoader); + try { + loadDataSource(); + } catch (MariaDB11Exception e) { + // Try to set up again using MariaDB driver + driverClassLoader = null; + dataSource = null; + useMariaDbDriver = true; + loadDataSource(); + } + + // Reset the context classloader back to what it was originally set to, now that the DataSource is created + currentThread.setContextClassLoader(previousClassLoader); + } + + private void loadDataSource() { try { HikariConfig hikariConfig = new HikariConfig(); @@ -107,12 +128,14 @@ public class MySQLDB extends SQLDB { String database = config.get(DatabaseSettings.MYSQL_DATABASE); String launchOptions = config.get(DatabaseSettings.MYSQL_LAUNCH_OPTIONS); // REGEX: match "?", match "word=word&" *-times, match "word=word" + if (launchOptions.isEmpty() || !launchOptions.matches("\\?((([\\w-])+=.+)&)*(([\\w-])+=.+)")) { launchOptions = "?rewriteBatchedStatements=true&useSSL=false"; logger.error(locale.getString(PluginLang.DB_MYSQL_LAUNCH_OPTIONS_FAIL, launchOptions)); } - hikariConfig.setDriverClassName("com.mysql.cj.jdbc.Driver"); - hikariConfig.setJdbcUrl("jdbc:mysql://" + host + ":" + port + "/" + database + launchOptions); + hikariConfig.setDriverClassName(useMariaDbDriver ? "org.mariadb.jdbc.Driver" : "com.mysql.cj.jdbc.Driver"); + String protocol = useMariaDbDriver ? "jdbc:mariadb" : "jdbc:mysql"; + hikariConfig.setJdbcUrl(protocol + "://" + host + ":" + port + "/" + database + launchOptions); String username = config.get(DatabaseSettings.MYSQL_USER); String password = config.get(DatabaseSettings.MYSQL_PASS); @@ -127,17 +150,34 @@ public class MySQLDB extends SQLDB { hikariConfig.setAutoCommit(false); setMaxConnections(hikariConfig); hikariConfig.setMaxLifetime(config.get(DatabaseSettings.MAX_LIFETIME)); - hikariConfig.setLeakDetectionThreshold(TimeUnit.SECONDS.toMillis(29L)); + hikariConfig.setLeakDetectionThreshold(config.get(DatabaseSettings.MAX_LIFETIME) + TimeUnit.SECONDS.toMillis(4L)); this.dataSource = new HikariDataSource(hikariConfig); } catch (HikariPool.PoolInitializationException e) { + if (e.getMessage().contains("Unknown system variable 'transaction_isolation'")) { + throw new MariaDB11Exception("MySQL driver is incompatible with database that is being used.", e); + } throw new DBInitException("Failed to set-up HikariCP Datasource: " + e.getMessage(), e); } finally { unloadMySQLDriver(); } - // Reset the context classloader back to what it was originally set to, now that the DataSource is created - currentThread.setContextClassLoader(previousClassLoader); + if (useMariaDbDriver) { + checkMariaDBVersionIncompatibility(); + } + } + + private void checkMariaDBVersionIncompatibility() { + executeTransaction(new OperationCriticalTransaction() { + @Override + protected void performOperations() { + query(MySQLSchemaQueries.getVersion()) + .filter("11.0.2-MariaDB"::equals) + .ifPresent(badVersion -> { + throw new DBInitException("MariaDB version " + badVersion + " inserts incorrect data due to a bug in query execution order so it is not supported. Upgrade MariaDB to 11.1.1 or newer, or downgrade to MariaDB 10."); + }); + } + }); } private void setMaxConnections(HikariConfig hikariConfig) { @@ -156,7 +196,8 @@ public class MySQLDB extends SQLDB { Driver driver = drivers.nextElement(); Class driverClass = driver.getClass(); // Checks that it's from our class loader to avoid unloading another plugin's/the server's driver - if ("com.mysql.cj.jdbc.Driver".equals(driverClass.getName()) && driverClass.getClassLoader() == driverClassLoader) { + String driverName = useMariaDbDriver ? "org.mariadb.jdbc.Driver" : "com.mysql.cj.jdbc.Driver"; + if (driverName.equals(driverClass.getName()) && driverClass.getClassLoader() == driverClassLoader) { try { DriverManager.deregisterDriver(driver); } catch (SQLException e) { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/schema/MySQLSchemaQueries.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/schema/MySQLSchemaQueries.java index babdf39dc..04a70dda2 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/schema/MySQLSchemaQueries.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/schema/MySQLSchemaQueries.java @@ -19,12 +19,14 @@ package com.djrapitops.plan.storage.database.queries.schema; import com.djrapitops.plan.storage.database.queries.HasMoreThanZeroQueryStatement; import com.djrapitops.plan.storage.database.queries.Query; import com.djrapitops.plan.storage.database.queries.QueryStatement; +import org.intellij.lang.annotations.Language; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import static com.djrapitops.plan.storage.database.sql.building.Sql.*; @@ -39,6 +41,12 @@ public class MySQLSchemaQueries { /* Static method class */ } + public static Query> getVersion() { + @Language("MySQL") + String sql = "SELECT VERSION()"; + return db -> db.queryOptional(sql, row -> row.getString(1)); + } + public static Query doesTableExist(String tableName) { String sql = SELECT + "COUNT(1) as c FROM information_schema.TABLES WHERE table_name=? AND TABLE_SCHEMA=DATABASE()"; return new HasMoreThanZeroQueryStatement(sql) { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/sql/tables/WorldTable.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/sql/tables/WorldTable.java index 72addb9d8..d25000b10 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/sql/tables/WorldTable.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/sql/tables/WorldTable.java @@ -50,8 +50,8 @@ public class WorldTable { public static final String SELECT_WORLD_ID_STATEMENT = '(' + SELECT + TABLE_NAME + '.' + ID + FROM + TABLE_NAME + - WHERE + '(' + NAME + "=?)" + - AND + '(' + TABLE_NAME + '.' + SERVER_UUID + "=?)" + + WHERE + NAME + "=?" + + AND + TABLE_NAME + '.' + SERVER_UUID + "=?" + " LIMIT 1)"; private WorldTable() {