From d2ffee87bd8c6f2e3239d41371f98e0855ede27a Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Tue, 23 Apr 2019 12:13:42 +0300 Subject: [PATCH] Removal of unsatisfied server conditional results Cleanup of data that requires a conditional, but the conditional has changed value after storage of the data. --- .../djrapitops/plan/db/tasks/DBCleanTask.java | 6 +- ...dConditionalPlayerResultsTransaction.java} | 2 +- ...edConditionalServerResultsTransaction.java | 107 ++++++++++++++++++ .../com/djrapitops/plan/db/CommonDBTest.java | 91 +++++++++++++-- 4 files changed, 194 insertions(+), 12 deletions(-) rename Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/{RemoveUnsatisfiedConditionalResultsTransaction.java => RemoveUnsatisfiedConditionalPlayerResultsTransaction.java} (98%) create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalServerResultsTransaction.java diff --git a/Plan/common/src/main/java/com/djrapitops/plan/db/tasks/DBCleanTask.java b/Plan/common/src/main/java/com/djrapitops/plan/db/tasks/DBCleanTask.java index d2253d13f..596a3c49c 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/db/tasks/DBCleanTask.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/db/tasks/DBCleanTask.java @@ -24,7 +24,8 @@ import com.djrapitops.plan.db.access.transactions.commands.RemovePlayerTransacti import com.djrapitops.plan.db.access.transactions.init.RemoveDuplicateUserInfoTransaction; import com.djrapitops.plan.db.access.transactions.init.RemoveOldSampledDataTransaction; import com.djrapitops.plan.db.sql.tables.SessionsTable; -import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalResultsTransaction; +import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalPlayerResultsTransaction; +import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalServerResultsTransaction; import com.djrapitops.plan.system.database.DBSystem; import com.djrapitops.plan.system.info.server.ServerInfo; import com.djrapitops.plan.system.locale.Locale; @@ -86,7 +87,8 @@ public class DBCleanTask extends AbsRunnable { if (database.getState() != Database.State.CLOSED) { database.executeTransaction(new RemoveOldSampledDataTransaction(serverInfo.getServerUUID())); database.executeTransaction(new RemoveDuplicateUserInfoTransaction()); - database.executeTransaction(new RemoveUnsatisfiedConditionalResultsTransaction()); + database.executeTransaction(new RemoveUnsatisfiedConditionalPlayerResultsTransaction()); + database.executeTransaction(new RemoveUnsatisfiedConditionalServerResultsTransaction()); int removed = cleanOldPlayers(database); if (removed > 0) { logger.info(locale.getString(PluginLang.DB_NOTIFY_CLEAN, removed)); diff --git a/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalResultsTransaction.java b/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalPlayerResultsTransaction.java similarity index 98% rename from Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalResultsTransaction.java rename to Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalPlayerResultsTransaction.java index f1e62ad27..93ea7b6fa 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalResultsTransaction.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalPlayerResultsTransaction.java @@ -39,7 +39,7 @@ import static com.djrapitops.plan.db.sql.parsing.Sql.*; * * @author Rsl1122 */ -public class RemoveUnsatisfiedConditionalResultsTransaction extends Transaction { +public class RemoveUnsatisfiedConditionalPlayerResultsTransaction extends Transaction { @Override protected void performOperations() { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalServerResultsTransaction.java b/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalServerResultsTransaction.java new file mode 100644 index 000000000..1c35f1c22 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/extension/implementation/storage/transactions/results/RemoveUnsatisfiedConditionalServerResultsTransaction.java @@ -0,0 +1,107 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.extension.implementation.storage.transactions.results; + +import com.djrapitops.plan.db.DBType; +import com.djrapitops.plan.db.access.ExecStatement; +import com.djrapitops.plan.db.access.Executable; +import com.djrapitops.plan.db.access.transactions.Transaction; +import com.djrapitops.plan.db.sql.tables.ExtensionProviderTable; +import com.djrapitops.plan.db.sql.tables.ExtensionServerValueTable; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import static com.djrapitops.plan.db.sql.parsing.Sql.*; + +/** + * Transaction to remove older results that violate an updated condition value. + *

+ * How it works: + * - Select all fulfilled conditions for all servers (conditionName when true and not_conditionName when false) + * - Left join with server value & provider tables when plugin_ids match, and when condition matches a condition in the + * query above. (plugin_ids can be linked to servers) + * - Filter the join query for values where the condition did not match any provided condition in the join (Is null) + * - Delete all server values with IDs that are returned by the left join query after filtering + * + * @author Rsl1122 + */ +public class RemoveUnsatisfiedConditionalServerResultsTransaction extends Transaction { + + @Override + protected void performOperations() { + execute(deleteUnsatisfied()); + } + + private Executable deleteUnsatisfied() { + String reversedCondition = dbType == DBType.SQLITE ? "'not_' || " + ExtensionProviderTable.PROVIDED_CONDITION : "CONCAT('not_'," + ExtensionProviderTable.PROVIDED_CONDITION + ')'; + + String providerTable = ExtensionProviderTable.TABLE_NAME; + String serverValueTable = ExtensionServerValueTable.TABLE_NAME; + + String selectSatisfiedPositiveConditions = SELECT + + ExtensionProviderTable.PROVIDED_CONDITION + ',' + + ExtensionProviderTable.PLUGIN_ID + + FROM + providerTable + + INNER_JOIN + serverValueTable + " on " + providerTable + '.' + ExtensionProviderTable.ID + "=" + ExtensionServerValueTable.PROVIDER_ID + + WHERE + ExtensionServerValueTable.BOOLEAN_VALUE + "=?" + + AND + ExtensionProviderTable.PROVIDED_CONDITION + IS_NOT_NULL; + String selectSatisfiedNegativeConditions = SELECT + + reversedCondition + " as " + ExtensionProviderTable.PROVIDED_CONDITION + ',' + + ExtensionProviderTable.PLUGIN_ID + + FROM + providerTable + + INNER_JOIN + serverValueTable + " on " + providerTable + '.' + ExtensionProviderTable.ID + "=" + ExtensionServerValueTable.PROVIDER_ID + + WHERE + ExtensionServerValueTable.BOOLEAN_VALUE + "=?" + + AND + ExtensionProviderTable.PROVIDED_CONDITION + IS_NOT_NULL; + + // Query contents: Set of provided_conditions + String selectSatisfiedConditions = '(' + selectSatisfiedPositiveConditions + " UNION " + selectSatisfiedNegativeConditions + ") q1"; + + // Query contents: + // id | provider_id | q1.provider_id | condition | q1.provided_condition + // -- | ----------- | -------------- | --------- | --------------------- + // 1 | ... | ... | A | A Satisfied condition + // 2 | ... | ... | not_B | not_B Satisfied condition + // 3 | ... | ... | NULL | NULL Satisfied condition + // 4 | ... | ... | B | NULL Unsatisfied condition, filtered to these in WHERE clause. + // 5 | ... | ... | not_C | NULL Unsatisfied condition + String selectUnsatisfiedValueIDs = SELECT + serverValueTable + '.' + ExtensionServerValueTable.ID + + FROM + providerTable + + INNER_JOIN + serverValueTable + " on " + providerTable + '.' + ExtensionProviderTable.ID + "=" + ExtensionServerValueTable.PROVIDER_ID + + LEFT_JOIN + selectSatisfiedConditions + // Left join to preserve values that don't have their condition fulfilled + " on (" + + ExtensionProviderTable.CONDITION + "=q1." + ExtensionProviderTable.PROVIDED_CONDITION + + AND + providerTable + '.' + ExtensionProviderTable.PLUGIN_ID + "=q1." + ExtensionProviderTable.PLUGIN_ID + + ')' + + WHERE + "q1." + ExtensionProviderTable.PROVIDED_CONDITION + IS_NULL + // Conditions that were not in the satisfied condition query + AND + ExtensionProviderTable.CONDITION + IS_NOT_NULL; // Ignore values that don't need condition + + // Nested query here is required because MySQL limits update statements with nested queries: + // The nested query creates a temporary table that bypasses the same table query-update limit. + // Note: MySQL versions 5.6.7+ might optimize this nested query away leading to an exception. + String sql = "DELETE FROM " + serverValueTable + + WHERE + ExtensionServerValueTable.ID + " IN (" + SELECT + ExtensionServerValueTable.ID + FROM + '(' + selectUnsatisfiedValueIDs + ") as ids)"; + + return new ExecStatement(sql) { + @Override + public void prepare(PreparedStatement statement) throws SQLException { + statement.setBoolean(1, true); // Select provided conditions with 'true' value + statement.setBoolean(2, false); // Select negated conditions with 'false' value + } + }; + } +} \ No newline at end of file diff --git a/Plan/common/src/test/java/com/djrapitops/plan/db/CommonDBTest.java b/Plan/common/src/test/java/com/djrapitops/plan/db/CommonDBTest.java index 85cec1ee5..e1ff46d20 100644 --- a/Plan/common/src/test/java/com/djrapitops/plan/db/CommonDBTest.java +++ b/Plan/common/src/test/java/com/djrapitops/plan/db/CommonDBTest.java @@ -63,7 +63,8 @@ import com.djrapitops.plan.extension.implementation.results.player.ExtensionPlay import com.djrapitops.plan.extension.implementation.results.server.ExtensionServerData; import com.djrapitops.plan.extension.implementation.storage.queries.ExtensionPlayerDataQuery; import com.djrapitops.plan.extension.implementation.storage.queries.ExtensionServerDataQuery; -import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalResultsTransaction; +import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalPlayerResultsTransaction; +import com.djrapitops.plan.extension.implementation.storage.transactions.results.RemoveUnsatisfiedConditionalServerResultsTransaction; import com.djrapitops.plan.extension.table.Table; import com.djrapitops.plan.system.PlanSystem; import com.djrapitops.plan.system.database.DBSystem; @@ -90,7 +91,6 @@ import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.security.NoSuchAlgorithmException; import java.util.*; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -1230,7 +1230,7 @@ public abstract class CommonDBTest { } @Test - public void unsatisfiedConditionalResultsAreCleaned() throws ExecutionException, InterruptedException { + public void unsatisfiedPlayerConditionalResultsAreCleaned() { ExtensionServiceImplementation extensionService = (ExtensionServiceImplementation) system.getExtensionService(); extensionService.register(new ConditionalExtension()); @@ -1239,28 +1239,28 @@ public abstract class CommonDBTest { extensionService.updatePlayerValues(playerUUID, TestConstants.PLAYER_ONE_NAME, CallEvents.MANUAL); // Check that the wanted data exists - checkThatDataExists(ConditionalExtension.condition); + checkThatPlayerDataExists(ConditionalExtension.condition); // Reverse condition ConditionalExtension.condition = false; extensionService.updatePlayerValues(playerUUID, TestConstants.PLAYER_ONE_NAME, CallEvents.MANUAL); - db.executeTransaction(new RemoveUnsatisfiedConditionalResultsTransaction()); + db.executeTransaction(new RemoveUnsatisfiedConditionalPlayerResultsTransaction()); // Check that the wanted data exists - checkThatDataExists(ConditionalExtension.condition); + checkThatPlayerDataExists(ConditionalExtension.condition); // Reverse condition ConditionalExtension.condition = false; extensionService.updatePlayerValues(playerUUID, TestConstants.PLAYER_ONE_NAME, CallEvents.MANUAL); - db.executeTransaction(new RemoveUnsatisfiedConditionalResultsTransaction()); + db.executeTransaction(new RemoveUnsatisfiedConditionalPlayerResultsTransaction()); // Check that the wanted data exists - checkThatDataExists(ConditionalExtension.condition); + checkThatPlayerDataExists(ConditionalExtension.condition); } - private void checkThatDataExists(boolean condition) { + private void checkThatPlayerDataExists(boolean condition) { if (condition) { // Condition is true, conditional values exist List ofServer = db.query(new ExtensionPlayerDataQuery(playerUUID)).get(serverUUID); assertTrue("There was no data left", ofServer != null && !ofServer.isEmpty() && !ofServer.get(0).getTabs().isEmpty()); @@ -1281,6 +1281,57 @@ public abstract class CommonDBTest { } } + @Test + public void unsatisfiedServerConditionalResultsAreCleaned() { + ExtensionServiceImplementation extensionService = (ExtensionServiceImplementation) system.getExtensionService(); + + ConditionalExtension.condition = true; + extensionService.register(new ConditionalExtension()); + extensionService.updateServerValues(CallEvents.MANUAL); + + // Check that the wanted data exists + checkThatServerDataExists(ConditionalExtension.condition); + + // Reverse condition + ConditionalExtension.condition = false; + extensionService.updateServerValues(CallEvents.MANUAL); + + db.executeTransaction(new RemoveUnsatisfiedConditionalServerResultsTransaction()); + + // Check that the wanted data exists + checkThatServerDataExists(ConditionalExtension.condition); + + // Reverse condition + ConditionalExtension.condition = false; + extensionService.updatePlayerValues(playerUUID, TestConstants.PLAYER_ONE_NAME, CallEvents.MANUAL); + + db.executeTransaction(new RemoveUnsatisfiedConditionalServerResultsTransaction()); + + // Check that the wanted data exists + checkThatServerDataExists(ConditionalExtension.condition); + } + + private void checkThatServerDataExists(boolean condition) { + if (condition) { // Condition is true, conditional values exist + List ofServer = db.query(new ExtensionServerDataQuery(serverUUID)); + assertTrue("There was no data left", ofServer != null && !ofServer.isEmpty() && !ofServer.get(0).getTabs().isEmpty()); + + ExtensionTabData tabData = ofServer.get(0).getTabs().get(0); + OptionalAssert.equals("Yes", tabData.getBoolean("isCondition").map(ExtensionBooleanData::getFormattedValue)); + OptionalAssert.equals("Conditional", tabData.getString("conditionalValue").map(ExtensionStringData::getFormattedValue)); + OptionalAssert.equals("unconditional", tabData.getString("unconditional").map(ExtensionStringData::getFormattedValue)); // Was not removed + assertFalse("Value was not removed: reversedConditionalValue", tabData.getString("reversedConditionalValue").isPresent()); + } else { // Condition is false, reversed conditional values exist + List ofServer = db.query(new ExtensionServerDataQuery(serverUUID)); + assertTrue("There was no data left", ofServer != null && !ofServer.isEmpty() && !ofServer.get(0).getTabs().isEmpty()); + ExtensionTabData tabData = ofServer.get(0).getTabs().get(0); + OptionalAssert.equals("No", tabData.getBoolean("isCondition").map(ExtensionBooleanData::getFormattedValue)); + OptionalAssert.equals("Reversed", tabData.getString("reversedConditionalValue").map(ExtensionStringData::getFormattedValue)); + OptionalAssert.equals("unconditional", tabData.getString("unconditional").map(ExtensionStringData::getFormattedValue)); // Was not removed + assertFalse("Value was not removed: conditionalValue", tabData.getString("conditionalValue").isPresent()); + } + } + @Test public void extensionServerTableValuesAreInserted() { ExtensionServiceImplementation extensionService = (ExtensionServiceImplementation) system.getExtensionService(); @@ -1371,6 +1422,28 @@ public abstract class CommonDBTest { public String unconditional(UUID playerUUID) { return "unconditional"; } + + @BooleanProvider(text = "a boolean", conditionName = "condition") + public boolean isCondition() { + return condition; + } + + @StringProvider(text = "Conditional Value") + @Conditional("condition") + public String conditionalValue() { + return "Conditional"; + } + + @StringProvider(text = "Reversed Conditional Value") + @Conditional(value = "condition", negated = true) + public String reversedConditionalValue() { + return "Reversed"; + } + + @StringProvider(text = "Unconditional") + public String unconditional() { + return "unconditional"; + } } @PluginInfo(name = "ServerExtension")