Upstream
/
Plan
mirror of https://github.com/plan-player-analytics/Plan.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

JSON File storage clean task 

- Cleans query files
- Cleans other json files that are old
This commit is contained in:
Risto Lahtela 2021-02-06 08:58:02 +02:00 committed by Risto Lahtela
parent a621b343e8
commit 3619ff814a
9 changed files with 106 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Plan/bukkit/src/main/java/com/djrapitops/plan/modules/bukkit/BukkitTaskModule.java
View File

@ -24,6 +24,7 @@ import com.djrapitops.plan.gathering.timed.BukkitPingCounter;
import com.djrapitops.plan.gathering.timed.ServerTPSCounter;
import com.djrapitops.plan.gathering.timed.SystemUsageBuffer;
import com.djrapitops.plan.settings.upkeep.ConfigStoreTask;
import com.djrapitops.plan.storage.json.JSONFileStorage;
import com.djrapitops.plan.storage.upkeep.DBCleanTask;
import com.djrapitops.plan.storage.upkeep.LogsFolderCleanTask;
import dagger.Binds;
@ -74,4 +75,8 @@ public interface BukkitTaskModule {
@IntoSet
TaskSystem.Task bindShutdownHookRegistration(ShutdownHook.Registrar registrar);
@Binds
@IntoSet
TaskSystem.Task bindJSONFileStorageCleanTask(JSONFileStorage.CleanTask cleanTask);
}

4
Plan/bungeecord/src/main/java/com/djrapitops/plan/modules/bungee/BungeeTaskModule.java
View File

@ -23,6 +23,7 @@ import com.djrapitops.plan.gathering.timed.BungeePingCounter;
import com.djrapitops.plan.gathering.timed.ProxyTPSCounter;
import com.djrapitops.plan.gathering.timed.SystemUsageBuffer;
import com.djrapitops.plan.settings.upkeep.NetworkConfigStoreTask;
import com.djrapitops.plan.storage.json.JSONFileStorage;
import com.djrapitops.plan.storage.upkeep.DBCleanTask;
import com.djrapitops.plan.storage.upkeep.LogsFolderCleanTask;
import dagger.Binds;
@ -68,4 +69,7 @@ public interface BungeeTaskModule {
@IntoSet
TaskSystem.Task bindDiskTask(SystemUsageBuffer.DiskTask diskTask);
@Binds
@IntoSet
TaskSystem.Task bindJSONFileStorageCleanTask(JSONFileStorage.CleanTask cleanTask);
}

1
Plan/common/src/main/java/com/djrapitops/plan/DebugChannels.java
View File

@ -30,5 +30,6 @@ public class DebugChannels {
public static final String IMPORTING = "Importing";
public static final String SQL = "SQL";
public static final String DATA_EXTENSIONS = "DataExtensions";
public static final String JSON_CACHE = "JSON Cache";
}

82
Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java
View File

@ -16,8 +16,15 @@
*/
package com.djrapitops.plan.storage.json;
import com.djrapitops.plan.DebugChannels;
import com.djrapitops.plan.TaskSystem;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.WebserverSettings;
import com.djrapitops.plan.storage.file.PlanFiles;
import com.djrapitops.plugin.api.TimeAmount;
import com.djrapitops.plugin.logging.console.PluginLogger;
import com.djrapitops.plugin.logging.debug.DebugLogger;
import com.djrapitops.plugin.task.RunnableFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -30,6 +37,8 @@ import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -49,10 +58,12 @@ public class JSONFileStorage implements JSONStorage {
private final Pattern timestampRegex = Pattern.compile(".*-([0-9]*).json");
private static final String JSON_FILE_EXTENSION = ".json";
private final DebugLogger debugLogger;
@Inject
public JSONFileStorage(PlanFiles files, PluginLogger logger) {
this.logger = logger;
debugLogger = logger.getDebugLogger();
jsonDirectory = files.getJSONStorageDirectory();
}
@ -156,8 +167,39 @@ public class JSONFileStorage implements JSONStorage {
// Ignore this file, malformed timestamp
}
}
deleteFiles(toDelete);
}
private void invalidateOlderButIgnore(long timestamp, String... ignoredIdentifiers) {
File[] stored = jsonDirectory.toFile().listFiles();
if (stored == null) return;
List<File> toDelete = new ArrayList<>();
outer:
for (File file : stored) {
try {
String fileName = file.getName();
if (fileName.endsWith(JSON_FILE_EXTENSION)) {
Matcher timestampMatch = timestampRegex.matcher(fileName);
if (timestampMatch.find() && Long.parseLong(timestampMatch.group(1)) < timestamp) {
for (String ignoredIdentifier : ignoredIdentifiers) {
if (fileName.startsWith(ignoredIdentifier)) continue outer;
}
toDelete.add(file);
}
}
} catch (NumberFormatException e) {
// Ignore this file, malformed timestamp
}
}
deleteFiles(toDelete);
}
private void deleteFiles(List<File> toDelete) {
for (File fileToDelete : toDelete) {
try {
debugLogger.logOn(DebugChannels.JSON_CACHE, "Deleting " + fileToDelete.getAbsolutePath());
Files.delete(fileToDelete.toPath());
} catch (IOException e) {
// Failed to delete, set for deletion on next server shutdown.
@ -165,4 +207,44 @@ public class JSONFileStorage implements JSONStorage {
}
}
}
@Singleton
public static class CleanTask extends TaskSystem.Task {
private final PlanConfig config;
private final JSONFileStorage jsonFileStorage;
private final DebugLogger debugLogger;
@Inject
public CleanTask(
PlanConfig config,
JSONFileStorage jsonFileStorage,
DebugLogger debugLogger
) {
this.config = config;
this.jsonFileStorage = jsonFileStorage;
this.debugLogger = debugLogger;
}
@Override
public void register(RunnableFactory runnableFactory) {
long delay = TimeAmount.toTicks(ThreadLocalRandom.current().nextInt(60), TimeUnit.SECONDS);
long period = TimeAmount.toTicks(1, TimeUnit.HOURS);
runnableFactory.create(null, this).runTaskTimerAsynchronously(delay, period);
}
@Override
public void run() {
long now = System.currentTimeMillis();
long invalidateDiskCacheAfterMs = config.get(WebserverSettings.INVALIDATE_DISK_CACHE);
long invalidateQueriesAfterMs = config.get(WebserverSettings.INVALIDATE_QUERY_RESULTS);
debugLogger.logOn(DebugChannels.JSON_CACHE, "Running clean task..");
jsonFileStorage.invalidateOlder("query", now - invalidateQueriesAfterMs);
jsonFileStorage.invalidateOlderButIgnore(now - invalidateDiskCacheAfterMs, "query");
}
public DebugLogger getDebugLogger() {
return debugLogger;
}
}
}

2
Plan/common/src/main/resources/assets/plan/bungeeconfig.yml
View File

@ -49,7 +49,7 @@ Webserver:
Unit: MINUTES
Invalidate_query_results_on_disk_after: 7
Unit: DAYS
Invalidate_disk_cache_after: 1
Invalidate_disk_cache_after: 2
Unit: DAYS
Invalidate_memory_cache_after: 5
Unit: MINUTES

2
Plan/common/src/main/resources/assets/plan/config.yml
View File

@ -54,7 +54,7 @@ Webserver:
Unit: MINUTES
Invalidate_query_results_on_disk_after: 7
Unit: DAYS
Invalidate_disk_cache_after: 1
Invalidate_disk_cache_after: 2
Unit: DAYS
Invalidate_memory_cache_after: 5
Unit: MINUTES

4
Plan/nukkit/src/main/java/com/djrapitops/plan/modules/nukkit/NukkitTaskModule.java
View File

@ -25,6 +25,7 @@ import com.djrapitops.plan.gathering.timed.NukkitPingCounter;
import com.djrapitops.plan.gathering.timed.ServerTPSCounter;
import com.djrapitops.plan.gathering.timed.SystemUsageBuffer;
import com.djrapitops.plan.settings.upkeep.ConfigStoreTask;
import com.djrapitops.plan.storage.json.JSONFileStorage;
import com.djrapitops.plan.storage.upkeep.DBCleanTask;
import com.djrapitops.plan.storage.upkeep.LogsFolderCleanTask;
import dagger.Binds;
@ -74,4 +75,7 @@ public interface NukkitTaskModule {
@IntoSet
TaskSystem.Task bindShutdownHookRegistration(ShutdownHook.Registrar registrar);
@Binds
@IntoSet
TaskSystem.Task bindJSONFileStorageCleanTask(JSONFileStorage.CleanTask cleanTask);
}

4
Plan/sponge/src/main/java/com/djrapitops/plan/modules/sponge/SpongeTaskModule.java
View File

@ -24,6 +24,7 @@ import com.djrapitops.plan.gathering.timed.ServerTPSCounter;
import com.djrapitops.plan.gathering.timed.SpongePingCounter;
import com.djrapitops.plan.gathering.timed.SystemUsageBuffer;
import com.djrapitops.plan.settings.upkeep.ConfigStoreTask;
import com.djrapitops.plan.storage.json.JSONFileStorage;
import com.djrapitops.plan.storage.upkeep.DBCleanTask;
import com.djrapitops.plan.storage.upkeep.LogsFolderCleanTask;
import dagger.Binds;
@ -74,4 +75,7 @@ public interface SpongeTaskModule {
@IntoSet
TaskSystem.Task bindShutdownHookRegistration(ShutdownHook.Registrar registrar);
@Binds
@IntoSet
TaskSystem.Task bindJSONFileStorageCleanTask(JSONFileStorage.CleanTask cleanTask);
}

4
Plan/velocity/src/main/java/com/djrapitops/plan/modules/velocity/VelocityTaskModule.java
View File

@ -23,6 +23,7 @@ import com.djrapitops.plan.gathering.timed.ProxyTPSCounter;
import com.djrapitops.plan.gathering.timed.SystemUsageBuffer;
import com.djrapitops.plan.gathering.timed.VelocityPingCounter;
import com.djrapitops.plan.settings.upkeep.NetworkConfigStoreTask;
import com.djrapitops.plan.storage.json.JSONFileStorage;
import com.djrapitops.plan.storage.upkeep.DBCleanTask;
import com.djrapitops.plan.storage.upkeep.LogsFolderCleanTask;
import dagger.Binds;
@ -68,4 +69,7 @@ public interface VelocityTaskModule {
@IntoSet
TaskSystem.Task bindDiskTask(SystemUsageBuffer.DiskTask diskTask);
@Binds
@IntoSet
TaskSystem.Task bindJSONFileStorageCleanTask(JSONFileStorage.CleanTask cleanTask);
}