From 05a2c20fdd6a3df38beab33e3ca22e06e2d2fc09 Mon Sep 17 00:00:00 2001 From: Risto Lahtela <24460436+Rsl1122@users.noreply.github.com> Date: Fri, 5 Feb 2021 16:42:50 +0200 Subject: [PATCH] Added a service for updating JSON asynchronously Still needs: - an in-memory cache that invalidates things - add the current timestamp to the returned json - javascript side that updates data if the timestamp is old --- .../json/AsyncJSONResolverService.java | 137 ++++++++++++++++++ .../resolver/json/NetworkJSONResolver.java | 6 +- .../resolver/json/NetworkTabJSONResolver.java | 28 +++- .../resolver/json/RootJSONResolver.java | 5 +- .../resolver/json/ServerTabJSONResolver.java | 29 +++- .../plan/storage/json/JSONFileStorage.java | 31 ++++ .../plan/storage/json/JSONStorage.java | 2 + 7 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/AsyncJSONResolverService.java diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/AsyncJSONResolverService.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/AsyncJSONResolverService.java new file mode 100644 index 000000000..5de4217b0 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/AsyncJSONResolverService.java @@ -0,0 +1,137 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.delivery.webserver.resolver.json; + +import com.djrapitops.plan.delivery.webserver.cache.DataID; +import com.djrapitops.plan.processing.Processing; +import com.djrapitops.plan.storage.json.JSONStorage; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Service for resolving json asynchronously in order to move database queries off server thread. + * + * @author Rsl1122 + */ +@Singleton +public class AsyncJSONResolverService { + + private final Processing processing; + private final JSONStorage jsonStorage; + private final Map> currentlyProcessing; + + @Inject + public AsyncJSONResolverService( + Processing processing, + JSONStorage jsonStorage + ) { + this.processing = processing; + this.jsonStorage = jsonStorage; + + currentlyProcessing = new ConcurrentHashMap<>(); + } + + public JSONStorage.StoredJSON resolve(long newerThanTimestamp, DataID dataID, UUID serverUUID, Function creator) { + String identifier = dataID.of(serverUUID); + + // Attempt to find a newer version of the json file from cache + Optional storedJSON = jsonStorage.fetchJsonMadeAfter(identifier, newerThanTimestamp); + if (storedJSON.isPresent()) { + return storedJSON.get(); + } + // No new enough version, let's refresh and send old version of the file + + // Check if the json is already being created + Future updatedJSON = currentlyProcessing.get(identifier); + if (updatedJSON == null) { + // Submit a task to refresh the data if the json is old + updatedJSON = processing.submitNonCritical(() -> { + JSONStorage.StoredJSON created = jsonStorage.storeJson(identifier, creator.apply(serverUUID)); + currentlyProcessing.remove(identifier); + jsonStorage.invalidateOlder(identifier, created.timestamp); + return created; + }); + currentlyProcessing.put(identifier, updatedJSON); + } + + // Get an old version from cache + storedJSON = jsonStorage.fetchJsonMadeBefore(identifier, newerThanTimestamp); + if (storedJSON.isPresent()) { + return storedJSON.get(); + } else { + // If there is no version available, block thread until the new finishes being generated. + try { + return updatedJSON.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + } + } + + public JSONStorage.StoredJSON resolve(long newerThanTimestamp, DataID dataID, Supplier creator) { + String identifier = dataID.name(); + + // Attempt to find a newer version of the json file from cache + Optional storedJSON = jsonStorage.fetchJsonMadeAfter(identifier, newerThanTimestamp); + if (storedJSON.isPresent()) { + return storedJSON.get(); + } + // No new enough version, let's refresh and send old version of the file + + // Check if the json is already being created + Future updatedJSON = currentlyProcessing.get(identifier); + if (updatedJSON == null) { + // Submit a task to refresh the data if the json is old + updatedJSON = processing.submitNonCritical(() -> { + JSONStorage.StoredJSON created = jsonStorage.storeJson(identifier, creator.get()); + currentlyProcessing.remove(identifier); + jsonStorage.invalidateOlder(identifier, created.timestamp); + return created; + }); + currentlyProcessing.put(identifier, updatedJSON); + } + + // Get an old version from cache + storedJSON = jsonStorage.fetchJsonMadeBefore(identifier, newerThanTimestamp); + if (storedJSON.isPresent()) { + return storedJSON.get(); + } else { + // If there is no version available, block thread until the new finishes being generated. + try { + return updatedJSON.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + } + } + +} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkJSONResolver.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkJSONResolver.java index 42d486791..975a98c67 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkJSONResolver.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkJSONResolver.java @@ -35,15 +35,17 @@ import javax.inject.Singleton; @Singleton public class NetworkJSONResolver { + private final AsyncJSONResolverService asyncJSONResolverService; private final CompositeResolver resolver; @Inject public NetworkJSONResolver( - JSONFactory jsonFactory, + AsyncJSONResolverService asyncJSONResolverService, JSONFactory jsonFactory, NetworkOverviewJSONCreator networkOverviewJSONCreator, NetworkPlayerBaseOverviewJSONCreator networkPlayerBaseOverviewJSONCreator, NetworkSessionsOverviewJSONCreator networkSessionsOverviewJSONCreator ) { + this.asyncJSONResolverService = asyncJSONResolverService; resolver = CompositeResolver.builder() .add("overview", forJSON(DataID.SERVER_OVERVIEW, networkOverviewJSONCreator)) .add("playerbaseOverview", forJSON(DataID.PLAYERBASE_OVERVIEW, networkPlayerBaseOverviewJSONCreator)) @@ -54,7 +56,7 @@ public class NetworkJSONResolver { } private NetworkTabJSONResolver forJSON(DataID dataID, NetworkTabJSONCreator tabJSONCreator) { - return new NetworkTabJSONResolver<>(dataID, tabJSONCreator); + return new NetworkTabJSONResolver<>(dataID, tabJSONCreator, asyncJSONResolverService); } public CompositeResolver getResolver() { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkTabJSONResolver.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkTabJSONResolver.java index 80df5e5a6..bab4a7083 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkTabJSONResolver.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkTabJSONResolver.java @@ -17,12 +17,13 @@ package com.djrapitops.plan.delivery.webserver.resolver.json; import com.djrapitops.plan.delivery.rendering.json.network.NetworkTabJSONCreator; +import com.djrapitops.plan.delivery.web.resolver.MimeType; import com.djrapitops.plan.delivery.web.resolver.Resolver; import com.djrapitops.plan.delivery.web.resolver.Response; +import com.djrapitops.plan.delivery.web.resolver.exception.BadRequestException; import com.djrapitops.plan.delivery.web.resolver.request.Request; import com.djrapitops.plan.delivery.web.resolver.request.WebUser; import com.djrapitops.plan.delivery.webserver.cache.DataID; -import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import java.util.Optional; import java.util.function.Supplier; @@ -36,10 +37,15 @@ public class NetworkTabJSONResolver implements Resolver { private final DataID dataID; private final Supplier jsonCreator; + private final AsyncJSONResolverService asyncJSONResolverService; - public NetworkTabJSONResolver(DataID dataID, NetworkTabJSONCreator jsonCreator) { + public NetworkTabJSONResolver( + DataID dataID, NetworkTabJSONCreator jsonCreator, + AsyncJSONResolverService asyncJSONResolverService + ) { this.dataID = dataID; this.jsonCreator = jsonCreator; + this.asyncJSONResolverService = asyncJSONResolverService; } @Override @@ -49,11 +55,23 @@ public class NetworkTabJSONResolver implements Resolver { @Override public Optional resolve(Request request) { - return Optional.of(getResponse()); + return Optional.of(getResponse(request)); } - private Response getResponse() { - return JSONCache.getOrCache(dataID, jsonCreator); + private Response getResponse(Request request) { + return Response.builder() + .setMimeType(MimeType.JSON) + .setJSONContent(asyncJSONResolverService.resolve(getTimestamp(request), dataID, jsonCreator).json) + .build(); } + private long getTimestamp(Request request) { + try { + return request.getQuery().get("timestamp") + .map(Long::parseLong) + .orElseGet(System::currentTimeMillis); + } catch (NumberFormatException nonNumberTimestamp) { + throw new BadRequestException("'timestamp' was not a number: " + nonNumberTimestamp.getMessage()); + } + } } \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/RootJSONResolver.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/RootJSONResolver.java index d8a8b6af5..5ee9e9dc9 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/RootJSONResolver.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/RootJSONResolver.java @@ -33,11 +33,13 @@ import javax.inject.Singleton; public class RootJSONResolver { private final Identifiers identifiers; + private final AsyncJSONResolverService asyncJSONResolverService; private final CompositeResolver resolver; @Inject public RootJSONResolver( Identifiers identifiers, + AsyncJSONResolverService asyncJSONResolverService, JSONFactory jsonFactory, GraphsJSONResolver graphsJSONResolver, @@ -57,6 +59,7 @@ public class RootJSONResolver { QueryJSONResolver queryJSONResolver ) { this.identifiers = identifiers; + this.asyncJSONResolverService = asyncJSONResolverService; resolver = CompositeResolver.builder() .add("players", playersTableJSONResolver) @@ -78,7 +81,7 @@ public class RootJSONResolver { } private ServerTabJSONResolver forJSON(DataID dataID, ServerTabJSONCreator tabJSONCreator) { - return new ServerTabJSONResolver<>(dataID, identifiers, tabJSONCreator); + return new ServerTabJSONResolver<>(dataID, identifiers, tabJSONCreator, asyncJSONResolverService); } public CompositeResolver getResolver() { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/ServerTabJSONResolver.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/ServerTabJSONResolver.java index a7f449e50..d1c645e4e 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/ServerTabJSONResolver.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/ServerTabJSONResolver.java @@ -17,12 +17,13 @@ package com.djrapitops.plan.delivery.webserver.resolver.json; import com.djrapitops.plan.delivery.rendering.json.ServerTabJSONCreator; +import com.djrapitops.plan.delivery.web.resolver.MimeType; import com.djrapitops.plan.delivery.web.resolver.Resolver; import com.djrapitops.plan.delivery.web.resolver.Response; +import com.djrapitops.plan.delivery.web.resolver.exception.BadRequestException; import com.djrapitops.plan.delivery.web.resolver.request.Request; import com.djrapitops.plan.delivery.web.resolver.request.WebUser; import com.djrapitops.plan.delivery.webserver.cache.DataID; -import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.identification.Identifiers; import java.util.Optional; @@ -39,15 +40,16 @@ public class ServerTabJSONResolver implements Resolver { private final DataID dataID; private final Identifiers identifiers; private final Function jsonCreator; + private final AsyncJSONResolverService asyncJSONResolverService; public ServerTabJSONResolver( - DataID dataID, - Identifiers identifiers, - ServerTabJSONCreator jsonCreator + DataID dataID, Identifiers identifiers, ServerTabJSONCreator jsonCreator, + AsyncJSONResolverService asyncJSONResolverService ) { this.dataID = dataID; this.identifiers = identifiers; this.jsonCreator = jsonCreator; + this.asyncJSONResolverService = asyncJSONResolverService; } @Override @@ -57,8 +59,25 @@ public class ServerTabJSONResolver implements Resolver { @Override public Optional resolve(Request request) { + return Optional.of(getResponse(request)); + } + + private Response getResponse(Request request) { UUID serverUUID = identifiers.getServerUUID(request); // Can throw BadRequestException - return Optional.of(JSONCache.getOrCache(dataID, serverUUID, () -> jsonCreator.apply(serverUUID))); + return Response.builder() + .setMimeType(MimeType.JSON) + .setJSONContent(asyncJSONResolverService.resolve(getTimestamp(request), dataID, serverUUID, jsonCreator).json) + .build(); + } + + private long getTimestamp(Request request) { + try { + return request.getQuery().get("timestamp") + .map(Long::parseLong) + .orElseGet(System::currentTimeMillis); + } catch (NumberFormatException nonNumberTimestamp) { + throw new BadRequestException("'timestamp' was not a number: " + nonNumberTimestamp.getMessage()); + } } } \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java index c2750d5ab..b386edc60 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java @@ -27,6 +27,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.function.BiPredicate; import java.util.regex.Matcher; @@ -134,4 +136,33 @@ public class JSONFileStorage implements JSONStorage { } return Optional.empty(); } + + @Override + public void invalidateOlder(String identifier, long timestamp) { + File[] stored = jsonDirectory.toFile().listFiles(); + if (stored == null) return; + + List toDelete = new ArrayList<>(); + for (File file : stored) { + try { + String fileName = file.getName(); + if (fileName.endsWith(JSON_FILE_EXTENSION) && fileName.startsWith(identifier)) { + Matcher timestampMatch = timestampRegex.matcher(fileName); + if (timestampMatch.find() && Long.parseLong(timestampMatch.group(1)) < timestamp) { + toDelete.add(file); + } + } + } catch (NumberFormatException e) { + // Ignore this file, malformed timestamp + } + } + for (File fileToDelete : toDelete) { + try { + Files.delete(fileToDelete.toPath()); + } catch (IOException e) { + // Failed to delete, set for deletion on next server shutdown. + fileToDelete.deleteOnExit(); + } + } + } } diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONStorage.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONStorage.java index 9bda04511..c77020c0c 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONStorage.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONStorage.java @@ -50,6 +50,8 @@ public interface JSONStorage { Optional fetchJsonMadeAfter(String identifier, long timestamp); + void invalidateOlder(String identifier, long timestamp); + final class StoredJSON { public final String json; public final long timestamp;