Upstream/Plan
1
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2024-09-19 18:21:00 +02:00
Code Issues Projects Releases Wiki Activity

Added a service for updating JSON asynchronously

Browse Source 
Still needs:
- an in-memory cache that invalidates things
- add the current timestamp to the returned json
- javascript side that updates data if the timestamp is old
This commit is contained in:
Risto Lahtela 2021-02-05 16:42:50 +02:00 committed by Risto Lahtela
parent 2a75e7fb5a
commit 05a2c20fdd
7 changed files with 225 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/AsyncJSONResolverService.java Normal file
View File

@ -0,0 +1,137 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.delivery.webserver.resolver.json;
import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.processing.Processing;
import com.djrapitops.plan.storage.json.JSONStorage;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.function.Supplier;
/**
* Service for resolving json asynchronously in order to move database queries off server thread.
*
* @author Rsl1122
*/
@Singleton
public class AsyncJSONResolverService {
private final Processing processing;
private final JSONStorage jsonStorage;
private final Map<String, Future<JSONStorage.StoredJSON>> currentlyProcessing;
@Inject
public AsyncJSONResolverService(
Processing processing,
JSONStorage jsonStorage
) {
this.processing = processing;
this.jsonStorage = jsonStorage;
currentlyProcessing = new ConcurrentHashMap<>();
}
public <T> JSONStorage.StoredJSON resolve(long newerThanTimestamp, DataID dataID, UUID serverUUID, Function<UUID, T> creator) {
String identifier = dataID.of(serverUUID);
// Attempt to find a newer version of the json file from cache
Optional<JSONStorage.StoredJSON> storedJSON = jsonStorage.fetchJsonMadeAfter(identifier, newerThanTimestamp);
if (storedJSON.isPresent()) {
return storedJSON.get();
}
// No new enough version, let's refresh and send old version of the file
// Check if the json is already being created
Future<JSONStorage.StoredJSON> updatedJSON = currentlyProcessing.get(identifier);
if (updatedJSON == null) {
// Submit a task to refresh the data if the json is old
updatedJSON = processing.submitNonCritical(() -> {
JSONStorage.StoredJSON created = jsonStorage.storeJson(identifier, creator.apply(serverUUID));
currentlyProcessing.remove(identifier);
jsonStorage.invalidateOlder(identifier, created.timestamp);
return created;
});
currentlyProcessing.put(identifier, updatedJSON);
}
// Get an old version from cache
storedJSON = jsonStorage.fetchJsonMadeBefore(identifier, newerThanTimestamp);
if (storedJSON.isPresent()) {
return storedJSON.get();
} else {
// If there is no version available, block thread until the new finishes being generated.
try {
return updatedJSON.get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
} catch (ExecutionException e) {
throw new IllegalStateException(e);
}
}
}
public <T> JSONStorage.StoredJSON resolve(long newerThanTimestamp, DataID dataID, Supplier<T> creator) {
String identifier = dataID.name();
// Attempt to find a newer version of the json file from cache
Optional<JSONStorage.StoredJSON> storedJSON = jsonStorage.fetchJsonMadeAfter(identifier, newerThanTimestamp);
if (storedJSON.isPresent()) {
return storedJSON.get();
}
// No new enough version, let's refresh and send old version of the file
// Check if the json is already being created
Future<JSONStorage.StoredJSON> updatedJSON = currentlyProcessing.get(identifier);
if (updatedJSON == null) {
// Submit a task to refresh the data if the json is old
updatedJSON = processing.submitNonCritical(() -> {
JSONStorage.StoredJSON created = jsonStorage.storeJson(identifier, creator.get());
currentlyProcessing.remove(identifier);
jsonStorage.invalidateOlder(identifier, created.timestamp);
return created;
});
currentlyProcessing.put(identifier, updatedJSON);
}
// Get an old version from cache
storedJSON = jsonStorage.fetchJsonMadeBefore(identifier, newerThanTimestamp);
if (storedJSON.isPresent()) {
return storedJSON.get();
} else {
// If there is no version available, block thread until the new finishes being generated.
try {
return updatedJSON.get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
} catch (ExecutionException e) {
throw new IllegalStateException(e);
}
}
}
}

6
Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkJSONResolver.java
View File

@ -35,15 +35,17 @@ import javax.inject.Singleton;
@Singleton
public class NetworkJSONResolver {
private final AsyncJSONResolverService asyncJSONResolverService;
private final CompositeResolver resolver;
@Inject
public NetworkJSONResolver(
JSONFactory jsonFactory,
AsyncJSONResolverService asyncJSONResolverService, JSONFactory jsonFactory,
NetworkOverviewJSONCreator networkOverviewJSONCreator,
NetworkPlayerBaseOverviewJSONCreator networkPlayerBaseOverviewJSONCreator,
NetworkSessionsOverviewJSONCreator networkSessionsOverviewJSONCreator
) {
this.asyncJSONResolverService = asyncJSONResolverService;
resolver = CompositeResolver.builder()
.add("overview", forJSON(DataID.SERVER_OVERVIEW, networkOverviewJSONCreator))
.add("playerbaseOverview", forJSON(DataID.PLAYERBASE_OVERVIEW, networkPlayerBaseOverviewJSONCreator))
@ -54,7 +56,7 @@ public class NetworkJSONResolver {
}
private <T> NetworkTabJSONResolver<T> forJSON(DataID dataID, NetworkTabJSONCreator<T> tabJSONCreator) {
return new NetworkTabJSONResolver<>(dataID, tabJSONCreator);
return new NetworkTabJSONResolver<>(dataID, tabJSONCreator, asyncJSONResolverService);
}
public CompositeResolver getResolver() {

28
Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/NetworkTabJSONResolver.java
View File

@ -17,12 +17,13 @@
package com.djrapitops.plan.delivery.webserver.resolver.json;
import com.djrapitops.plan.delivery.rendering.json.network.NetworkTabJSONCreator;
import com.djrapitops.plan.delivery.web.resolver.MimeType;
import com.djrapitops.plan.delivery.web.resolver.Resolver;
import com.djrapitops.plan.delivery.web.resolver.Response;
import com.djrapitops.plan.delivery.web.resolver.exception.BadRequestException;
import com.djrapitops.plan.delivery.web.resolver.request.Request;
import com.djrapitops.plan.delivery.web.resolver.request.WebUser;
import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import java.util.Optional;
import java.util.function.Supplier;
@ -36,10 +37,15 @@ public class NetworkTabJSONResolver<T> implements Resolver {
private final DataID dataID;
private final Supplier<T> jsonCreator;
private final AsyncJSONResolverService asyncJSONResolverService;
public NetworkTabJSONResolver(DataID dataID, NetworkTabJSONCreator<T> jsonCreator) {
public NetworkTabJSONResolver(
DataID dataID, NetworkTabJSONCreator<T> jsonCreator,
AsyncJSONResolverService asyncJSONResolverService
) {
this.dataID = dataID;
this.jsonCreator = jsonCreator;
this.asyncJSONResolverService = asyncJSONResolverService;
}
@Override
@ -49,11 +55,23 @@ public class NetworkTabJSONResolver<T> implements Resolver {
@Override
public Optional<Response> resolve(Request request) {
return Optional.of(getResponse());
return Optional.of(getResponse(request));
}
private Response getResponse() {
return JSONCache.getOrCache(dataID, jsonCreator);
private Response getResponse(Request request) {
return Response.builder()
.setMimeType(MimeType.JSON)
.setJSONContent(asyncJSONResolverService.resolve(getTimestamp(request), dataID, jsonCreator).json)
.build();
}
private long getTimestamp(Request request) {
try {
return request.getQuery().get("timestamp")
.map(Long::parseLong)
.orElseGet(System::currentTimeMillis);
} catch (NumberFormatException nonNumberTimestamp) {
throw new BadRequestException("'timestamp' was not a number: " + nonNumberTimestamp.getMessage());
}
}
}

5
Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/RootJSONResolver.java
View File

@ -33,11 +33,13 @@ import javax.inject.Singleton;
public class RootJSONResolver {
private final Identifiers identifiers;
private final AsyncJSONResolverService asyncJSONResolverService;
private final CompositeResolver resolver;
@Inject
public RootJSONResolver(
Identifiers identifiers,
AsyncJSONResolverService asyncJSONResolverService,
JSONFactory jsonFactory,
GraphsJSONResolver graphsJSONResolver,
@ -57,6 +59,7 @@ public class RootJSONResolver {
QueryJSONResolver queryJSONResolver
) {
this.identifiers = identifiers;
this.asyncJSONResolverService = asyncJSONResolverService;
resolver = CompositeResolver.builder()
.add("players", playersTableJSONResolver)
@ -78,7 +81,7 @@ public class RootJSONResolver {
}
private <T> ServerTabJSONResolver<T> forJSON(DataID dataID, ServerTabJSONCreator<T> tabJSONCreator) {
return new ServerTabJSONResolver<>(dataID, identifiers, tabJSONCreator);
return new ServerTabJSONResolver<>(dataID, identifiers, tabJSONCreator, asyncJSONResolverService);
}
public CompositeResolver getResolver() {

29
Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/resolver/json/ServerTabJSONResolver.java
View File

@ -17,12 +17,13 @@
package com.djrapitops.plan.delivery.webserver.resolver.json;
import com.djrapitops.plan.delivery.rendering.json.ServerTabJSONCreator;
import com.djrapitops.plan.delivery.web.resolver.MimeType;
import com.djrapitops.plan.delivery.web.resolver.Resolver;
import com.djrapitops.plan.delivery.web.resolver.Response;
import com.djrapitops.plan.delivery.web.resolver.exception.BadRequestException;
import com.djrapitops.plan.delivery.web.resolver.request.Request;
import com.djrapitops.plan.delivery.web.resolver.request.WebUser;
import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.identification.Identifiers;
import java.util.Optional;
@ -39,15 +40,16 @@ public class ServerTabJSONResolver<T> implements Resolver {
private final DataID dataID;
private final Identifiers identifiers;
private final Function<UUID, T> jsonCreator;
private final AsyncJSONResolverService asyncJSONResolverService;
public ServerTabJSONResolver(
DataID dataID,
Identifiers identifiers,
ServerTabJSONCreator<T> jsonCreator
DataID dataID, Identifiers identifiers, ServerTabJSONCreator<T> jsonCreator,
AsyncJSONResolverService asyncJSONResolverService
) {
this.dataID = dataID;
this.identifiers = identifiers;
this.jsonCreator = jsonCreator;
this.asyncJSONResolverService = asyncJSONResolverService;
}
@Override
@ -57,8 +59,25 @@ public class ServerTabJSONResolver<T> implements Resolver {
@Override
public Optional<Response> resolve(Request request) {
return Optional.of(getResponse(request));
}
private Response getResponse(Request request) {
UUID serverUUID = identifiers.getServerUUID(request); // Can throw BadRequestException
return Optional.of(JSONCache.getOrCache(dataID, serverUUID, () -> jsonCreator.apply(serverUUID)));
return Response.builder()
.setMimeType(MimeType.JSON)
.setJSONContent(asyncJSONResolverService.resolve(getTimestamp(request), dataID, serverUUID, jsonCreator).json)
.build();
}
private long getTimestamp(Request request) {
try {
return request.getQuery().get("timestamp")
.map(Long::parseLong)
.orElseGet(System::currentTimeMillis);
} catch (NumberFormatException nonNumberTimestamp) {
throw new BadRequestException("'timestamp' was not a number: " + nonNumberTimestamp.getMessage());
}
}
}

31
Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONFileStorage.java
View File

@ -27,6 +27,8 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
@ -134,4 +136,33 @@ public class JSONFileStorage implements JSONStorage {
}
return Optional.empty();
}
@Override
public void invalidateOlder(String identifier, long timestamp) {
File[] stored = jsonDirectory.toFile().listFiles();
if (stored == null) return;
List<File> toDelete = new ArrayList<>();
for (File file : stored) {
try {
String fileName = file.getName();
if (fileName.endsWith(JSON_FILE_EXTENSION) && fileName.startsWith(identifier)) {
Matcher timestampMatch = timestampRegex.matcher(fileName);
if (timestampMatch.find() && Long.parseLong(timestampMatch.group(1)) < timestamp) {
toDelete.add(file);
}
}
} catch (NumberFormatException e) {
// Ignore this file, malformed timestamp
}
}
for (File fileToDelete : toDelete) {
try {
Files.delete(fileToDelete.toPath());
} catch (IOException e) {
// Failed to delete, set for deletion on next server shutdown.
fileToDelete.deleteOnExit();
}
}
}
}

2
Plan/common/src/main/java/com/djrapitops/plan/storage/json/JSONStorage.java
View File

@ -50,6 +50,8 @@ public interface JSONStorage {
Optional<StoredJSON> fetchJsonMadeAfter(String identifier, long timestamp);
void invalidateOlder(String identifier, long timestamp);
final class StoredJSON {
public final String json;
public final long timestamp;