Implement Access log functionality to Plan

- Store access log in database, clean logs after 30 days by default
- Add Webserver.Security.Access_log.Print_to_console setting
- Add Webserver.Security.Access_log.Remove_logs_after_days setting

Affects issues:
- Close #2328
This commit is contained in:
Aurora Lahtela 2022-06-24 11:09:14 +03:00
parent 58cd2d6a8f
commit 6aae823850
17 changed files with 307 additions and 26 deletions

View File

@ -96,6 +96,8 @@ public final class URIQuery {
}
public String asString() {
if (byKey.isEmpty()) return "";
StringBuilder builder = new StringBuilder("?");
int i = 0;
int max = byKey.size();

View File

@ -48,6 +48,10 @@ public class WebserverConfiguration {
return webserverLogMessages;
}
public boolean logAccessToConsole() {
return config.isTrue(WebserverSettings.LOG_ACCESS_TO_CONSOLE);
}
public boolean isAuthenticationDisabled() {
return config.isTrue(WebserverSettings.DISABLED_AUTHENTICATION);
}

View File

@ -0,0 +1,69 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.delivery.webserver.http;
import com.djrapitops.plan.delivery.web.resolver.Response;
import com.djrapitops.plan.delivery.web.resolver.request.Request;
import com.djrapitops.plan.delivery.webserver.configuration.WebserverConfiguration;
import com.djrapitops.plan.exceptions.database.DBOpException;
import com.djrapitops.plan.storage.database.DBSystem;
import com.djrapitops.plan.storage.database.transactions.events.StoreRequestTransaction;
import com.djrapitops.plan.utilities.logging.ErrorContext;
import com.djrapitops.plan.utilities.logging.ErrorLogger;
import net.playeranalytics.plugin.server.PluginLogger;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.concurrent.CompletionException;
@Singleton
public class AccessLogger {
private final WebserverConfiguration webserverConfiguration;
private final DBSystem dbSystem;
private final PluginLogger logger;
private final ErrorLogger errorLogger;
@Inject
public AccessLogger(WebserverConfiguration webserverConfiguration, DBSystem dbSystem, PluginLogger logger, ErrorLogger errorLogger) {
this.webserverConfiguration = webserverConfiguration;
this.dbSystem = dbSystem;
this.logger = logger;
this.errorLogger = errorLogger;
}
public void log(InternalRequest internalRequest, Request request, Response response) {
if (webserverConfiguration.logAccessToConsole()) {
logger.info("Access Log: " + internalRequest.getMethod() + " " + getRequestURI(internalRequest, request) + " (from " + internalRequest.getAccessAddress(webserverConfiguration) + ") - " + response.getCode());
}
try {
dbSystem.getDatabase().executeTransaction(
new StoreRequestTransaction(webserverConfiguration, internalRequest, request, response)
);
} catch (CompletionException | DBOpException e) {
errorLogger.warn(e, ErrorContext.builder()
.related("Logging request failed")
.related(getRequestURI(internalRequest, request))
.build());
}
}
private String getRequestURI(InternalRequest internalRequest, Request request) {
return request != null ? request.getPath().asString() + request.getQuery().asString()
: internalRequest.getRequestedURIString();
}
}

View File

@ -34,6 +34,8 @@ import java.util.Optional;
*/
public interface InternalRequest {
long getTimestamp();
default String getAccessAddress(WebserverConfiguration webserverConfiguration) {
AccessAddressPolicy accessAddressPolicy = webserverConfiguration.getAccessAddressPolicy();
if (accessAddressPolicy == AccessAddressPolicy.X_FORWARDED_FOR_HEADER) {
@ -52,6 +54,8 @@ public interface InternalRequest {
List<Cookie> getCookies();
String getMethod();
String getAccessAddressFromSocketIp();
String getAccessAddressFromHeader();

View File

@ -53,6 +53,12 @@ public class JettyInternalRequest implements InternalRequest {
this.authenticationExtractor = authenticationExtractor;
}
@Override
public long getTimestamp() {return baseRequest.getTimeStamp();}
@Override
public String getMethod() {return baseRequest.getMethod();}
@Override
public String getAccessAddressFromSocketIp() {
return baseRequest.getRemoteAddr();

View File

@ -39,12 +39,14 @@ public class RequestHandler {
private final ResponseResolver responseResolver;
private final PassBruteForceGuard bruteForceGuard;
private final AccessLogger accessLogger;
@Inject
public RequestHandler(WebserverConfiguration webserverConfiguration, ResponseFactory responseFactory, ResponseResolver responseResolver) {
public RequestHandler(WebserverConfiguration webserverConfiguration, ResponseFactory responseFactory, ResponseResolver responseResolver, AccessLogger accessLogger) {
this.webserverConfiguration = webserverConfiguration;
this.responseFactory = responseFactory;
this.responseResolver = responseResolver;
this.accessLogger = accessLogger;
bruteForceGuard = new PassBruteForceGuard();
}
@ -52,37 +54,38 @@ public class RequestHandler {
public Response getResponse(InternalRequest internalRequest) {
String accessAddress = internalRequest.getAccessAddress(webserverConfiguration);
Response response;
Request request = null;
if (bruteForceGuard.shouldPreventRequest(accessAddress)) {
return responseFactory.failedLoginAttempts403();
}
if (!webserverConfiguration.getAllowedIpList().isAllowed(accessAddress)) {
response = responseFactory.failedLoginAttempts403();
} else if (!webserverConfiguration.getAllowedIpList().isAllowed(accessAddress)) {
webserverConfiguration.getWebserverLogMessages()
.warnAboutWhitelistBlock(accessAddress, internalRequest.getRequestedURIString());
return responseFactory.ipWhitelist403(accessAddress);
response = responseFactory.ipWhitelist403(accessAddress);
} else {
try {
request = internalRequest.toRequest();
response = attemptToResolve(request, accessAddress);
} catch (WebUserAuthException thrownByAuthentication) {
response = processFailedAuthentication(internalRequest, accessAddress, thrownByAuthentication);
}
}
Response response = attemptToResolve(internalRequest);
response.getHeaders().putIfAbsent("Access-Control-Allow-Origin", webserverConfiguration.getAllowedCorsOrigin());
response.getHeaders().putIfAbsent("Access-Control-Allow-Methods", "GET, OPTIONS");
response.getHeaders().putIfAbsent("Access-Control-Allow-Credentials", "true");
response.getHeaders().putIfAbsent("X-Robots-Tag", "noindex, nofollow");
accessLogger.log(internalRequest, request, response);
return response;
}
private Response attemptToResolve(InternalRequest internalRequest) {
String accessAddress = internalRequest.getAccessAddress(webserverConfiguration);
try {
Request request = internalRequest.toRequest();
Response response = protocolUpgradeResponse(request)
.orElseGet(() -> responseResolver.getResponse(request));
request.getUser().ifPresent(user -> processSuccessfulLogin(response.getCode(), accessAddress));
return response;
} catch (WebUserAuthException thrownByAuthentication) {
return processFailedAuthentication(internalRequest, accessAddress, thrownByAuthentication);
}
private Response attemptToResolve(Request request, String accessAddress) {
Response response = protocolUpgradeResponse(request)
.orElseGet(() -> responseResolver.getResponse(request));
request.getUser().ifPresent(user -> processSuccessfulLogin(response.getCode(), accessAddress));
return response;
}
private Optional<Response> protocolUpgradeResponse(Request request) {

View File

@ -42,6 +42,7 @@ public class WebserverSettings {
public static final Setting<List<String>> WHITELIST = new StringListSetting("Webserver.Security.IP_whitelist.Whitelist");
public static final Setting<Boolean> DISABLED = new BooleanSetting("Webserver.Disable_Webserver");
public static final Setting<Boolean> DISABLED_AUTHENTICATION = new BooleanSetting("Webserver.Security.Disable_authentication");
public static final Setting<Boolean> LOG_ACCESS_TO_CONSOLE = new BooleanSetting("Webserver.Security.Access_log.Print_to_console");
public static final Setting<String> EXTERNAL_LINK = new StringSetting("Webserver.External_Webserver_address");
public static final Setting<Long> REDUCED_REFRESH_BARRIER = new TimeSetting("Webserver.Cache.Reduced_refresh_barrier");
@ -49,7 +50,7 @@ public class WebserverSettings {
public static final Setting<Long> INVALIDATE_DISK_CACHE = new TimeSetting("Webserver.Cache.Invalidate_disk_cache_after");
public static final Setting<Long> INVALIDATE_MEMORY_CACHE = new TimeSetting("Webserver.Cache.Invalidate_memory_cache_after", TimeUnit.MINUTES.toMillis(5L));
public static final Setting<Long> COOKIES_EXPIRE_AFTER = new TimeSetting("Webserver.Security.Cookies_expire_after", TimeUnit.HOURS.toMillis(2L));
public static final Setting<Integer> REMOVE_ACCESS_LOG_AFTER_DAYS = new IntegerSetting("Webserver.Security.Access_log.Remove_logs_after_days");
private WebserverSettings() {
/* static variable class */
}

View File

@ -50,6 +50,7 @@ public abstract class Sql {
public static final String IS_NOT_NULL = " IS NOT NULL";
public static final String LIMIT = " LIMIT ";
public static final String OFFSET = " OFFSET ";
public static final String TEXT = "TEXT";
private static final String FLOOR = "FLOOR(";
private static final String MIN = "MIN(";

View File

@ -0,0 +1,55 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.sql.tables;
import com.djrapitops.plan.storage.database.DBType;
import com.djrapitops.plan.storage.database.sql.building.CreateTableBuilder;
import com.djrapitops.plan.storage.database.sql.building.Sql;
public class AccessLogTable {
public static final String TABLE_NAME = "plan_access_log";
public static final String ID = "id";
public static final String TIME = "time";
public static final String FROM_IP = "from_ip";
public static final String REQUEST_METHOD = "request_method";
public static final String REQUEST_URI = "request_uri";
public static final String RESPONSE_CODE = "response_code";
public static final String USERNAME = "username";
public static final String INSERT_WITH_USER = "INSERT INTO " + TABLE_NAME + " (" +
TIME + ',' + FROM_IP + ',' + REQUEST_METHOD + ',' + REQUEST_URI + ',' + RESPONSE_CODE + ',' + USERNAME +
") VALUES (?, ?, ?, ?, ?, ?)";
public static final String INSERT_NO_USER = "INSERT INTO " + TABLE_NAME + " (" +
TIME + ',' + FROM_IP + ',' + REQUEST_METHOD + ',' + REQUEST_URI + ',' + RESPONSE_CODE +
") VALUES (?, ?, ?, ?, ?)";
private AccessLogTable() {
/* Static constant class */
}
public static String createTableSql(DBType dbType) {
return CreateTableBuilder.create(TABLE_NAME, dbType)
.column(ID, Sql.INT).primaryKey()
.column(TIME, Sql.LONG).notNull()
.column(FROM_IP, Sql.varchar(45)) // Max IPv6 text length 45 chars
.column(REQUEST_METHOD, Sql.varchar(8)).notNull()
.column(REQUEST_URI, Sql.TEXT).notNull()
.column(RESPONSE_CODE, Sql.INT)
.column(USERNAME, Sql.varchar(100))
.build();
}
}

View File

@ -0,0 +1,89 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.transactions.events;
import com.djrapitops.plan.delivery.web.resolver.Response;
import com.djrapitops.plan.delivery.web.resolver.request.Request;
import com.djrapitops.plan.delivery.web.resolver.request.WebUser;
import com.djrapitops.plan.delivery.webserver.configuration.WebserverConfiguration;
import com.djrapitops.plan.delivery.webserver.http.InternalRequest;
import com.djrapitops.plan.storage.database.sql.tables.AccessLogTable;
import com.djrapitops.plan.storage.database.transactions.ExecStatement;
import com.djrapitops.plan.storage.database.transactions.Transaction;
import org.apache.commons.lang3.StringUtils;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.util.Optional;
public class StoreRequestTransaction extends Transaction {
private final WebserverConfiguration webserverConfiguration;
private final InternalRequest internalRequest;
private final Request request; // can be null
private final Response response;
public StoreRequestTransaction(WebserverConfiguration webserverConfiguration, InternalRequest internalRequest, Request request, Response response) {
this.webserverConfiguration = webserverConfiguration;
this.internalRequest = internalRequest;
this.request = request;
this.response = response;
}
@Override
protected void performOperations() {
if (request == null || request.getUser().isEmpty()) { // login failed / auth disabled
execute(new ExecStatement(AccessLogTable.INSERT_NO_USER) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setLong(1, internalRequest.getTimestamp());
statement.setString(2, internalRequest.getAccessAddress(webserverConfiguration));
String method = internalRequest.getMethod();
statement.setString(3, method != null ? method : "?");
statement.setString(4, getTruncatedURI());
statement.setInt(5, response.getCode());
}
});
} else {
execute(new ExecStatement(AccessLogTable.INSERT_WITH_USER) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setLong(1, internalRequest.getTimestamp());
statement.setString(2, internalRequest.getAccessAddress(webserverConfiguration));
statement.setString(3, request.getMethod());
statement.setString(4, getTruncatedURI());
statement.setInt(5, response.getCode());
Optional<String> webUsername = request.getUser().map(WebUser::getUsername);
if (webUsername.isPresent()) {
statement.setString(6, webUsername.get());
} else {
statement.setNull(6, Types.VARCHAR);
}
}
});
}
}
private String getTruncatedURI() {
String uri = request != null ? request.getPath().asString() + request.getQuery().asString()
: internalRequest.getRequestedURIString();
return StringUtils.truncate(uri, 65000);
}
}

View File

@ -48,6 +48,7 @@ public class CreateTablesTransaction extends OperationCriticalTransaction {
execute(SecurityTable.createTableSQL(dbType));
execute(SettingsTable.createTableSQL(dbType));
execute(CookieTable.createTableSQL(dbType));
execute(AccessLogTable.createTableSql(dbType));
// DataExtension tables
execute(ExtensionIconTable.createTableSQL(dbType));

View File

@ -0,0 +1,37 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.transactions.init;
import com.djrapitops.plan.storage.database.sql.tables.AccessLogTable;
import com.djrapitops.plan.storage.database.transactions.ThrowawayTransaction;
import static com.djrapitops.plan.storage.database.sql.building.Sql.DELETE_FROM;
import static com.djrapitops.plan.storage.database.sql.building.Sql.WHERE;
public class RemoveOldAccessLogTransaction extends ThrowawayTransaction {
private final long thresholdMs;
public RemoveOldAccessLogTransaction(long thresholdMs) {
this.thresholdMs = thresholdMs;
}
@Override
protected void performOperations() {
execute(DELETE_FROM + AccessLogTable.TABLE_NAME + WHERE + AccessLogTable.TIME + "<" + (System.currentTimeMillis() - thresholdMs));
}
}

View File

@ -24,6 +24,7 @@ import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.query.QuerySvc;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.TimeSettings;
import com.djrapitops.plan.settings.config.paths.WebserverSettings;
import com.djrapitops.plan.settings.locale.Locale;
import com.djrapitops.plan.settings.locale.lang.PluginLang;
import com.djrapitops.plan.storage.database.DBSystem;
@ -35,6 +36,7 @@ import com.djrapitops.plan.storage.database.sql.tables.SessionsTable;
import com.djrapitops.plan.storage.database.sql.tables.UsersTable;
import com.djrapitops.plan.storage.database.transactions.commands.RemovePlayerTransaction;
import com.djrapitops.plan.storage.database.transactions.init.RemoveDuplicateUserInfoTransaction;
import com.djrapitops.plan.storage.database.transactions.init.RemoveOldAccessLogTransaction;
import com.djrapitops.plan.storage.database.transactions.init.RemoveOldExtensionsTransaction;
import com.djrapitops.plan.storage.database.transactions.init.RemoveOldSampledDataTransaction;
import com.djrapitops.plan.utilities.logging.ErrorLogger;
@ -103,6 +105,7 @@ public class DBCleanTask extends TaskSystem.Task {
try {
if (database.getState() != Database.State.CLOSED) {
database.executeTransaction(new RemoveOldAccessLogTransaction(TimeUnit.DAYS.toMillis(config.get(WebserverSettings.REMOVE_ACCESS_LOG_AFTER_DAYS))));
database.executeTransaction(new RemoveOldSampledDataTransaction(
serverInfo.getServerUUID(),
config.get(TimeSettings.DELETE_TPS_DATA_AFTER),

View File

@ -69,6 +69,9 @@ Webserver:
# Allows using the whitelist & brute-force shield with a reverse-proxy.
# ! Make sure non-proxy access is not possible, it would allow IP spoofing !
Use_X-Forwarded-For_Header: false
Access_log:
Print_to_console: false
Remove_logs_after_days: 30
IP_whitelist: false
Whitelist:
- "192.168.0.0"

View File

@ -71,6 +71,9 @@ Webserver:
# Allows using the whitelist with a reverse-proxy.
# ! Make sure non-proxy access is not possible, it would allow IP spoofing !
Use_X-Forwarded-For_Header: false
Access_log:
Print_to_console: false
Remove_logs_after_days: 30
IP_whitelist: false
Whitelist:
- "192.168.0.0"

View File

@ -27,7 +27,6 @@ import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.mockito.Mockito;
import utilities.dagger.PlanPluginComponent;
import utilities.mocks.PluginMockComponent;
import java.nio.file.Path;
@ -45,10 +44,10 @@ class PlanCommandTest {
@BeforeEach
void preparePlanCommand(@TempDir Path tempDir) throws Exception {
PlanPluginComponent component = new PluginMockComponent(tempDir).getComponent();
underTest = component.planCommand();
system = component.system();
PluginMockComponent mockComponent = new PluginMockComponent(tempDir);
system = mockComponent.getPlanSystem();
system.enable();
underTest = mockComponent.getComponent().planCommand();
}
@AfterEach

View File

@ -48,8 +48,9 @@ class PlanPlaceholdersTest {
@BeforeAll
static void prepareSystem(@TempDir Path tempDir) throws Exception {
component = new PluginMockComponent(tempDir).getComponent();
component.system().enable();
PluginMockComponent mockComponent = new PluginMockComponent(tempDir);
component = mockComponent.getComponent();
mockComponent.getPlanSystem().enable();
serverUUID = component.system().getServerInfo().getServerUUID();
underTest = component.placeholders();