From 01217e43eae3330158e544e14104bd021a9240bc Mon Sep 17 00:00:00 2001 From: FluxCapacitor <31071265+FluxCapacitor2@users.noreply.github.com> Date: Mon, 12 Oct 2020 12:51:25 -0400 Subject: [PATCH] Disallow robots on Plan pages by FluxCapacitor2 (#1601) Add robots.txt, X-Robots-Tag header, and tags disallowing spiders from indexing Plan pages. Affects issues: - Close #1575 --- .../plan/delivery/rendering/html/Contributors.java | 3 ++- .../plan/delivery/webserver/RequestHandler.java | 1 + .../plan/delivery/webserver/ResponseFactory.java | 13 ++++++++++++- .../plan/delivery/webserver/ResponseResolver.java | 1 + .../src/main/resources/assets/plan/web/error.html | 1 + .../src/main/resources/assets/plan/web/login.html | 1 + .../src/main/resources/assets/plan/web/network.html | 1 + .../src/main/resources/assets/plan/web/player.html | 1 + .../src/main/resources/assets/plan/web/players.html | 1 + .../main/resources/assets/plan/web/register.html | 1 + .../src/main/resources/assets/plan/web/robots.txt | 2 ++ .../src/main/resources/assets/plan/web/server.html | 1 + 12 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 Plan/common/src/main/resources/assets/plan/web/robots.txt diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java index 369bb0dcf..6ec0d002a 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java @@ -66,7 +66,8 @@ public class Contributors { new Contributor("yukieji", LANG), new Contributor("qsefthuopq", LANG), new Contributor("Karlatemp", CODE, LANG), - new Contributor("Mastory_Md5", LANG) + new Contributor("Mastory_Md5", LANG), + new Contributor("FluxCapacitor2", CODE) }; int estimatedLength = contributors.length * 40 + 50; StringBuilder html = new StringBuilder(estimatedLength); diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/RequestHandler.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/RequestHandler.java index 83408647e..f10615839 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/RequestHandler.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/RequestHandler.java @@ -98,6 +98,7 @@ public class RequestHandler implements HttpHandler { response.getHeaders().putIfAbsent("Access-Control-Allow-Origin", config.get(WebserverSettings.CORS_ALLOW_ORIGIN)); response.getHeaders().putIfAbsent("Access-Control-Allow-Methods", "GET, OPTIONS"); response.getHeaders().putIfAbsent("Access-Control-Allow-Credentials", "true"); + response.getHeaders().putIfAbsent("X-Robots-Tag", "noindex, nofollow"); ResponseSender sender = new ResponseSender(addresses, exchange, response); sender.send(); } catch (Exception e) { diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseFactory.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseFactory.java index ae181b719..3562e5c5a 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseFactory.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseFactory.java @@ -254,6 +254,17 @@ public class ResponseFactory { } } + public Response robotsResponse() { + try { + return Response.builder() + .setMimeType("text/plain") + .setContent(getResource("robots.txt")) + .build(); + } catch (UncheckedIOException e) { + return forInternalError(e, "Could not read robots.txt"); + } + } + public Response pageNotFound404() { return notFound404(locale.getString(ErrorPageLang.UNKNOWN_PAGE_404)); } @@ -412,4 +423,4 @@ public class ResponseFactory { return forInternalError(e, "Failed to generate player page"); } } -} \ No newline at end of file +} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseResolver.java b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseResolver.java index 4b502c38e..5dbc4769f 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseResolver.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseResolver.java @@ -113,6 +113,7 @@ public class ResponseResolver { public void registerPages() { String plugin = "Plan"; + resolverService.registerResolver(plugin, "/robots.txt", (NoAuthResolver) request -> Optional.of(responseFactory.robotsResponse())); resolverService.registerResolver(plugin, "/debug", debugPageResolver); resolverService.registerResolver(plugin, "/players", playersPageResolver); resolverService.registerResolver(plugin, "/player", playerPageResolver); diff --git a/Plan/common/src/main/resources/assets/plan/web/error.html b/Plan/common/src/main/resources/assets/plan/web/error.html index e50c1e23a..93283cf66 100644 --- a/Plan/common/src/main/resources/assets/plan/web/error.html +++ b/Plan/common/src/main/resources/assets/plan/web/error.html @@ -8,6 +8,7 @@ + ${titleText} diff --git a/Plan/common/src/main/resources/assets/plan/web/login.html b/Plan/common/src/main/resources/assets/plan/web/login.html index b0f4e03e1..f64f6ec25 100644 --- a/Plan/common/src/main/resources/assets/plan/web/login.html +++ b/Plan/common/src/main/resources/assets/plan/web/login.html @@ -8,6 +8,7 @@ + Plan | Login diff --git a/Plan/common/src/main/resources/assets/plan/web/network.html b/Plan/common/src/main/resources/assets/plan/web/network.html index b190c5917..ee29b40a8 100644 --- a/Plan/common/src/main/resources/assets/plan/web/network.html +++ b/Plan/common/src/main/resources/assets/plan/web/network.html @@ -9,6 +9,7 @@ + Plan | Network diff --git a/Plan/common/src/main/resources/assets/plan/web/player.html b/Plan/common/src/main/resources/assets/plan/web/player.html index 460f195b0..7829993ce 100644 --- a/Plan/common/src/main/resources/assets/plan/web/player.html +++ b/Plan/common/src/main/resources/assets/plan/web/player.html @@ -9,6 +9,7 @@ + Plan | ${playerName} diff --git a/Plan/common/src/main/resources/assets/plan/web/players.html b/Plan/common/src/main/resources/assets/plan/web/players.html index b0f93ceaf..798583620 100644 --- a/Plan/common/src/main/resources/assets/plan/web/players.html +++ b/Plan/common/src/main/resources/assets/plan/web/players.html @@ -8,6 +8,7 @@ + Plan | Players diff --git a/Plan/common/src/main/resources/assets/plan/web/register.html b/Plan/common/src/main/resources/assets/plan/web/register.html index 4493938e6..ee76db33f 100644 --- a/Plan/common/src/main/resources/assets/plan/web/register.html +++ b/Plan/common/src/main/resources/assets/plan/web/register.html @@ -8,6 +8,7 @@ + Plan | Register diff --git a/Plan/common/src/main/resources/assets/plan/web/robots.txt b/Plan/common/src/main/resources/assets/plan/web/robots.txt new file mode 100644 index 000000000..1f53798bb --- /dev/null +++ b/Plan/common/src/main/resources/assets/plan/web/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/Plan/common/src/main/resources/assets/plan/web/server.html b/Plan/common/src/main/resources/assets/plan/web/server.html index 8053eb2c4..d38ef57bc 100644 --- a/Plan/common/src/main/resources/assets/plan/web/server.html +++ b/Plan/common/src/main/resources/assets/plan/web/server.html @@ -8,6 +8,7 @@ + Plan | Server Analysis