Disallow robots on Plan pages by FluxCapacitor2 (#1601)

Add robots.txt, X-Robots-Tag header, and <meta> tags disallowing spiders from indexing Plan pages.

Affects issues:
- Close #1575

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java

@@ -66,7 +66,8 @@ public class Contributors {
                 new Contributor("yukieji", LANG),
                 new Contributor("qsefthuopq", LANG),
                 new Contributor("Karlatemp", CODE, LANG),
-                new Contributor("Mastory_Md5", LANG)
+                new Contributor("Mastory_Md5", LANG),
+                new Contributor("FluxCapacitor2", CODE)
         };
         int estimatedLength = contributors.length * 40 + 50;
         StringBuilder html = new StringBuilder(estimatedLength);

Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/RequestHandler.java

@@ -98,6 +98,7 @@ public class RequestHandler implements HttpHandler {
             response.getHeaders().putIfAbsent("Access-Control-Allow-Origin", config.get(WebserverSettings.CORS_ALLOW_ORIGIN));
             response.getHeaders().putIfAbsent("Access-Control-Allow-Methods", "GET, OPTIONS");
             response.getHeaders().putIfAbsent("Access-Control-Allow-Credentials", "true");
+            response.getHeaders().putIfAbsent("X-Robots-Tag", "noindex, nofollow");
             ResponseSender sender = new ResponseSender(addresses, exchange, response);
             sender.send();
         } catch (Exception e) {

Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseFactory.java

@@ -254,6 +254,17 @@ public class ResponseFactory {
         }
     }
 
+    public Response robotsResponse() {
+        try {
+            return Response.builder()
+                    .setMimeType("text/plain")
+                    .setContent(getResource("robots.txt"))
+                    .build();
+        } catch (UncheckedIOException e) {
+            return forInternalError(e, "Could not read robots.txt");
+        }
+    }
+
     public Response pageNotFound404() {
         return notFound404(locale.getString(ErrorPageLang.UNKNOWN_PAGE_404));
     }

Plan/common/src/main/java/com/djrapitops/plan/delivery/webserver/ResponseResolver.java

@@ -113,6 +113,7 @@ public class ResponseResolver {
 
     public void registerPages() {
         String plugin = "Plan";
+        resolverService.registerResolver(plugin, "/robots.txt", (NoAuthResolver) request -> Optional.of(responseFactory.robotsResponse()));
         resolverService.registerResolver(plugin, "/debug", debugPageResolver);
         resolverService.registerResolver(plugin, "/players", playersPageResolver);
         resolverService.registerResolver(plugin, "/player", playerPageResolver);

Plan/common/src/main/resources/assets/plan/web/error.html

@@ -8,6 +8,7 @@
     <meta content="width=device-width, initial-scale=1, maximum-scale=1" name="viewport">
     <meta content="Player Analytics, generic error/debug page" name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>${titleText}</title>
 

Plan/common/src/main/resources/assets/plan/web/login.html

@@ -8,6 +8,7 @@
     <meta content="width=device-width, initial-scale=1, shrink-to-fit=no" name="viewport">
     <meta content="Player Analytics, login page" name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | Login</title>
 

Plan/common/src/main/resources/assets/plan/web/network.html

@@ -9,6 +9,7 @@
     <meta content="Player Analytics, network page that allows gaining insights to how players play on your network & servers."
           name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | Network</title>
 

Plan/common/src/main/resources/assets/plan/web/player.html

@@ -9,6 +9,7 @@
     <meta content="Player Analytics, player page that displays more insights about a specific player"
           name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | ${playerName}</title>
 

Plan/common/src/main/resources/assets/plan/web/players.html

@@ -8,6 +8,7 @@
     <meta content="width=device-width, initial-scale=1, maximum-scale=1" name="viewport">
     <meta content="Player Analytics, players page that displays list of players" name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | Players</title>
 

Plan/common/src/main/resources/assets/plan/web/register.html

@@ -8,6 +8,7 @@
     <meta content="width=device-width, initial-scale=1, shrink-to-fit=no" name="viewport">
     <meta content="Player Analytics, login page" name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | Register</title>
 

Plan/common/src/main/resources/assets/plan/web/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

Plan/common/src/main/resources/assets/plan/web/server.html

@@ -8,6 +8,7 @@
     <meta content="Player Analytics, server analysis page that allows gaining insights to how players play on your server."
           name="description">
     <meta content="Rsl1122" name="author">
+    <meta content="noindex, nofollow" name="robots">
 
     <title>Plan | Server Analysis</title>