From 16a5b41db580dd25a2962a930d361df28cdd6d2a Mon Sep 17 00:00:00 2001 From: Rsl1122 <24460436+Rsl1122@users.noreply.github.com> Date: Mon, 6 Jan 2020 13:18:42 +0200 Subject: [PATCH] Implemented new GeoLite2 & IP2C geolocators - GeoLite2 downloads the file using License key, only if EULA is accepted - Fallback to IP2C if GeoLite2 is not available - Remove GeoIP.dat after successfully downloading GeoLite2-Country.mmdb - Added case where geolocation fails to enable and doesn't cause issues - Adds Apache commons-compress to the dependencies because of a tar archive Affects issues: - Fixed #1273 --- Plan/build.gradle | 1 + .../importing/importers/BukkitImporter.java | 12 +- .../importers/OfflinePlayerImporter.java | 2 +- .../bukkit/PlayerOnlineListener.java | 2 +- .../bungee/PlayerOnlineListener.java | 2 +- Plan/common/build.gradle | 1 + .../plan/exceptions/PreparationException.java | 29 +++ .../plan/gathering/cache/CacheSystem.java | 1 + .../gathering/cache/GeolocationCache.java | 202 ------------------ .../geolocation/GeoLite2Geolocator.java | 150 +++++++++++++ .../geolocation/GeolocationCache.java | 155 ++++++++++++++ .../gathering/geolocation/Geolocator.java | 54 +++++ .../gathering/geolocation/IP2CGeolocator.java | 106 +++++++++ .../config/paths/DataGatheringSettings.java | 1 + .../events/GeoInfoStoreTransaction.java | 15 +- .../resources/assets/plan/bungeeconfig.yml | 3 + .../src/main/resources/assets/plan/config.yml | 3 + .../GeolocationTest.java} | 44 ++-- .../nukkit/PlayerOnlineListener.java | 5 +- .../sponge/PlayerOnlineListener.java | 2 +- .../velocity/PlayerOnlineListener.java | 2 +- 21 files changed, 554 insertions(+), 238 deletions(-) create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/exceptions/PreparationException.java delete mode 100644 Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/GeolocationCache.java create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeoLite2Geolocator.java create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeolocationCache.java create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/Geolocator.java create mode 100644 Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/IP2CGeolocator.java rename Plan/common/src/test/java/com/djrapitops/plan/gathering/{cache/GeolocationCacheTest.java => geolocation/GeolocationTest.java} (68%) diff --git a/Plan/build.gradle b/Plan/build.gradle index 6de6926be..405ddf5bb 100644 --- a/Plan/build.gradle +++ b/Plan/build.gradle @@ -85,6 +85,7 @@ subprojects { ext.httpClientVersion = "4.5.10" ext.commonsTextVersion = "1.8" + ext.commonsCompressVersion = "1.19" ext.htmlCompressorVersion = "1.5.2" ext.caffeineVersion = "2.8.0" ext.h2Version = "1.4.199" diff --git a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/BukkitImporter.java b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/BukkitImporter.java index ffdbed27b..188152b00 100644 --- a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/BukkitImporter.java +++ b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/BukkitImporter.java @@ -18,8 +18,8 @@ package com.djrapitops.plan.gathering.importing.importers; import com.djrapitops.plan.Plan; import com.djrapitops.plan.delivery.domain.Nickname; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.domain.*; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.gathering.importing.data.BukkitUserImportRefiner; import com.djrapitops.plan.gathering.importing.data.ServerImportData; import com.djrapitops.plan.gathering.importing.data.UserImportData; @@ -196,10 +196,10 @@ public abstract class BukkitImporter implements Importer { private List convertGeoInfo(UserImportData userImportData) { long date = System.currentTimeMillis(); - return userImportData.getIps().parallelStream() - .map(ip -> { - String geoLoc = geolocationCache.getCountry(ip); - return new GeoInfo(geoLoc, date); - }).collect(Collectors.toList()); + return userImportData.getIps().stream() + .map(geolocationCache::getCountry) + .filter(Objects::nonNull) + .map(geoLocation -> new GeoInfo(geoLocation, date)) + .collect(Collectors.toList()); } } diff --git a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/OfflinePlayerImporter.java b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/OfflinePlayerImporter.java index e7987473f..df4223d39 100644 --- a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/OfflinePlayerImporter.java +++ b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/OfflinePlayerImporter.java @@ -17,7 +17,7 @@ package com.djrapitops.plan.gathering.importing.importers; import com.djrapitops.plan.Plan; -import com.djrapitops.plan.gathering.cache.GeolocationCache; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.gathering.importing.data.ServerImportData; import com.djrapitops.plan.gathering.importing.data.UserImportData; import com.djrapitops.plan.identification.ServerInfo; diff --git a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/listeners/bukkit/PlayerOnlineListener.java b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/listeners/bukkit/PlayerOnlineListener.java index d371dbfb4..da20936dc 100644 --- a/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/listeners/bukkit/PlayerOnlineListener.java +++ b/Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/listeners/bukkit/PlayerOnlineListener.java @@ -23,10 +23,10 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID; import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.extension.CallEvents; import com.djrapitops.plan.extension.ExtensionServiceImplementation; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.cache.NicknameCache; import com.djrapitops.plan.gathering.cache.SessionCache; import com.djrapitops.plan.gathering.domain.Session; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.gathering.listeners.Status; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.processing.Processing; diff --git a/Plan/bungeecord/src/main/java/com/djrapitops/plan/gathering/listeners/bungee/PlayerOnlineListener.java b/Plan/bungeecord/src/main/java/com/djrapitops/plan/gathering/listeners/bungee/PlayerOnlineListener.java index 393bea71f..8b60f26fe 100644 --- a/Plan/bungeecord/src/main/java/com/djrapitops/plan/gathering/listeners/bungee/PlayerOnlineListener.java +++ b/Plan/bungeecord/src/main/java/com/djrapitops/plan/gathering/listeners/bungee/PlayerOnlineListener.java @@ -22,9 +22,9 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID; import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.extension.CallEvents; import com.djrapitops.plan.extension.ExtensionServiceImplementation; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.cache.SessionCache; import com.djrapitops.plan.gathering.domain.Session; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.processing.Processing; import com.djrapitops.plan.settings.config.PlanConfig; diff --git a/Plan/common/build.gradle b/Plan/common/build.gradle index d03e0f8e6..ddfb987ff 100644 --- a/Plan/common/build.gradle +++ b/Plan/common/build.gradle @@ -4,6 +4,7 @@ dependencies { compile project(path: ":extensions", configuration: 'shadow') compile "org.apache.httpcomponents:httpclient:$httpClientVersion" compile "org.apache.commons:commons-text:$commonsTextVersion" + compile "org.apache.commons:commons-compress:$commonsCompressVersion" compile "com.googlecode.htmlcompressor:htmlcompressor:$htmlCompressorVersion" compile "com.github.ben-manes.caffeine:caffeine:$caffeineVersion" compile "com.h2database:h2:$h2Version" diff --git a/Plan/common/src/main/java/com/djrapitops/plan/exceptions/PreparationException.java b/Plan/common/src/main/java/com/djrapitops/plan/exceptions/PreparationException.java new file mode 100644 index 000000000..bbcdac2b9 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/exceptions/PreparationException.java @@ -0,0 +1,29 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.exceptions; + +/** + * Illegal State somewhere during preparation. + * + * @author Rsl1122 + */ +public class PreparationException extends IllegalStateException { + + public PreparationException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/CacheSystem.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/CacheSystem.java index 1c4b867d7..536f8a37f 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/CacheSystem.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/CacheSystem.java @@ -18,6 +18,7 @@ package com.djrapitops.plan.gathering.cache; import com.djrapitops.plan.SubSystem; import com.djrapitops.plan.exceptions.EnableException; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import javax.inject.Inject; import javax.inject.Singleton; diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/GeolocationCache.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/GeolocationCache.java deleted file mode 100644 index 417aad962..000000000 --- a/Plan/common/src/main/java/com/djrapitops/plan/gathering/cache/GeolocationCache.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * This file is part of Player Analytics (Plan). - * - * Plan is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License v3 as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Plan is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Plan. If not, see . - */ -package com.djrapitops.plan.gathering.cache; - -import com.djrapitops.plan.SubSystem; -import com.djrapitops.plan.exceptions.EnableException; -import com.djrapitops.plan.settings.config.PlanConfig; -import com.djrapitops.plan.settings.config.paths.DataGatheringSettings; -import com.djrapitops.plan.settings.locale.Locale; -import com.djrapitops.plan.settings.locale.lang.PluginLang; -import com.djrapitops.plan.storage.file.PlanFiles; -import com.djrapitops.plugin.logging.L; -import com.djrapitops.plugin.logging.console.PluginLogger; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.maxmind.geoip2.DatabaseReader; -import com.maxmind.geoip2.exception.GeoIp2Exception; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.record.Country; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; -import java.nio.file.Files; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -/** - * This class contains the geolocation cache. - *

- * It caches all IPs with their matching country. - * - * @author Fuzzlemann - */ -@Singleton -public class GeolocationCache implements SubSystem { - - private final Locale locale; - private final PlanFiles files; - private final PlanConfig config; - private final PluginLogger logger; - private final Cache cache; - - private File geolocationDB; - - @Inject - public GeolocationCache( - Locale locale, - PlanFiles files, - PlanConfig config, - PluginLogger logger - ) { - this.locale = locale; - this.files = files; - this.config = config; - this.logger = logger; - - this.cache = Caffeine.newBuilder() - .expireAfterAccess(1, TimeUnit.MINUTES) - .build(); - } - - @Override - public void enable() throws EnableException { - geolocationDB = files.getFileFromPluginFolder("GeoIP.dat"); - if (config.isTrue(DataGatheringSettings.GEOLOCATIONS)) { - try { - checkDB(); - } catch (UnknownHostException e) { - logger.error(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_INTERNET_REQUIRED)); - } catch (IOException e) { - throw new EnableException(locale.getString(PluginLang.ENABLE_FAIL_GEODB_WRITE), e); - } - } else { - logger.log(L.INFO_COLOR, "§e" + locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_DISABLED)); - } - } - - /** - * Retrieves the country in full length (e.g. United States) from the IP Address. - *

- * This method uses {@code cached}, every first access is getting cached and then retrieved later. - * - * @param ipAddress The IP Address from which the country is retrieved - * @return The name of the country in full length. - *

- * An exception from that rule is when the country is unknown or the retrieval of the country failed in any way, - * if that happens, "Not Known" will be returned. - * @see #getUnCachedCountry(String) - */ - public String getCountry(String ipAddress) { - return cache.get(ipAddress, this::getUnCachedCountry); - } - - /** - * Retrieves the country in full length (e.g. United States) from the IP Address. - *

- * This product includes GeoLite2 data created by MaxMind, available from - * http://www.maxmind.com. - * - * @param ipAddress The IP Address from which the country is retrieved - * @return The name of the country in full length. - *

- * An exception from that rule is when the country is unknown or the retrieval of the country failed in any way, - * if that happens, "Not Known" will be returned. - * @see http://maxmind.com - * @see #getCountry(String) - */ - private String getUnCachedCountry(String ipAddress) { - if ("127.0.0.1".equals(ipAddress)) { - return "Local Machine"; - } - try { - checkDB(); - - try ( - // See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows - // for why InputStream is being used here instead. - InputStream in = Files.newInputStream(geolocationDB.toPath()); - DatabaseReader reader = new DatabaseReader.Builder(in).build() - ) { - InetAddress inetAddress = InetAddress.getByName(ipAddress); - - CountryResponse response = reader.country(inetAddress); - Country country = response.getCountry(); - String countryName = country.getName(); - - return countryName != null ? countryName : "Not Known"; - } - - } catch (IOException | GeoIp2Exception e) { - return "Not Known"; - } - } - - /** - * Checks if the DB exists, if not, it downloads it - * - * @throws IOException when an error at download or saving the DB happens - */ - private void checkDB() throws IOException { - if (geolocationDB.exists()) { - return; - } - URL downloadSite = new URL("http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz"); - try ( - InputStream in = downloadSite.openStream(); - GZIPInputStream gzipIn = new GZIPInputStream(in); - ReadableByteChannel rbc = Channels.newChannel(gzipIn); - FileOutputStream fos = new FileOutputStream(geolocationDB.getAbsoluteFile()); - FileChannel channel = fos.getChannel() - ) { - channel.transferFrom(rbc, 0, Long.MAX_VALUE); - } - } - - /** - * Checks if the IP Address is cached - * - * @param ipAddress The IP Address which is checked - * @return true if the IP Address is cached - */ - boolean isCached(String ipAddress) { - return cache.getIfPresent(ipAddress) != null; - } - - @Override - public void disable() { - clearCache(); - } - - /** - * Clears the cache - */ - public void clearCache() { - cache.invalidateAll(); - cache.cleanUp(); - } -} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeoLite2Geolocator.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeoLite2Geolocator.java new file mode 100644 index 000000000..444704f80 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeoLite2Geolocator.java @@ -0,0 +1,150 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.gathering.geolocation; + +import com.djrapitops.plan.exceptions.PreparationException; +import com.djrapitops.plan.settings.config.PlanConfig; +import com.djrapitops.plan.settings.config.paths.DataGatheringSettings; +import com.djrapitops.plan.storage.file.PlanFiles; +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.record.Country; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.utils.IOUtils; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.URL; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +/** + * {@link Geolocator} implementation for MaxMind GeoLite2 database. + *

+ * This product includes GeoLite2 data created by MaxMind, available from + * http://www.maxmind.com. + * + * @author Rsl1122 + * @see http://maxmind.com + */ +@Singleton +public class GeoLite2Geolocator implements Geolocator { + + private final PlanFiles files; + private final PlanConfig config; + + private File geolocationDB; + + @Inject + public GeoLite2Geolocator(PlanFiles files, PlanConfig config) { + this.files = files; + this.config = config; + } + + @Override + public void prepare() throws IOException { + if (config.isFalse(DataGatheringSettings.ACCEPT_GEOLITE2_EULA)) { + throw new PreparationException("Downloading GeoLite2 requires accepting GeoLite2 EULA - see '" + + DataGatheringSettings.ACCEPT_GEOLITE2_EULA.getPath() + "' in the config."); + } + + geolocationDB = files.getFileFromPluginFolder("GeoLite2-Country.mmdb"); + + if (geolocationDB.exists()) { + if (geolocationDB.lastModified() >= System.currentTimeMillis() - TimeUnit.DAYS.toMillis(7L)) { + return; // Database is new enough + } else { + Files.delete(geolocationDB.toPath()); // Delete old data according to restriction 3. in EULA + } + } + + downloadDatabase(); + // Delete old Geolocation database file if it still exists (on success to avoid a no-file situation) + Files.deleteIfExists(files.getFileFromPluginFolder("GeoIP.dat").toPath()); + } + + private void downloadDatabase() throws IOException { + // Avoid Socket leak with the parameters in case download url has proxy + // https://rsl1122.github.io/mishaps/java_socket_leak_incident + Properties properties = System.getProperties(); + properties.setProperty("sun.net.client.defaultConnectTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L))); + properties.setProperty("sun.net.client.defaultReadTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L))); + properties.setProperty("sun.net.http.retryPost", Boolean.toString(false)); + + String downloadFrom = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=DEyDUKfCwNbtc5eK&suffix=tar.gz"; + URL downloadSite = new URL(downloadFrom); + try ( + InputStream in = downloadSite.openStream(); + GZIPInputStream gzipIn = new GZIPInputStream(in); + TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn); + FileOutputStream fos = new FileOutputStream(geolocationDB.getAbsoluteFile()) + ) { + findAndCopyFromTar(tarIn, fos); + } + } + + private void findAndCopyFromTar(TarArchiveInputStream tarIn, FileOutputStream fos) throws IOException { + // Breadth first search + Queue entries = new ArrayDeque<>(); + entries.add(tarIn.getNextTarEntry()); + while (!entries.isEmpty()) { + TarArchiveEntry entry = entries.poll(); + if (entry.isDirectory()) { + entries.addAll(Arrays.asList(entry.getDirectoryEntries())); + } + + // Looking for this file + if (entry.getName().endsWith("GeoLite2-Country.mmdb")) { + IOUtils.copy(tarIn, fos); + break; // Found it + } + + TarArchiveEntry next = tarIn.getNextTarEntry(); + if (next != null) entries.add(next); + } + } + + @Override + public Optional getCountry(InetAddress inetAddress) { + if (inetAddress == null) return Optional.empty(); + if (inetAddress.getHostAddress().contains("127.0.0.1")) return Optional.of("Local Machine"); + + try ( + // See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows + // for why InputStream is being used here instead. + InputStream in = Files.newInputStream(geolocationDB.toPath()); + DatabaseReader reader = new DatabaseReader.Builder(in).build() + ) { + CountryResponse response = reader.country(inetAddress); + Country country = response.getCountry(); + String countryName = country.getName(); + + return Optional.ofNullable(countryName); + } catch (IOException | GeoIp2Exception e) { + return Optional.empty(); + } + } +} \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeolocationCache.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeolocationCache.java new file mode 100644 index 000000000..5df0c5980 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/GeolocationCache.java @@ -0,0 +1,155 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.gathering.geolocation; + +import com.djrapitops.plan.SubSystem; +import com.djrapitops.plan.exceptions.PreparationException; +import com.djrapitops.plan.settings.config.PlanConfig; +import com.djrapitops.plan.settings.config.paths.DataGatheringSettings; +import com.djrapitops.plan.settings.locale.Locale; +import com.djrapitops.plan.settings.locale.lang.PluginLang; +import com.djrapitops.plugin.logging.console.PluginLogger; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.concurrent.TimeUnit; + +/** + * This class contains the geolocation cache. + *

+ * It caches all IPs with their matching country. + * + * @author Rsl1122 + * @author Fuzzlemann + */ +@Singleton +public class GeolocationCache implements SubSystem { + + private final Locale locale; + private final PlanConfig config; + private final PluginLogger logger; + private final Cache cache; + + private final Geolocator geoLite2Geolocator; + private final Geolocator ip2cGeolocator; + + private Geolocator inUseGeolocator; + + @Inject + public GeolocationCache( + Locale locale, + PlanConfig config, + GeoLite2Geolocator geoLite2Geolocator, + IP2CGeolocator ip2cGeolocator, + PluginLogger logger + ) { + this.locale = locale; + this.config = config; + this.geoLite2Geolocator = geoLite2Geolocator; + this.ip2cGeolocator = ip2cGeolocator; + this.logger = logger; + + this.cache = Caffeine.newBuilder() + .expireAfterAccess(1, TimeUnit.MINUTES) + .build(); + } + + @Override + public void enable() { + if (config.isTrue(DataGatheringSettings.GEOLOCATIONS)) { + if (inUseGeolocator == null) tryToPrepareGeoLite2(); + if (inUseGeolocator == null) tryToPrepareIP2CGeolocator(); + if (inUseGeolocator == null) logger.error("Failed to enable geolocation."); + } else { + logger.info(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_DISABLED)); + } + } + + public boolean canGeolocate() { + return inUseGeolocator != null; + } + + private void tryToPrepareIP2CGeolocator() { + logger.warn("Fallback: using IP2C for Geolocation (doesn't support IPv6)."); + try { + ip2cGeolocator.prepare(); + inUseGeolocator = ip2cGeolocator; + } catch (PreparationException e) { + logger.warn(e.getMessage()); + } catch (IOException e) { + logger.error("Fallback to IP2C failed: " + e.getMessage()); + } + } + + public void tryToPrepareGeoLite2() { + try { + geoLite2Geolocator.prepare(); + inUseGeolocator = geoLite2Geolocator; + } catch (PreparationException e) { + logger.info(e.getMessage()); + } catch (UnknownHostException e) { + logger.error(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_INTERNET_REQUIRED)); + } catch (IOException e) { + logger.error(locale.getString(PluginLang.ENABLE_FAIL_GEODB_WRITE) + ": " + e.getMessage()); + } + } + + /** + * Retrieves the country in full length (e.g. United States) from the IP Address. + * + * @param ipAddress The IP Address for which the country is retrieved + * @return The name of the country in full length or null if the country could not be fetched. + */ + public String getCountry(String ipAddress) { + return cache.get(ipAddress, this::getUnCachedCountry); + } + + /** + * Retrieves the country in full length (e.g. United States) from the IP Address. + */ + private String getUnCachedCountry(String ipAddress) { + if (inUseGeolocator == null) return null; + return inUseGeolocator.getCountry(ipAddress).orElse("Not Found"); + } + + /** + * Checks if the IP Address is cached + * + * @param ipAddress The IP Address which is checked + * @return true if the IP Address is cached + */ + boolean isCached(String ipAddress) { + return cache.getIfPresent(ipAddress) != null; + } + + @Override + public void disable() { + clearCache(); + } + + /** + * Clears the cache + */ + public void clearCache() { + cache.invalidateAll(); + cache.cleanUp(); + } +} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/Geolocator.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/Geolocator.java new file mode 100644 index 000000000..5d8f9d969 --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/Geolocator.java @@ -0,0 +1,54 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.gathering.geolocation; + +import com.djrapitops.plan.exceptions.PreparationException; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Optional; + +/** + * Interface for different Geolocation service calls. + * + * @author Rsl1122 + */ +public interface Geolocator { + + /** + * Do everything that is needed for the geolocator to function. + * + * @throws IOException If the preparation fails + * @throws UnknownHostException If preparation requires internet, but internet is not available. + * @throws PreparationException If preparation fails due to Plan settings + */ + void prepare() throws IOException; + + Optional getCountry(InetAddress inetAddress); + + default Optional getCountry(String address) { + try { + InetAddress inetAddress = InetAddress.getByName(address); + return getCountry(inetAddress); + } catch (UnknownHostException e) { + e.printStackTrace(); + return Optional.empty(); + } + } + +} diff --git a/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/IP2CGeolocator.java b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/IP2CGeolocator.java new file mode 100644 index 000000000..f7fe7850e --- /dev/null +++ b/Plan/common/src/main/java/com/djrapitops/plan/gathering/geolocation/IP2CGeolocator.java @@ -0,0 +1,106 @@ +/* + * This file is part of Player Analytics (Plan). + * + * Plan is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License v3 as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Plan is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Plan. If not, see . + */ +package com.djrapitops.plan.gathering.geolocation; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.URL; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** + * Fallback {@link Geolocator} implementation using ip2c. + * + * @author Rsl1122 + * @see + */ +@Singleton +public class IP2CGeolocator implements Geolocator { + + @Inject + public IP2CGeolocator() { + // Inject constructor required for Dagger + } + + @Override + public void prepare() throws IOException { + // Avoid Socket leak with the parameters in case download url has proxy + // https://rsl1122.github.io/mishaps/java_socket_leak_incident + Properties properties = System.getProperties(); + properties.setProperty("sun.net.client.defaultConnectTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L))); + properties.setProperty("sun.net.client.defaultReadTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L))); + properties.setProperty("sun.net.http.retryPost", Boolean.toString(false)); + + // Run a test to see if Internet is available. + readIPFromURL("0.0.0.0"); + } + + @Override + public Optional getCountry(InetAddress inetAddress) { + if (inetAddress instanceof Inet6Address) return Optional.empty(); + String address = inetAddress.getHostAddress(); + return getCountry(address); + } + + @Override + public Optional getCountry(String address) { + try { + return readIPFromURL(address); + } catch (IOException e) { + e.printStackTrace(); + return Optional.empty(); + } + } + + public Optional readIPFromURL(String address) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL("http://ip2c.org/" + address).openConnection(); + connection.setDefaultUseCaches(false); + connection.setUseCaches(false); + connection.connect(); + try ( + InputStream in = connection.getInputStream() + ) { + String answer = readAnswer(in); + return resolveIP(answer); + } + } + + public Optional resolveIP(String s) { + switch (s.charAt(0)) { + case '1': + String[] reply = s.split(";"); + return reply.length >= 4 ? Optional.of(reply[3]) : Optional.empty(); + case '0': // No reply + case '2': // Not in database + default: // Not known char + return Optional.empty(); + } + } + + public String readAnswer(InputStream is) throws IOException { + int read; + StringBuilder answer = new StringBuilder(); + while ((read = is.read()) != -1) answer.append((char) read); + return answer.toString(); + } +} \ No newline at end of file diff --git a/Plan/common/src/main/java/com/djrapitops/plan/settings/config/paths/DataGatheringSettings.java b/Plan/common/src/main/java/com/djrapitops/plan/settings/config/paths/DataGatheringSettings.java index 58051c53e..9e08f1870 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/settings/config/paths/DataGatheringSettings.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/settings/config/paths/DataGatheringSettings.java @@ -27,6 +27,7 @@ import com.djrapitops.plan.settings.config.paths.key.Setting; public class DataGatheringSettings { public static final Setting GEOLOCATIONS = new BooleanSetting("Data_gathering.Geolocations"); + public static final Setting ACCEPT_GEOLITE2_EULA = new BooleanSetting("Data_gathering.Accept_GeoLite2_EULA"); public static final Setting PING = new BooleanSetting("Data_gathering.Ping"); public static final Setting LOG_UNKNOWN_COMMANDS = new BooleanSetting("Data_gathering.Commands.Log_unknown"); public static final Setting COMBINE_COMMAND_ALIASES = new BooleanSetting("Data_gathering.Commands.Log_aliases_as_main_command"); diff --git a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/events/GeoInfoStoreTransaction.java b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/events/GeoInfoStoreTransaction.java index 364f24973..6506a005c 100644 --- a/Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/events/GeoInfoStoreTransaction.java +++ b/Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/events/GeoInfoStoreTransaction.java @@ -32,12 +32,19 @@ import java.util.function.UnaryOperator; public class GeoInfoStoreTransaction extends Transaction { private final UUID playerUUID; - private InetAddress ip; + private String ip; private long time; private UnaryOperator geolocationFunction; private GeoInfo geoInfo; + public GeoInfoStoreTransaction(UUID playerUUID, String ip, long time, UnaryOperator geolocationFunction) { + this.playerUUID = playerUUID; + this.ip = ip; + this.time = time; + this.geolocationFunction = geolocationFunction; + } + public GeoInfoStoreTransaction( UUID playerUUID, InetAddress ip, @@ -45,7 +52,7 @@ public class GeoInfoStoreTransaction extends Transaction { UnaryOperator geolocationFunction ) { this.playerUUID = playerUUID; - this.ip = ip; + this.ip = ip.getHostAddress(); this.time = time; this.geolocationFunction = geolocationFunction; } @@ -56,13 +63,15 @@ public class GeoInfoStoreTransaction extends Transaction { } private GeoInfo createGeoInfo() { - String country = geolocationFunction.apply(ip.getHostAddress()); + // Can return null + String country = geolocationFunction.apply(ip); return new GeoInfo(country, time); } @Override protected void performOperations() { if (geoInfo == null) geoInfo = createGeoInfo(); + if (geoInfo.getGeolocation() == null) return; // Don't save null geolocation. execute(DataStoreQueries.storeGeoInfo(playerUUID, geoInfo)); } } \ No newline at end of file diff --git a/Plan/common/src/main/resources/assets/plan/bungeeconfig.yml b/Plan/common/src/main/resources/assets/plan/bungeeconfig.yml index 1458e7b76..41aee9713 100644 --- a/Plan/common/src/main/resources/assets/plan/bungeeconfig.yml +++ b/Plan/common/src/main/resources/assets/plan/bungeeconfig.yml @@ -59,6 +59,9 @@ Webserver: # ----------------------------------------------------- Data_gathering: Geolocations: true + # Please accept the EULA to download GeoLite2 IP-Country Database + # https://www.maxmind.com/en/geolite2/eula + Accept_GeoLite2_EULA: false Ping: true # ----------------------------------------------------- # Supported time units: MILLISECONDS, SECONDS, MINUTES, HOURS, DAYS diff --git a/Plan/common/src/main/resources/assets/plan/config.yml b/Plan/common/src/main/resources/assets/plan/config.yml index 8c4d30c01..5cac14a49 100644 --- a/Plan/common/src/main/resources/assets/plan/config.yml +++ b/Plan/common/src/main/resources/assets/plan/config.yml @@ -61,6 +61,9 @@ Webserver: # ----------------------------------------------------- Data_gathering: Geolocations: true + # Please accept the EULA to download GeoLite2 IP-Country Database + # https://www.maxmind.com/en/geolite2/eula + Accept_GeoLite2_EULA: false Ping: true Commands: Log_unknown: false diff --git a/Plan/common/src/test/java/com/djrapitops/plan/gathering/cache/GeolocationCacheTest.java b/Plan/common/src/test/java/com/djrapitops/plan/gathering/geolocation/GeolocationTest.java similarity index 68% rename from Plan/common/src/test/java/com/djrapitops/plan/gathering/cache/GeolocationCacheTest.java rename to Plan/common/src/test/java/com/djrapitops/plan/gathering/geolocation/GeolocationTest.java index d8f3469d8..7c9651d6f 100644 --- a/Plan/common/src/test/java/com/djrapitops/plan/gathering/cache/GeolocationCacheTest.java +++ b/Plan/common/src/test/java/com/djrapitops/plan/gathering/geolocation/GeolocationTest.java @@ -14,9 +14,8 @@ * You should have received a copy of the GNU Lesser General Public License * along with Plan. If not, see . */ -package com.djrapitops.plan.gathering.cache; +package com.djrapitops.plan.gathering.geolocation; -import com.djrapitops.plan.exceptions.EnableException; import com.djrapitops.plan.settings.config.PlanConfig; import com.djrapitops.plan.settings.config.paths.DataGatheringSettings; import com.djrapitops.plan.settings.locale.Locale; @@ -41,16 +40,18 @@ import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; /** - * Tests for {@link GeolocationCache}. + * Tests for Geolocation functionality. * + * @author Rsl1122 * @author Fuzzlemann */ @RunWith(JUnitPlatform.class) @ExtendWith(MockitoExtension.class) -class GeolocationCacheTest { +class GeolocationTest { private static final Map TEST_DATA = new HashMap<>(); private static File IP_STORE; @@ -65,29 +66,35 @@ class GeolocationCacheTest { @BeforeAll static void setUpTestData(@TempDir Path tempDir) { - GeolocationCacheTest.tempDir = tempDir; - IP_STORE = GeolocationCacheTest.tempDir.resolve("GeoIP.dat").toFile(); + GeolocationTest.tempDir = tempDir; + IP_STORE = GeolocationTest.tempDir.resolve("GeoLite2-Country.mmdb").toFile(); - TEST_DATA.put("8.8.8.8", "United States"); - TEST_DATA.put("8.8.4.4", "United States"); - TEST_DATA.put("4.4.2.2", "United States"); - TEST_DATA.put("208.67.222.222", "United States"); - TEST_DATA.put("208.67.220.220", "United States"); + TEST_DATA.put("8.8.8.8", "United States"); // California, US + TEST_DATA.put("8.8.4.4", "United States"); // California, US + TEST_DATA.put("4.4.2.2", "United States"); // Colorado, US + TEST_DATA.put("156.53.159.86", "United States"); // Oregon, US + TEST_DATA.put("208.67.222.222", "United States"); // California, US + TEST_DATA.put("208.67.220.220", "United States"); // California, US TEST_DATA.put("205.210.42.205", "Canada"); TEST_DATA.put("64.68.200.200", "Canada"); - TEST_DATA.put("0.0.0.0", "Not Known"); + TEST_DATA.put("0.0.0.0", "Not Found"); // Invalid IP TEST_DATA.put("127.0.0.1", "Local Machine"); } @BeforeEach - void setUpCache() throws EnableException { + void setUpCache() { when(config.isTrue(DataGatheringSettings.GEOLOCATIONS)).thenReturn(true); - when(files.getFileFromPluginFolder("GeoIP.dat")).thenReturn(IP_STORE); + lenient().when(config.isTrue(DataGatheringSettings.ACCEPT_GEOLITE2_EULA)).thenReturn(true); + when(files.getFileFromPluginFolder("GeoLite2-Country.mmdb")).thenReturn(IP_STORE); + when(files.getFileFromPluginFolder("GeoIP.dat")).thenReturn(tempDir.resolve("Non-file").toFile()); assertTrue(config.isTrue(DataGatheringSettings.GEOLOCATIONS)); - underTest = new GeolocationCache(new Locale(), files, config, new TestPluginLogger()); + GeoLite2Geolocator geoLite2Geolocator = new GeoLite2Geolocator(files, config); + underTest = new GeolocationCache(new Locale(), config, geoLite2Geolocator, new IP2CGeolocator(), new TestPluginLogger()); underTest.enable(); + + assertTrue(underTest.canGeolocate()); } @AfterEach @@ -100,11 +107,10 @@ class GeolocationCacheTest { void countryIsFetched() { for (Map.Entry entry : TEST_DATA.entrySet()) { String ip = entry.getKey(); - String expCountry = entry.getValue(); + String expected = entry.getValue(); + String result = underTest.getCountry(ip); - String country = underTest.getCountry(ip); - - assertEquals(expCountry, country); + assertEquals(expected, result, "Tested " + ip + ", expected: <" + expected + "> but was: <" + result + '>'); } } diff --git a/Plan/nukkit/src/main/java/com/djrapitops/plan/gathering/listeners/nukkit/PlayerOnlineListener.java b/Plan/nukkit/src/main/java/com/djrapitops/plan/gathering/listeners/nukkit/PlayerOnlineListener.java index 9e1367039..a8c5c3fa0 100644 --- a/Plan/nukkit/src/main/java/com/djrapitops/plan/gathering/listeners/nukkit/PlayerOnlineListener.java +++ b/Plan/nukkit/src/main/java/com/djrapitops/plan/gathering/listeners/nukkit/PlayerOnlineListener.java @@ -31,12 +31,11 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID; import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.extension.CallEvents; import com.djrapitops.plan.extension.ExtensionServiceImplementation; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.cache.NicknameCache; import com.djrapitops.plan.gathering.cache.SessionCache; import com.djrapitops.plan.gathering.domain.GMTimes; -import com.djrapitops.plan.gathering.domain.GeoInfo; import com.djrapitops.plan.gathering.domain.Session; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.gathering.listeners.Status; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.processing.Processing; @@ -168,7 +167,7 @@ public class PlayerOnlineListener implements Listener { boolean gatheringGeolocations = config.isTrue(DataGatheringSettings.GEOLOCATIONS); if (gatheringGeolocations) { database.executeTransaction( - new GeoInfoStoreTransaction(playerUUID, new GeoInfo(geolocationCache.getCountry(address), time)) + new GeoInfoStoreTransaction(playerUUID, address, time, geolocationCache::getCountry) ); } diff --git a/Plan/sponge/src/main/java/com/djrapitops/plan/gathering/listeners/sponge/PlayerOnlineListener.java b/Plan/sponge/src/main/java/com/djrapitops/plan/gathering/listeners/sponge/PlayerOnlineListener.java index 31cce10cd..eb11e8bde 100644 --- a/Plan/sponge/src/main/java/com/djrapitops/plan/gathering/listeners/sponge/PlayerOnlineListener.java +++ b/Plan/sponge/src/main/java/com/djrapitops/plan/gathering/listeners/sponge/PlayerOnlineListener.java @@ -23,10 +23,10 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID; import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.extension.CallEvents; import com.djrapitops.plan.extension.ExtensionServiceImplementation; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.cache.NicknameCache; import com.djrapitops.plan.gathering.cache.SessionCache; import com.djrapitops.plan.gathering.domain.Session; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.gathering.listeners.Status; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.processing.Processing; diff --git a/Plan/velocity/src/main/java/com/djrapitops/plan/gathering/listeners/velocity/PlayerOnlineListener.java b/Plan/velocity/src/main/java/com/djrapitops/plan/gathering/listeners/velocity/PlayerOnlineListener.java index 85adbe36c..b05f01e9a 100644 --- a/Plan/velocity/src/main/java/com/djrapitops/plan/gathering/listeners/velocity/PlayerOnlineListener.java +++ b/Plan/velocity/src/main/java/com/djrapitops/plan/gathering/listeners/velocity/PlayerOnlineListener.java @@ -22,9 +22,9 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID; import com.djrapitops.plan.delivery.webserver.cache.JSONCache; import com.djrapitops.plan.extension.CallEvents; import com.djrapitops.plan.extension.ExtensionServiceImplementation; -import com.djrapitops.plan.gathering.cache.GeolocationCache; import com.djrapitops.plan.gathering.cache.SessionCache; import com.djrapitops.plan.gathering.domain.Session; +import com.djrapitops.plan.gathering.geolocation.GeolocationCache; import com.djrapitops.plan.identification.ServerInfo; import com.djrapitops.plan.processing.Processing; import com.djrapitops.plan.settings.config.PlanConfig;