Implemented new GeoLite2 & IP2C geolocators

- GeoLite2 downloads the file using License key, only if EULA is accepted
- Fallback to IP2C if GeoLite2 is not available
- Remove GeoIP.dat after successfully downloading GeoLite2-Country.mmdb
- Added case where geolocation fails to enable and doesn't cause issues

- Adds Apache commons-compress to the dependencies because of a tar archive

Affects issues:
- Fixed #1273
This commit is contained in:
Rsl1122 2020-01-06 13:18:42 +02:00
parent 8d86a752a9
commit 16a5b41db5
21 changed files with 554 additions and 238 deletions

View File

@ -85,6 +85,7 @@ subprojects {
ext.httpClientVersion = "4.5.10"
ext.commonsTextVersion = "1.8"
ext.commonsCompressVersion = "1.19"
ext.htmlCompressorVersion = "1.5.2"
ext.caffeineVersion = "2.8.0"
ext.h2Version = "1.4.199"

View File

@ -18,8 +18,8 @@ package com.djrapitops.plan.gathering.importing.importers;
import com.djrapitops.plan.Plan;
import com.djrapitops.plan.delivery.domain.Nickname;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.domain.*;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.gathering.importing.data.BukkitUserImportRefiner;
import com.djrapitops.plan.gathering.importing.data.ServerImportData;
import com.djrapitops.plan.gathering.importing.data.UserImportData;
@ -196,10 +196,10 @@ public abstract class BukkitImporter implements Importer {
private List<GeoInfo> convertGeoInfo(UserImportData userImportData) {
long date = System.currentTimeMillis();
return userImportData.getIps().parallelStream()
.map(ip -> {
String geoLoc = geolocationCache.getCountry(ip);
return new GeoInfo(geoLoc, date);
}).collect(Collectors.toList());
return userImportData.getIps().stream()
.map(geolocationCache::getCountry)
.filter(Objects::nonNull)
.map(geoLocation -> new GeoInfo(geoLocation, date))
.collect(Collectors.toList());
}
}

View File

@ -17,7 +17,7 @@
package com.djrapitops.plan.gathering.importing.importers;
import com.djrapitops.plan.Plan;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.gathering.importing.data.ServerImportData;
import com.djrapitops.plan.gathering.importing.data.UserImportData;
import com.djrapitops.plan.identification.ServerInfo;

View File

@ -23,10 +23,10 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.extension.CallEvents;
import com.djrapitops.plan.extension.ExtensionServiceImplementation;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.cache.NicknameCache;
import com.djrapitops.plan.gathering.cache.SessionCache;
import com.djrapitops.plan.gathering.domain.Session;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.gathering.listeners.Status;
import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.processing.Processing;

View File

@ -22,9 +22,9 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.extension.CallEvents;
import com.djrapitops.plan.extension.ExtensionServiceImplementation;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.cache.SessionCache;
import com.djrapitops.plan.gathering.domain.Session;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.processing.Processing;
import com.djrapitops.plan.settings.config.PlanConfig;

View File

@ -4,6 +4,7 @@ dependencies {
compile project(path: ":extensions", configuration: 'shadow')
compile "org.apache.httpcomponents:httpclient:$httpClientVersion"
compile "org.apache.commons:commons-text:$commonsTextVersion"
compile "org.apache.commons:commons-compress:$commonsCompressVersion"
compile "com.googlecode.htmlcompressor:htmlcompressor:$htmlCompressorVersion"
compile "com.github.ben-manes.caffeine:caffeine:$caffeineVersion"
compile "com.h2database:h2:$h2Version"

View File

@ -0,0 +1,29 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.exceptions;
/**
* Illegal State somewhere during preparation.
*
* @author Rsl1122
*/
public class PreparationException extends IllegalStateException {
public PreparationException(String s) {
super(s);
}
}

View File

@ -18,6 +18,7 @@ package com.djrapitops.plan.gathering.cache;
import com.djrapitops.plan.SubSystem;
import com.djrapitops.plan.exceptions.EnableException;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import javax.inject.Inject;
import javax.inject.Singleton;

View File

@ -1,202 +0,0 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.cache;
import com.djrapitops.plan.SubSystem;
import com.djrapitops.plan.exceptions.EnableException;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
import com.djrapitops.plan.settings.locale.Locale;
import com.djrapitops.plan.settings.locale.lang.PluginLang;
import com.djrapitops.plan.storage.file.PlanFiles;
import com.djrapitops.plugin.logging.L;
import com.djrapitops.plugin.logging.console.PluginLogger;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.record.Country;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.file.Files;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
/**
* This class contains the geolocation cache.
* <p>
* It caches all IPs with their matching country.
*
* @author Fuzzlemann
*/
@Singleton
public class GeolocationCache implements SubSystem {
private final Locale locale;
private final PlanFiles files;
private final PlanConfig config;
private final PluginLogger logger;
private final Cache<String, String> cache;
private File geolocationDB;
@Inject
public GeolocationCache(
Locale locale,
PlanFiles files,
PlanConfig config,
PluginLogger logger
) {
this.locale = locale;
this.files = files;
this.config = config;
this.logger = logger;
this.cache = Caffeine.newBuilder()
.expireAfterAccess(1, TimeUnit.MINUTES)
.build();
}
@Override
public void enable() throws EnableException {
geolocationDB = files.getFileFromPluginFolder("GeoIP.dat");
if (config.isTrue(DataGatheringSettings.GEOLOCATIONS)) {
try {
checkDB();
} catch (UnknownHostException e) {
logger.error(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_INTERNET_REQUIRED));
} catch (IOException e) {
throw new EnableException(locale.getString(PluginLang.ENABLE_FAIL_GEODB_WRITE), e);
}
} else {
logger.log(L.INFO_COLOR, "§e" + locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_DISABLED));
}
}
/**
* Retrieves the country in full length (e.g. United States) from the IP Address.
* <p>
* This method uses {@code cached}, every first access is getting cached and then retrieved later.
*
* @param ipAddress The IP Address from which the country is retrieved
* @return The name of the country in full length.
* <p>
* An exception from that rule is when the country is unknown or the retrieval of the country failed in any way,
* if that happens, "Not Known" will be returned.
* @see #getUnCachedCountry(String)
*/
public String getCountry(String ipAddress) {
return cache.get(ipAddress, this::getUnCachedCountry);
}
/**
* Retrieves the country in full length (e.g. United States) from the IP Address.
* <p>
* This product includes GeoLite2 data created by MaxMind, available from
* <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
*
* @param ipAddress The IP Address from which the country is retrieved
* @return The name of the country in full length.
* <p>
* An exception from that rule is when the country is unknown or the retrieval of the country failed in any way,
* if that happens, "Not Known" will be returned.
* @see <a href="http://maxmind.com">http://maxmind.com</a>
* @see #getCountry(String)
*/
private String getUnCachedCountry(String ipAddress) {
if ("127.0.0.1".equals(ipAddress)) {
return "Local Machine";
}
try {
checkDB();
try (
// See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows
// for why InputStream is being used here instead.
InputStream in = Files.newInputStream(geolocationDB.toPath());
DatabaseReader reader = new DatabaseReader.Builder(in).build()
) {
InetAddress inetAddress = InetAddress.getByName(ipAddress);
CountryResponse response = reader.country(inetAddress);
Country country = response.getCountry();
String countryName = country.getName();
return countryName != null ? countryName : "Not Known";
}
} catch (IOException | GeoIp2Exception e) {
return "Not Known";
}
}
/**
* Checks if the DB exists, if not, it downloads it
*
* @throws IOException when an error at download or saving the DB happens
*/
private void checkDB() throws IOException {
if (geolocationDB.exists()) {
return;
}
URL downloadSite = new URL("http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz");
try (
InputStream in = downloadSite.openStream();
GZIPInputStream gzipIn = new GZIPInputStream(in);
ReadableByteChannel rbc = Channels.newChannel(gzipIn);
FileOutputStream fos = new FileOutputStream(geolocationDB.getAbsoluteFile());
FileChannel channel = fos.getChannel()
) {
channel.transferFrom(rbc, 0, Long.MAX_VALUE);
}
}
/**
* Checks if the IP Address is cached
*
* @param ipAddress The IP Address which is checked
* @return true if the IP Address is cached
*/
boolean isCached(String ipAddress) {
return cache.getIfPresent(ipAddress) != null;
}
@Override
public void disable() {
clearCache();
}
/**
* Clears the cache
*/
public void clearCache() {
cache.invalidateAll();
cache.cleanUp();
}
}

View File

@ -0,0 +1,150 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.geolocation;
import com.djrapitops.plan.exceptions.PreparationException;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
import com.djrapitops.plan.storage.file.PlanFiles;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.record.Country;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.utils.IOUtils;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URL;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
/**
* {@link Geolocator} implementation for MaxMind GeoLite2 database.
* <p>
* This product includes GeoLite2 data created by MaxMind, available from
* <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
*
* @author Rsl1122
* @see <a href="http://maxmind.com">http://maxmind.com</a>
*/
@Singleton
public class GeoLite2Geolocator implements Geolocator {
private final PlanFiles files;
private final PlanConfig config;
private File geolocationDB;
@Inject
public GeoLite2Geolocator(PlanFiles files, PlanConfig config) {
this.files = files;
this.config = config;
}
@Override
public void prepare() throws IOException {
if (config.isFalse(DataGatheringSettings.ACCEPT_GEOLITE2_EULA)) {
throw new PreparationException("Downloading GeoLite2 requires accepting GeoLite2 EULA - see '"
+ DataGatheringSettings.ACCEPT_GEOLITE2_EULA.getPath() + "' in the config.");
}
geolocationDB = files.getFileFromPluginFolder("GeoLite2-Country.mmdb");
if (geolocationDB.exists()) {
if (geolocationDB.lastModified() >= System.currentTimeMillis() - TimeUnit.DAYS.toMillis(7L)) {
return; // Database is new enough
} else {
Files.delete(geolocationDB.toPath()); // Delete old data according to restriction 3. in EULA
}
}
downloadDatabase();
// Delete old Geolocation database file if it still exists (on success to avoid a no-file situation)
Files.deleteIfExists(files.getFileFromPluginFolder("GeoIP.dat").toPath());
}
private void downloadDatabase() throws IOException {
// Avoid Socket leak with the parameters in case download url has proxy
// https://rsl1122.github.io/mishaps/java_socket_leak_incident
Properties properties = System.getProperties();
properties.setProperty("sun.net.client.defaultConnectTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
properties.setProperty("sun.net.client.defaultReadTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
properties.setProperty("sun.net.http.retryPost", Boolean.toString(false));
String downloadFrom = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=DEyDUKfCwNbtc5eK&suffix=tar.gz";
URL downloadSite = new URL(downloadFrom);
try (
InputStream in = downloadSite.openStream();
GZIPInputStream gzipIn = new GZIPInputStream(in);
TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn);
FileOutputStream fos = new FileOutputStream(geolocationDB.getAbsoluteFile())
) {
findAndCopyFromTar(tarIn, fos);
}
}
private void findAndCopyFromTar(TarArchiveInputStream tarIn, FileOutputStream fos) throws IOException {
// Breadth first search
Queue<TarArchiveEntry> entries = new ArrayDeque<>();
entries.add(tarIn.getNextTarEntry());
while (!entries.isEmpty()) {
TarArchiveEntry entry = entries.poll();
if (entry.isDirectory()) {
entries.addAll(Arrays.asList(entry.getDirectoryEntries()));
}
// Looking for this file
if (entry.getName().endsWith("GeoLite2-Country.mmdb")) {
IOUtils.copy(tarIn, fos);
break; // Found it
}
TarArchiveEntry next = tarIn.getNextTarEntry();
if (next != null) entries.add(next);
}
}
@Override
public Optional<String> getCountry(InetAddress inetAddress) {
if (inetAddress == null) return Optional.empty();
if (inetAddress.getHostAddress().contains("127.0.0.1")) return Optional.of("Local Machine");
try (
// See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows
// for why InputStream is being used here instead.
InputStream in = Files.newInputStream(geolocationDB.toPath());
DatabaseReader reader = new DatabaseReader.Builder(in).build()
) {
CountryResponse response = reader.country(inetAddress);
Country country = response.getCountry();
String countryName = country.getName();
return Optional.ofNullable(countryName);
} catch (IOException | GeoIp2Exception e) {
return Optional.empty();
}
}
}

View File

@ -0,0 +1,155 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.geolocation;
import com.djrapitops.plan.SubSystem;
import com.djrapitops.plan.exceptions.PreparationException;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
import com.djrapitops.plan.settings.locale.Locale;
import com.djrapitops.plan.settings.locale.lang.PluginLang;
import com.djrapitops.plugin.logging.console.PluginLogger;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.concurrent.TimeUnit;
/**
* This class contains the geolocation cache.
* <p>
* It caches all IPs with their matching country.
*
* @author Rsl1122
* @author Fuzzlemann
*/
@Singleton
public class GeolocationCache implements SubSystem {
private final Locale locale;
private final PlanConfig config;
private final PluginLogger logger;
private final Cache<String, String> cache;
private final Geolocator geoLite2Geolocator;
private final Geolocator ip2cGeolocator;
private Geolocator inUseGeolocator;
@Inject
public GeolocationCache(
Locale locale,
PlanConfig config,
GeoLite2Geolocator geoLite2Geolocator,
IP2CGeolocator ip2cGeolocator,
PluginLogger logger
) {
this.locale = locale;
this.config = config;
this.geoLite2Geolocator = geoLite2Geolocator;
this.ip2cGeolocator = ip2cGeolocator;
this.logger = logger;
this.cache = Caffeine.newBuilder()
.expireAfterAccess(1, TimeUnit.MINUTES)
.build();
}
@Override
public void enable() {
if (config.isTrue(DataGatheringSettings.GEOLOCATIONS)) {
if (inUseGeolocator == null) tryToPrepareGeoLite2();
if (inUseGeolocator == null) tryToPrepareIP2CGeolocator();
if (inUseGeolocator == null) logger.error("Failed to enable geolocation.");
} else {
logger.info(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_DISABLED));
}
}
public boolean canGeolocate() {
return inUseGeolocator != null;
}
private void tryToPrepareIP2CGeolocator() {
logger.warn("Fallback: using IP2C for Geolocation (doesn't support IPv6).");
try {
ip2cGeolocator.prepare();
inUseGeolocator = ip2cGeolocator;
} catch (PreparationException e) {
logger.warn(e.getMessage());
} catch (IOException e) {
logger.error("Fallback to IP2C failed: " + e.getMessage());
}
}
public void tryToPrepareGeoLite2() {
try {
geoLite2Geolocator.prepare();
inUseGeolocator = geoLite2Geolocator;
} catch (PreparationException e) {
logger.info(e.getMessage());
} catch (UnknownHostException e) {
logger.error(locale.getString(PluginLang.ENABLE_NOTIFY_GEOLOCATIONS_INTERNET_REQUIRED));
} catch (IOException e) {
logger.error(locale.getString(PluginLang.ENABLE_FAIL_GEODB_WRITE) + ": " + e.getMessage());
}
}
/**
* Retrieves the country in full length (e.g. United States) from the IP Address.
*
* @param ipAddress The IP Address for which the country is retrieved
* @return The name of the country in full length or null if the country could not be fetched.
*/
public String getCountry(String ipAddress) {
return cache.get(ipAddress, this::getUnCachedCountry);
}
/**
* Retrieves the country in full length (e.g. United States) from the IP Address.
*/
private String getUnCachedCountry(String ipAddress) {
if (inUseGeolocator == null) return null;
return inUseGeolocator.getCountry(ipAddress).orElse("Not Found");
}
/**
* Checks if the IP Address is cached
*
* @param ipAddress The IP Address which is checked
* @return true if the IP Address is cached
*/
boolean isCached(String ipAddress) {
return cache.getIfPresent(ipAddress) != null;
}
@Override
public void disable() {
clearCache();
}
/**
* Clears the cache
*/
public void clearCache() {
cache.invalidateAll();
cache.cleanUp();
}
}

View File

@ -0,0 +1,54 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.geolocation;
import com.djrapitops.plan.exceptions.PreparationException;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Optional;
/**
* Interface for different Geolocation service calls.
*
* @author Rsl1122
*/
public interface Geolocator {
/**
* Do everything that is needed for the geolocator to function.
*
* @throws IOException If the preparation fails
* @throws UnknownHostException If preparation requires internet, but internet is not available.
* @throws PreparationException If preparation fails due to Plan settings
*/
void prepare() throws IOException;
Optional<String> getCountry(InetAddress inetAddress);
default Optional<String> getCountry(String address) {
try {
InetAddress inetAddress = InetAddress.getByName(address);
return getCountry(inetAddress);
} catch (UnknownHostException e) {
e.printStackTrace();
return Optional.empty();
}
}
}

View File

@ -0,0 +1,106 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.geolocation;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.URL;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
/**
* Fallback {@link Geolocator} implementation using ip2c.
*
* @author Rsl1122
* @see <a href="about.ip2c.org"></a>
*/
@Singleton
public class IP2CGeolocator implements Geolocator {
@Inject
public IP2CGeolocator() {
// Inject constructor required for Dagger
}
@Override
public void prepare() throws IOException {
// Avoid Socket leak with the parameters in case download url has proxy
// https://rsl1122.github.io/mishaps/java_socket_leak_incident
Properties properties = System.getProperties();
properties.setProperty("sun.net.client.defaultConnectTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
properties.setProperty("sun.net.client.defaultReadTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
properties.setProperty("sun.net.http.retryPost", Boolean.toString(false));
// Run a test to see if Internet is available.
readIPFromURL("0.0.0.0");
}
@Override
public Optional<String> getCountry(InetAddress inetAddress) {
if (inetAddress instanceof Inet6Address) return Optional.empty();
String address = inetAddress.getHostAddress();
return getCountry(address);
}
@Override
public Optional<String> getCountry(String address) {
try {
return readIPFromURL(address);
} catch (IOException e) {
e.printStackTrace();
return Optional.empty();
}
}
public Optional<String> readIPFromURL(String address) throws IOException {
HttpURLConnection connection = (HttpURLConnection) new URL("http://ip2c.org/" + address).openConnection();
connection.setDefaultUseCaches(false);
connection.setUseCaches(false);
connection.connect();
try (
InputStream in = connection.getInputStream()
) {
String answer = readAnswer(in);
return resolveIP(answer);
}
}
public Optional<String> resolveIP(String s) {
switch (s.charAt(0)) {
case '1':
String[] reply = s.split(";");
return reply.length >= 4 ? Optional.of(reply[3]) : Optional.empty();
case '0': // No reply
case '2': // Not in database
default: // Not known char
return Optional.empty();
}
}
public String readAnswer(InputStream is) throws IOException {
int read;
StringBuilder answer = new StringBuilder();
while ((read = is.read()) != -1) answer.append((char) read);
return answer.toString();
}
}

View File

@ -27,6 +27,7 @@ import com.djrapitops.plan.settings.config.paths.key.Setting;
public class DataGatheringSettings {
public static final Setting<Boolean> GEOLOCATIONS = new BooleanSetting("Data_gathering.Geolocations");
public static final Setting<Boolean> ACCEPT_GEOLITE2_EULA = new BooleanSetting("Data_gathering.Accept_GeoLite2_EULA");
public static final Setting<Boolean> PING = new BooleanSetting("Data_gathering.Ping");
public static final Setting<Boolean> LOG_UNKNOWN_COMMANDS = new BooleanSetting("Data_gathering.Commands.Log_unknown");
public static final Setting<Boolean> COMBINE_COMMAND_ALIASES = new BooleanSetting("Data_gathering.Commands.Log_aliases_as_main_command");

View File

@ -32,12 +32,19 @@ import java.util.function.UnaryOperator;
public class GeoInfoStoreTransaction extends Transaction {
private final UUID playerUUID;
private InetAddress ip;
private String ip;
private long time;
private UnaryOperator<String> geolocationFunction;
private GeoInfo geoInfo;
public GeoInfoStoreTransaction(UUID playerUUID, String ip, long time, UnaryOperator<String> geolocationFunction) {
this.playerUUID = playerUUID;
this.ip = ip;
this.time = time;
this.geolocationFunction = geolocationFunction;
}
public GeoInfoStoreTransaction(
UUID playerUUID,
InetAddress ip,
@ -45,7 +52,7 @@ public class GeoInfoStoreTransaction extends Transaction {
UnaryOperator<String> geolocationFunction
) {
this.playerUUID = playerUUID;
this.ip = ip;
this.ip = ip.getHostAddress();
this.time = time;
this.geolocationFunction = geolocationFunction;
}
@ -56,13 +63,15 @@ public class GeoInfoStoreTransaction extends Transaction {
}
private GeoInfo createGeoInfo() {
String country = geolocationFunction.apply(ip.getHostAddress());
// Can return null
String country = geolocationFunction.apply(ip);
return new GeoInfo(country, time);
}
@Override
protected void performOperations() {
if (geoInfo == null) geoInfo = createGeoInfo();
if (geoInfo.getGeolocation() == null) return; // Don't save null geolocation.
execute(DataStoreQueries.storeGeoInfo(playerUUID, geoInfo));
}
}

View File

@ -59,6 +59,9 @@ Webserver:
# -----------------------------------------------------
Data_gathering:
Geolocations: true
# Please accept the EULA to download GeoLite2 IP-Country Database
# https://www.maxmind.com/en/geolite2/eula
Accept_GeoLite2_EULA: false
Ping: true
# -----------------------------------------------------
# Supported time units: MILLISECONDS, SECONDS, MINUTES, HOURS, DAYS

View File

@ -61,6 +61,9 @@ Webserver:
# -----------------------------------------------------
Data_gathering:
Geolocations: true
# Please accept the EULA to download GeoLite2 IP-Country Database
# https://www.maxmind.com/en/geolite2/eula
Accept_GeoLite2_EULA: false
Ping: true
Commands:
Log_unknown: false

View File

@ -14,9 +14,8 @@
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.cache;
package com.djrapitops.plan.gathering.geolocation;
import com.djrapitops.plan.exceptions.EnableException;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
import com.djrapitops.plan.settings.locale.Locale;
@ -41,16 +40,18 @@ import java.util.HashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Tests for {@link GeolocationCache}.
* Tests for Geolocation functionality.
*
* @author Rsl1122
* @author Fuzzlemann
*/
@RunWith(JUnitPlatform.class)
@ExtendWith(MockitoExtension.class)
class GeolocationCacheTest {
class GeolocationTest {
private static final Map<String, String> TEST_DATA = new HashMap<>();
private static File IP_STORE;
@ -65,29 +66,35 @@ class GeolocationCacheTest {
@BeforeAll
static void setUpTestData(@TempDir Path tempDir) {
GeolocationCacheTest.tempDir = tempDir;
IP_STORE = GeolocationCacheTest.tempDir.resolve("GeoIP.dat").toFile();
GeolocationTest.tempDir = tempDir;
IP_STORE = GeolocationTest.tempDir.resolve("GeoLite2-Country.mmdb").toFile();
TEST_DATA.put("8.8.8.8", "United States");
TEST_DATA.put("8.8.4.4", "United States");
TEST_DATA.put("4.4.2.2", "United States");
TEST_DATA.put("208.67.222.222", "United States");
TEST_DATA.put("208.67.220.220", "United States");
TEST_DATA.put("8.8.8.8", "United States"); // California, US
TEST_DATA.put("8.8.4.4", "United States"); // California, US
TEST_DATA.put("4.4.2.2", "United States"); // Colorado, US
TEST_DATA.put("156.53.159.86", "United States"); // Oregon, US
TEST_DATA.put("208.67.222.222", "United States"); // California, US
TEST_DATA.put("208.67.220.220", "United States"); // California, US
TEST_DATA.put("205.210.42.205", "Canada");
TEST_DATA.put("64.68.200.200", "Canada");
TEST_DATA.put("0.0.0.0", "Not Known");
TEST_DATA.put("0.0.0.0", "Not Found"); // Invalid IP
TEST_DATA.put("127.0.0.1", "Local Machine");
}
@BeforeEach
void setUpCache() throws EnableException {
void setUpCache() {
when(config.isTrue(DataGatheringSettings.GEOLOCATIONS)).thenReturn(true);
when(files.getFileFromPluginFolder("GeoIP.dat")).thenReturn(IP_STORE);
lenient().when(config.isTrue(DataGatheringSettings.ACCEPT_GEOLITE2_EULA)).thenReturn(true);
when(files.getFileFromPluginFolder("GeoLite2-Country.mmdb")).thenReturn(IP_STORE);
when(files.getFileFromPluginFolder("GeoIP.dat")).thenReturn(tempDir.resolve("Non-file").toFile());
assertTrue(config.isTrue(DataGatheringSettings.GEOLOCATIONS));
underTest = new GeolocationCache(new Locale(), files, config, new TestPluginLogger());
GeoLite2Geolocator geoLite2Geolocator = new GeoLite2Geolocator(files, config);
underTest = new GeolocationCache(new Locale(), config, geoLite2Geolocator, new IP2CGeolocator(), new TestPluginLogger());
underTest.enable();
assertTrue(underTest.canGeolocate());
}
@AfterEach
@ -100,11 +107,10 @@ class GeolocationCacheTest {
void countryIsFetched() {
for (Map.Entry<String, String> entry : TEST_DATA.entrySet()) {
String ip = entry.getKey();
String expCountry = entry.getValue();
String expected = entry.getValue();
String result = underTest.getCountry(ip);
String country = underTest.getCountry(ip);
assertEquals(expCountry, country);
assertEquals(expected, result, "Tested " + ip + ", expected: <" + expected + "> but was: <" + result + '>');
}
}

View File

@ -31,12 +31,11 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.extension.CallEvents;
import com.djrapitops.plan.extension.ExtensionServiceImplementation;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.cache.NicknameCache;
import com.djrapitops.plan.gathering.cache.SessionCache;
import com.djrapitops.plan.gathering.domain.GMTimes;
import com.djrapitops.plan.gathering.domain.GeoInfo;
import com.djrapitops.plan.gathering.domain.Session;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.gathering.listeners.Status;
import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.processing.Processing;
@ -168,7 +167,7 @@ public class PlayerOnlineListener implements Listener {
boolean gatheringGeolocations = config.isTrue(DataGatheringSettings.GEOLOCATIONS);
if (gatheringGeolocations) {
database.executeTransaction(
new GeoInfoStoreTransaction(playerUUID, new GeoInfo(geolocationCache.getCountry(address), time))
new GeoInfoStoreTransaction(playerUUID, address, time, geolocationCache::getCountry)
);
}

View File

@ -23,10 +23,10 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.extension.CallEvents;
import com.djrapitops.plan.extension.ExtensionServiceImplementation;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.cache.NicknameCache;
import com.djrapitops.plan.gathering.cache.SessionCache;
import com.djrapitops.plan.gathering.domain.Session;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.gathering.listeners.Status;
import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.processing.Processing;

View File

@ -22,9 +22,9 @@ import com.djrapitops.plan.delivery.webserver.cache.DataID;
import com.djrapitops.plan.delivery.webserver.cache.JSONCache;
import com.djrapitops.plan.extension.CallEvents;
import com.djrapitops.plan.extension.ExtensionServiceImplementation;
import com.djrapitops.plan.gathering.cache.GeolocationCache;
import com.djrapitops.plan.gathering.cache.SessionCache;
import com.djrapitops.plan.gathering.domain.Session;
import com.djrapitops.plan.gathering.geolocation.GeolocationCache;
import com.djrapitops.plan.identification.ServerInfo;
import com.djrapitops.plan.processing.Processing;
import com.djrapitops.plan.settings.config.PlanConfig;