Upstream/Plan
1
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2024-08-20 11:42:50 +02:00
Code Issues Projects Releases Wiki Activity

IP storage updated for GDPR compliance

Browse Source
This commit is contained in:
Rsl1122 2018-05-21 11:31:18 +03:00
parent 2c7b514719
commit 71098e59f4
11 changed files with 187 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Plan/src/main/java/com/djrapitops/plan/data/PlayerProfile.java
View File

@ -241,7 +241,7 @@ public class PlayerProfile {
public GeoInfo getMostRecentGeoInfo() {
if (geoInformation.isEmpty()) {
return new GeoInfo("-", "Not Known", MiscUtils.getTime());
return new GeoInfo("-", "Not Known", MiscUtils.getTime(), "");
}
geoInformation.sort(new GeoInfoComparator());
return geoInformation.get(0);

18
Plan/src/main/java/com/djrapitops/plan/data/container/GeoInfo.java
View File

@ -4,8 +4,13 @@
*/
package com.djrapitops.plan.data.container;
import com.djrapitops.plan.utilities.FormatUtils;
import com.djrapitops.plan.utilities.SHA256Hash;
import com.google.common.base.Objects;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
/**
* Data class that contains information about IP and Geolocation.
*
@ -15,12 +20,19 @@ public class GeoInfo {
private final String ip;
private final String geolocation;
private final String ipHash;
private final long lastUsed;
public GeoInfo(String ip, String geolocation, long lastUsed) {
public GeoInfo(String ip, String geolocation, long lastUsed)
throws UnsupportedEncodingException, NoSuchAlgorithmException {
this(FormatUtils.formatIP(ip), geolocation, lastUsed, new SHA256Hash(ip).create());
}
public GeoInfo(String ip, String geolocation, long lastUsed, String ipHash) {
this.ip = ip;
this.geolocation = geolocation;
this.lastUsed = lastUsed;
this.ipHash = ipHash;
}
public String getIp() {
@ -35,6 +47,10 @@ public class GeoInfo {
return lastUsed;
}
public String getIpHash() {
return ipHash;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

6
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/SQLDB.java
View File

@ -133,7 +133,7 @@ public abstract class SQLDB extends Database {
if (newDatabase) {
Log.info("New Database created.");
versionTable.setVersion(16);
versionTable.setVersion(17);
}
int version = versionTable.getVersion();
@ -177,6 +177,10 @@ public abstract class SQLDB extends Database {
worldTimesTable.alterTableV16();
versionTable.setVersion(16);
}
if (version < 17) {
geoInfoTable.alterTableV17();
versionTable.setVersion(17);
}
} catch (SQLException e) {
throw new DBInitException("Failed to set-up Database", e);
}

128
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/tables/GeoInfoTable.java
View File

@ -10,8 +10,14 @@ import com.djrapitops.plan.system.database.databases.sql.statements.Column;
import com.djrapitops.plan.system.database.databases.sql.statements.Select;
import com.djrapitops.plan.system.database.databases.sql.statements.Sql;
import com.djrapitops.plan.system.database.databases.sql.statements.TableSqlParser;
import com.djrapitops.plan.system.settings.Settings;
import com.djrapitops.plugin.api.utility.log.Log;
import com.djrapitops.plugin.task.AbsRunnable;
import com.djrapitops.plugin.task.RunnableFactory;
import com.djrapitops.plugin.utilities.Verify;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
@ -33,11 +39,12 @@ public class GeoInfoTable extends UserIDTable {
insertStatement = "INSERT INTO " + tableName + " ("
+ Col.USER_ID + ", "
+ Col.IP + ", "
+ Col.IP_HASH + ", "
+ Col.GEOLOCATION + ", "
+ Col.LAST_USED
+ ") VALUES ("
+ usersTable.statementSelectID + ", "
+ "?, ?, ?)";
+ "?, ?, ?, ?)";
}
private String insertStatement;
@ -48,6 +55,7 @@ public class GeoInfoTable extends UserIDTable {
.column(Col.USER_ID, Sql.INT).notNull()
.column(Col.IP, Sql.varchar(39)).notNull()
.column(Col.GEOLOCATION, Sql.varchar(50)).notNull()
.column(Col.IP_HASH, Sql.varchar(200))
.column(Col.LAST_USED, Sql.LONG).notNull().defaultValue("0")
.foreignKey(Col.USER_ID, usersTable.getTableName(), UsersTable.Col.ID)
.toString()
@ -64,6 +72,49 @@ public class GeoInfoTable extends UserIDTable {
addColumns(Col.LAST_USED + " bigint NOT NULL DEFAULT 0");
}
public void alterTableV17() {
addColumns(Col.IP_HASH.get() + " varchar(200) DEFAULT ''");
RunnableFactory.createNew("DB Version 16->17", new AbsRunnable() {
@Override
public void run() {
try {
Map<UUID, List<GeoInfo>> allGeoInfo = getAllGeoInfo();
String sql = "UPDATE " + tableName + " SET " +
Col.IP + "=?, " +
Col.IP_HASH + "=? " +
"WHERE " + Col.IP + "=?";
executeBatch(new ExecStatement(sql) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
for (List<GeoInfo> geoInfos : allGeoInfo.values()) {
for (GeoInfo geoInfo : geoInfos) {
try {
GeoInfo updatedInfo = new GeoInfo(
geoInfo.getIp(),
geoInfo.getGeolocation(),
geoInfo.getLastUsed()
);
statement.setString(1, updatedInfo.getIp());
statement.setString(2, updatedInfo.getIpHash());
statement.setString(3, geoInfo.getIp());
} catch (UnsupportedEncodingException | NoSuchAlgorithmException e) {
if (Settings.DEV_MODE.isTrue()) {
Log.toLog(this.getClass(), e);
}
}
}
}
}
});
} catch (SQLException e) {
e.printStackTrace();
}
}
});
}
public List<GeoInfo> getGeoInfo(UUID uuid) throws SQLException {
String sql = "SELECT DISTINCT * FROM " + tableName +
" WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID;
@ -80,8 +131,9 @@ public class GeoInfoTable extends UserIDTable {
while (set.next()) {
String ip = set.getString(Col.IP.get());
String geolocation = set.getString(Col.GEOLOCATION.get());
String ipHash = set.getString(Col.IP_HASH.get());
long lastUsed = set.getLong(Col.LAST_USED.get());
geoInfo.add(new GeoInfo(ip, geolocation, lastUsed));
geoInfo.add(new GeoInfo(ip, geolocation, lastUsed, ipHash));
}
return geoInfo;
}
@ -92,7 +144,7 @@ public class GeoInfoTable extends UserIDTable {
String sql = "UPDATE " + tableName + " SET "
+ Col.LAST_USED + "=?" +
" WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID +
" AND " + Col.IP + "=?" +
" AND " + Col.IP_HASH + "=?" +
" AND " + Col.GEOLOCATION + "=?";
execute(new ExecStatement(sql) {
@ -100,7 +152,7 @@ public class GeoInfoTable extends UserIDTable {
public void prepare(PreparedStatement statement) throws SQLException {
statement.setLong(1, info.getLastUsed());
statement.setString(2, uuid.toString());
statement.setString(3, info.getIp());
statement.setString(3, info.getIpHash());
statement.setString(4, info.getGeolocation());
}
});
@ -120,8 +172,9 @@ public class GeoInfoTable extends UserIDTable {
public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, uuid.toString());
statement.setString(2, info.getIp());
statement.setString(3, info.getGeolocation());
statement.setLong(4, info.getLastUsed());
statement.setString(3, info.getIpHash());
statement.setString(4, info.getGeolocation());
statement.setLong(5, info.getLastUsed());
}
});
}
@ -154,6 +207,7 @@ public class GeoInfoTable extends UserIDTable {
Col.IP + ", " +
Col.GEOLOCATION + ", " +
Col.LAST_USED + ", " +
Col.IP_HASH + ", " +
usersUUIDColumn +
" FROM " + tableName +
" INNER JOIN " + usersTable + " on " + usersIDColumn + "=" + Col.USER_ID;
@ -169,8 +223,9 @@ public class GeoInfoTable extends UserIDTable {
String ip = set.getString(Col.IP.get());
String geolocation = set.getString(Col.GEOLOCATION.get());
String ipHash = set.getString(Col.IP_HASH.get());
long lastUsed = set.getLong(Col.LAST_USED.get());
userGeoInfo.add(new GeoInfo(ip, geolocation, lastUsed));
userGeoInfo.add(new GeoInfo(ip, geolocation, lastUsed, ipHash));
geoLocations.put(uuid, userGeoInfo);
}
@ -203,9 +258,40 @@ public class GeoInfoTable extends UserIDTable {
});
}
public void insertAllGeoInfo(Map<UUID, List<GeoInfo>> allIPsAndGeolocations) throws SQLException {
if (Verify.isEmpty(allIPsAndGeolocations)) {
return;
}
executeBatch(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
// Every User
for (UUID uuid : allIPsAndGeolocations.keySet()) {
// Every GeoInfo
for (GeoInfo info : allIPsAndGeolocations.get(uuid)) {
String ip = info.getIp();
String ipHash = info.getIpHash();
String geoLocation = info.getGeolocation();
long lastUsed = info.getLastUsed();
statement.setString(1, uuid.toString());
statement.setString(2, ip);
statement.setString(3, ipHash);
statement.setString(4, geoLocation);
statement.setLong(5, lastUsed);
statement.addBatch();
}
}
}
});
}
public enum Col implements Column {
USER_ID(UserIDTable.Col.USER_ID.get()),
IP("ip"),
IP_HASH("ip_hash"),
GEOLOCATION("geolocation"),
LAST_USED("last_used");
@ -225,32 +311,4 @@ public class GeoInfoTable extends UserIDTable {
return column;
}
}
public void insertAllGeoInfo(Map<UUID, List<GeoInfo>> allIPsAndGeolocations) throws SQLException {
if (Verify.isEmpty(allIPsAndGeolocations)) {
return;
}
executeBatch(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
// Every User
for (UUID uuid : allIPsAndGeolocations.keySet()) {
// Every GeoInfo
for (GeoInfo info : allIPsAndGeolocations.get(uuid)) {
String ip = info.getIp();
String geoLocation = info.getGeolocation();
long lastUsed = info.getLastUsed();
statement.setString(1, uuid.toString());
statement.setString(2, ip);
statement.setString(3, geoLocation);
statement.setLong(4, lastUsed);
statement.addBatch();
}
}
}
});
}
}

8
Plan/src/main/java/com/djrapitops/plan/system/processing/importing/importers/Importer.java
View File

@ -23,6 +23,8 @@ import com.djrapitops.plugin.api.utility.log.Log;
import com.djrapitops.plugin.utilities.Verify;
import com.google.common.collect.ImmutableMap;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -228,7 +230,11 @@ public abstract class Importer {
return userImportData.getIps().parallelStream()
.map(ip -> {
String geoLoc = GeolocationCache.getCountry(ip);
return new GeoInfo(ip, geoLoc, date);
try {
return new GeoInfo(ip, geoLoc, date);
} catch (UnsupportedEncodingException | NoSuchAlgorithmException e) {
throw new IllegalArgumentException(e);
}
}).collect(Collectors.toList());
}

4
Plan/src/main/java/com/djrapitops/plan/system/processing/processors/player/IPUpdateProcessor.java
View File

@ -13,6 +13,8 @@ import com.djrapitops.plan.system.settings.Settings;
import com.djrapitops.plugin.api.Check;
import com.djrapitops.plugin.api.utility.log.Log;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.util.UUID;
/**
@ -38,7 +40,7 @@ public class IPUpdateProcessor implements CriticalRunnable {
String country = GeolocationCache.getCountry(ip);
try {
Database.getActive().save().geoInfo(uuid, new GeoInfo(ip, country, time));
} catch (DBException e) {
} catch (DBException | UnsupportedEncodingException | NoSuchAlgorithmException e) {
Log.toLog(this.getClass(), e);
}
}

7
Plan/src/main/java/com/djrapitops/plan/utilities/FormatUtils.java
View File

@ -276,10 +276,13 @@ public class FormatUtils {
}
public static String formatIP(String ip) {
if ("localhost".equals(ip)) {
return ip;
}
StringBuilder b = new StringBuilder();
int i = 0;
for (String part : ip.split("\\.")) {
if (i >= 3) {
if (i >= 2) {
break;
}
@ -288,7 +291,7 @@ public class FormatUtils {
i++;
}
return b.append("xx").toString();
return b.append("xx.xx").toString();
}
/**

21
Plan/src/main/java/com/djrapitops/plan/utilities/SHA256Hash.java Normal file
View File

@ -0,0 +1,21 @@
package com.djrapitops.plan.utilities;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
public class SHA256Hash {
private final String original;
public SHA256Hash(String original) {
this.original = original;
}
public String create() throws NoSuchAlgorithmException, UnsupportedEncodingException {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
digest.update(original.getBytes("UTF-8"));
return Base64.getEncoder().encodeToString(digest.digest());
}
}

26
Plan/src/test/java/com/djrapitops/plan/system/database/databases/SQLiteTest.java
View File

@ -22,6 +22,7 @@ import com.djrapitops.plan.system.processing.processors.player.RegisterProcessor
import com.djrapitops.plan.utilities.Base64Util;
import com.djrapitops.plan.utilities.ManageUtils;
import com.djrapitops.plan.utilities.MiscUtils;
import com.djrapitops.plan.utilities.SHA256Hash;
import com.djrapitops.plan.utilities.analysis.MathUtils;
import com.djrapitops.plugin.StaticHolder;
import com.djrapitops.plugin.api.utility.log.Log;
@ -36,8 +37,10 @@ import utilities.TestConstants;
import utilities.TestErrorManager;
import utilities.mocks.SystemMockUtil;
import java.io.UnsupportedEncodingException;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.security.NoSuchAlgorithmException;
import java.sql.SQLException;
import java.util.*;
@ -258,7 +261,7 @@ public class SQLiteTest {
String expectedGeoLoc = "TestLocation";
long time = MiscUtils.getTime();
GeoInfo expected = new GeoInfo(expectedIP, expectedGeoLoc, time);
GeoInfo expected = new GeoInfo(expectedIP, expectedGeoLoc, time, "3");
geoInfoTable.saveGeoInfo(playerUUID, expected);
geoInfoTable.saveGeoInfo(playerUUID, expected);
commitTest();
@ -563,7 +566,7 @@ public class SQLiteTest {
sessionsTable.saveSession(playerUUID, session);
nicknamesTable.saveUserName(playerUUID, "TestNick");
geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L, "3"));
actionsTable.insertAction(playerUUID, new Action(1324L, Actions.FIRST_SESSION, "Add"));
assertTrue(usersTable.isRegistered(playerUUID));
@ -579,7 +582,7 @@ public class SQLiteTest {
}
@Test
public void testRemovalEverything() throws SQLException, DBException {
public void testRemovalEverything() throws SQLException, DBException, UnsupportedEncodingException, NoSuchAlgorithmException {
UserInfoTable userInfoTable = db.getUserInfoTable();
UsersTable usersTable = db.getUsersTable();
SessionsTable sessionsTable = db.getSessionsTable();
@ -608,7 +611,7 @@ public class SQLiteTest {
assertTrue(securityTable.getUsers().isEmpty());
}
private void saveAllData(SQLDB database) throws SQLException {
private void saveAllData(SQLDB database) throws SQLException, UnsupportedEncodingException, NoSuchAlgorithmException {
System.out.println("Saving all possible data to the Database..");
UserInfoTable userInfoTable = database.getUserInfoTable();
UsersTable usersTable = database.getUsersTable();
@ -632,7 +635,8 @@ public class SQLiteTest {
sessionsTable.saveSession(playerUUID, session);
nicknamesTable.saveUserName(playerUUID, "TestNick");
geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L,
new SHA256Hash("1.2.3.4").create()));
actionsTable.insertAction(playerUUID, new Action(1324L, Actions.FIRST_SESSION, "Add"));
assertTrue(usersTable.isRegistered(playerUUID));
@ -756,7 +760,7 @@ public class SQLiteTest {
}
@Test
public void testBackupAndRestore() throws SQLException, DBInitException {
public void testBackupAndRestore() throws SQLException, DBInitException, UnsupportedEncodingException, NoSuchAlgorithmException {
System.out.println("- Creating Backup Database -");
SQLiteDB backup = new SQLiteDB("debug-backup" + MiscUtils.getTime());
backup.init();
@ -894,7 +898,7 @@ public class SQLiteTest {
}
@Test
public void testWorldTableGetWorldNamesNoException() throws SQLException {
public void testWorldTableGetWorldNamesNoException() throws SQLException, UnsupportedEncodingException, NoSuchAlgorithmException {
saveAllData(db);
Set<String> worldNames = db.getWorldTable().getWorldNames(TestConstants.SERVER_UUID);
assertEquals(new HashSet<>(worlds), worldNames);
@ -924,11 +928,11 @@ public class SQLiteTest {
usersTable.registerUser(secondUuid, 0, "");
usersTable.registerUser(thirdUuid, 0, "");
geoInfoTable.saveGeoInfo(firstUuid, new GeoInfo("-", "Test1", 0));
GeoInfo secondInfo = new GeoInfo("-", "Test2", 5);
geoInfoTable.saveGeoInfo(firstUuid, new GeoInfo("-", "Test1", 0, "3"));
GeoInfo secondInfo = new GeoInfo("-", "Test2", 5, "3");
geoInfoTable.saveGeoInfo(firstUuid, secondInfo);
geoInfoTable.saveGeoInfo(secondUuid, new GeoInfo("-", "Test3", 0));
geoInfoTable.saveGeoInfo(thirdUuid, new GeoInfo("-", "Test4", 0));
geoInfoTable.saveGeoInfo(secondUuid, new GeoInfo("-", "Test3", 0, "3"));
geoInfoTable.saveGeoInfo(thirdUuid, new GeoInfo("-", "Test4", 0, "3"));
List<String> geolocations = geoInfoTable.getNetworkGeolocations();
System.out.println(geolocations);

2
Plan/src/test/java/com/djrapitops/plan/utilities/FormatUtilsTest.java
View File

@ -161,7 +161,7 @@ public class FormatUtilsTest {
String ip = "1.2.3.4";
String ip2 = "1.2.3.26";
String ip3 = "1.2.3.235";
String expected = "1.2.3.xx";
String expected = "1.2.xx.xx";
assertEquals(expected, FormatUtils.formatIP(ip));
assertEquals(expected, FormatUtils.formatIP(ip2));

19
Plan/src/test/java/com/djrapitops/plan/utilities/SHA256HashTest.java Normal file
View File

@ -0,0 +1,19 @@
package com.djrapitops.plan.utilities;
import org.junit.Test;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import static org.junit.Assert.assertEquals;
public class SHA256HashTest {
@Test
public void sameStringReturnsSameHash() throws UnsupportedEncodingException, NoSuchAlgorithmException {
String expected = new SHA256Hash("1.3.4.5").create();
String result = new SHA256Hash("1.3.4.5").create();
assertEquals(expected, result);
}
}