Upstream/Plan
1
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2024-09-07 04:18:26 +02:00
Code Issues Projects Releases Wiki Activity

Deleted IP related code

Browse Source
This commit is contained in:
Rsl1122 2019-09-21 12:36:35 +03:00
parent cc0737d6da
commit 826faa1b1a
14 changed files with 41 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Plan/bukkit/src/main/java/com/djrapitops/plan/gathering/importing/importers/BukkitImporter.java
View File

@ -200,7 +200,7 @@ public abstract class BukkitImporter implements Importer {
return userImportData.getIps().parallelStream()
.map(ip -> {
String geoLoc = geolocationCache.getCountry(ip);
return new GeoInfo(ip, geoLoc, date);
return new GeoInfo(geoLoc, date);
}).collect(Collectors.toList());
}
}

55
Plan/common/src/main/java/com/djrapitops/plan/gathering/domain/GeoInfo.java
View File

@ -18,11 +18,8 @@ package com.djrapitops.plan.gathering.domain;
import com.djrapitops.plan.delivery.domain.DateHolder;
import com.djrapitops.plan.delivery.domain.DateMap;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.util.Objects;
/**
@ -36,14 +33,10 @@ public class GeoInfo implements DateHolder, Serializable {
private final String geolocation;
private final long date;
public GeoInfo(InetAddress address, String geolocation, long lastUsed) {
this(formatIP(address), geolocation, lastUsed);
}
public GeoInfo(String ip, String geolocation, long date) {
this.ip = ip;
public GeoInfo(String geolocation, long lastUsed) {
this.ip = "ip";
this.geolocation = geolocation;
this.date = date;
this.date = lastUsed;
}
public static DateMap<GeoInfo> intoDateMap(Iterable<GeoInfo> geoInfo) {
@ -54,42 +47,6 @@ public class GeoInfo implements DateHolder, Serializable {
return map;
}
static String formatIP(InetAddress address) {
String ip = address.getHostAddress();
if ("localhost".equals(ip)) {
return ip;
}
if (address instanceof Inet6Address) {
StringBuilder b = new StringBuilder();
int i = 0;
for (String part : StringUtils.split(ip, ':')) {
if (i >= 3) {
break;
}
b.append(part).append(':');
i++;
}
return b.append("xx..").toString();
} else {
StringBuilder b = new StringBuilder();
int i = 0;
for (String part : StringUtils.split(ip, '.')) {
if (i >= 2) {
break;
}
b.append(part).append('.');
i++;
}
return b.append("xx.xx").toString();
}
}
@Deprecated
public String getIp() {
return ip;
@ -109,8 +66,7 @@ public class GeoInfo implements DateHolder, Serializable {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GeoInfo geoInfo = (GeoInfo) o;
return Objects.equals(ip, geoInfo.ip) &&
Objects.equals(geolocation, geoInfo.geolocation);
return Objects.equals(geolocation, geoInfo.geolocation);
}
@Override
@ -121,8 +77,7 @@ public class GeoInfo implements DateHolder, Serializable {
@Override
public String toString() {
return "GeoInfo{" +
"ip='" + ip + '\'' +
", geolocation='" + geolocation + '\'' +
"geolocation='" + geolocation + '\'' +
", date=" + date +
'}';
}

1
Plan/common/src/main/java/com/djrapitops/plan/storage/database/SQLDB.java
View File

@ -164,7 +164,6 @@ public abstract class SQLDB extends AbstractDatabase {
new UserInfoOptimizationPatch(),
new GeoInfoOptimizationPatch(),
new TransferTableRemovalPatch(),
new IPAnonPatch(),
new BadAFKThresholdValuePatch(),
new DeleteIPHashesPatch(),
new ExtensionShowInPlayersTablePatch(),

5
Plan/common/src/main/java/com/djrapitops/plan/storage/database/queries/objects/GeoInfoQueries.java
View File

@ -69,10 +69,9 @@ public class GeoInfoQueries {
List<GeoInfo> userGeoInfo = geoInformation.getOrDefault(uuid, new ArrayList<>());
String ip = set.getString(GeoInfoTable.IP);
String geolocation = set.getString(GeoInfoTable.GEOLOCATION);
long lastUsed = set.getLong(GeoInfoTable.LAST_USED);
userGeoInfo.add(new GeoInfo(ip, geolocation, lastUsed));
userGeoInfo.add(new GeoInfo(geolocation, lastUsed));
geoInformation.put(uuid, userGeoInfo);
}
@ -105,7 +104,7 @@ public class GeoInfoQueries {
while (set.next()) {
String geolocation = set.getString(GeoInfoTable.GEOLOCATION);
long lastUsed = set.getLong(GeoInfoTable.LAST_USED);
geoInfo.add(new GeoInfo("", geolocation, lastUsed));
geoInfo.add(new GeoInfo(geolocation, lastUsed));
}
return geoInfo;
}

7
Plan/common/src/main/java/com/djrapitops/plan/storage/database/sql/tables/GeoInfoTable.java
View File

@ -19,7 +19,10 @@ package com.djrapitops.plan.storage.database.sql.tables;
import com.djrapitops.plan.storage.database.DBType;
import com.djrapitops.plan.storage.database.sql.parsing.CreateTableParser;
import com.djrapitops.plan.storage.database.sql.parsing.Sql;
import com.djrapitops.plan.storage.database.transactions.patches.*;
import com.djrapitops.plan.storage.database.transactions.patches.DeleteIPHashesPatch;
import com.djrapitops.plan.storage.database.transactions.patches.GeoInfoLastUsedPatch;
import com.djrapitops.plan.storage.database.transactions.patches.GeoInfoOptimizationPatch;
import com.djrapitops.plan.storage.database.transactions.patches.Version10Patch;
import static com.djrapitops.plan.storage.database.sql.parsing.Sql.AND;
import static com.djrapitops.plan.storage.database.sql.parsing.Sql.WHERE;
@ -30,7 +33,6 @@ import static com.djrapitops.plan.storage.database.sql.parsing.Sql.WHERE;
* Patches related to this table:
* {@link Version10Patch}
* {@link GeoInfoLastUsedPatch}
* {@link IPAnonPatch}
* {@link GeoInfoOptimizationPatch}
* {@link DeleteIPHashesPatch}
*
@ -42,6 +44,7 @@ public class GeoInfoTable {
public static final String ID = "id";
public static final String USER_UUID = "uuid";
@Deprecated
public static final String IP = "ip";
public static final String GEOLOCATION = "geolocation";
public static final String LAST_USED = "last_used";

2
Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/events/GeoInfoStoreTransaction.java
View File

@ -57,7 +57,7 @@ public class GeoInfoStoreTransaction extends Transaction {
private GeoInfo createGeoInfo() {
String country = geolocationFunction.apply(ip.getHostAddress());
return new GeoInfo(ip, country, time);
return new GeoInfo(country, time);
}
@Override

1
Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/patches/DeleteIPHashesPatch.java
View File

@ -34,6 +34,7 @@ import static com.djrapitops.plan.storage.database.sql.parsing.Sql.*;
*
* @author Rsl1122
*/
@Deprecated
public class DeleteIPHashesPatch extends Patch {
private static final String IP_HASH = "ip_hash";

135
Plan/common/src/main/java/com/djrapitops/plan/storage/database/transactions/patches/IPAnonPatch.java
View File

@ -1,135 +0,0 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.storage.database.transactions.patches;
import com.djrapitops.plan.gathering.domain.GeoInfo;
import com.djrapitops.plan.storage.database.queries.QueryStatement;
import com.djrapitops.plan.storage.database.queries.objects.GeoInfoQueries;
import com.djrapitops.plan.storage.database.sql.tables.GeoInfoTable;
import com.djrapitops.plan.storage.database.transactions.ExecBatchStatement;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static com.djrapitops.plan.storage.database.sql.parsing.Sql.*;
public class IPAnonPatch extends Patch {
private String tableName;
private String tempTableName;
public IPAnonPatch() {
tableName = GeoInfoTable.TABLE_NAME;
tempTableName = "plan_ips_temp";
}
@Override
public boolean hasBeenApplied() {
return !containsUnAnonymizedIPs() && !hasTable(tempTableName);
}
private Boolean containsUnAnonymizedIPs() {
String sql = SELECT + '*' + FROM + tableName +
WHERE + GeoInfoTable.IP + " NOT LIKE ? LIMIT 1";
return query(new QueryStatement<Boolean>(sql) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, "%x%");
}
@Override
public Boolean processResults(ResultSet set) throws SQLException {
return set.next();
}
});
}
@Override
protected void applyPatch() {
Map<UUID, List<GeoInfo>> allGeoInfo = query(GeoInfoQueries.fetchAllGeoInformation());
anonymizeIPs(allGeoInfo);
groupHashedIPs();
}
private void anonymizeIPs(Map<UUID, List<GeoInfo>> allGeoInfo) {
String sql = "UPDATE " + GeoInfoTable.TABLE_NAME + " SET " +
GeoInfoTable.IP + "=?" +
WHERE + GeoInfoTable.IP + "=?";
execute(new ExecBatchStatement(sql) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
for (List<GeoInfo> geoInfos : allGeoInfo.values()) {
for (GeoInfo geoInfo : geoInfos) {
addToBatch(statement, geoInfo);
}
}
}
private void addToBatch(PreparedStatement statement, GeoInfo geoInfo) throws SQLException {
try {
String oldIP = geoInfo.getIp();
if (oldIP.endsWith(".xx.xx") || oldIP.endsWith("xx..")) {
return;
}
GeoInfo updatedInfo = new GeoInfo(
InetAddress.getByName(oldIP),
geoInfo.getGeolocation(),
geoInfo.getDate()
);
statement.setString(1, updatedInfo.getIp());
statement.setString(2, geoInfo.getIp());
statement.addBatch();
} catch (UnknownHostException ignore) {
// This ip is completely unusable.
}
}
});
}
private void groupHashedIPs() {
if (!hasTable(tempTableName)) {
tempOldTable();
}
execute(GeoInfoTable.createTableSQL(dbType));
String userIdColumn = "user_id";
boolean hasUserIdColumn = hasColumn(tempTableName, userIdColumn);
String identifiers = hasUserIdColumn ? userIdColumn : "id, uuid";
execute("INSERT INTO plan_ips (" +
identifiers + ", ip, geolocation, last_used" +
") SELECT " +
identifiers + ", ip, geolocation, MAX(last_used) FROM plan_ips_temp GROUP BY ip, " +
(hasUserIdColumn ? userIdColumn : "uuid") +
", geolocation, id");
dropTable(tempTableName);
}
private void tempOldTable() {
if (!hasTable(tempTableName)) {
renameTable(tableName, tempTableName);
}
}
}

65
Plan/common/src/test/java/com/djrapitops/plan/gathering/domain/GeoInfoTest.java
View File

@ -1,65 +0,0 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.gathering.domain;
import org.junit.jupiter.api.Test;
import org.junit.platform.runner.JUnitPlatform;
import org.junit.runner.RunWith;
import java.net.InetAddress;
import java.net.UnknownHostException;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Test for functionality of GeoInfo object.
*
* @author Rsl1122
*/
@RunWith(JUnitPlatform.class)
class GeoInfoTest {
@Test
void automaticallyHidesLast16Bits() throws UnknownHostException {
InetAddress test = InetAddress.getByName("1.2.3.4");
String expected = "1.2.xx.xx";
String result = new GeoInfo(test, "Irrelevant", 3).getIp();
assertEquals(expected, result);
}
@Test
void testFormatIP() throws UnknownHostException {
InetAddress ip = InetAddress.getByName("1.2.3.4");
InetAddress ip2 = InetAddress.getByName("1.2.3.26");
InetAddress ip3 = InetAddress.getByName("1.2.3.235");
String expected = "1.2.xx.xx";
assertEquals(expected, GeoInfo.formatIP(ip));
assertEquals(expected, GeoInfo.formatIP(ip2));
assertEquals(expected, GeoInfo.formatIP(ip3));
}
@Test
void testFormatIPv6() throws UnknownHostException {
InetAddress ip = InetAddress.getByName("1234:1234:1234:1234:1234:1234:1234:1234%0");
String expected = "1234:1234:1234:xx..";
assertEquals(expected, GeoInfo.formatIP(ip));
}
}

18
Plan/common/src/test/java/com/djrapitops/plan/storage/database/DatabaseTest.java
View File

@ -228,18 +228,14 @@ public interface DatabaseTest {
default void geoInformationIsStored() {
saveUserOne();
String expectedIP = "1.2.3.4";
String expectedGeoLoc = "TestLocation";
long time = System.currentTimeMillis();
saveGeoInfo(playerUUID, new GeoInfo(expectedIP, expectedGeoLoc, time));
GeoInfo expected = new GeoInfo("TestLocation", time);
saveGeoInfo(playerUUID, expected);
commitTest();
List<GeoInfo> geolocations = db().query(GeoInfoQueries.fetchAllGeoInformation()).getOrDefault(playerUUID, new ArrayList<>());
assertEquals(1, geolocations.size());
GeoInfo expected = new GeoInfo("1.2.xx.xx", expectedGeoLoc, time);
assertEquals(expected, geolocations.get(0));
List<GeoInfo> result = db().query(GeoInfoQueries.fetchAllGeoInformation()).getOrDefault(playerUUID, new ArrayList<>());
assertEquals(Collections.singletonList(expected), result);
}
@Test
@ -474,7 +470,7 @@ public interface DatabaseTest {
execute(DataStoreQueries.storeSession(session));
db().executeTransaction(new NicknameStoreTransaction(playerUUID, new Nickname("TestNick", System.currentTimeMillis(), serverUUID()), (uuid, name) -> false /* Not cached */));
saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
saveGeoInfo(playerUUID, new GeoInfo("TestLoc", 223456789L));
assertTrue(db().query(PlayerFetchQueries.isPlayerRegistered(playerUUID)));
@ -525,7 +521,7 @@ public interface DatabaseTest {
db().executeTransaction(
new NicknameStoreTransaction(playerUUID, new Nickname("TestNick", System.currentTimeMillis(), serverUUID()), (uuid, name) -> false /* Not cached */)
);
saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
saveGeoInfo(playerUUID, new GeoInfo("TestLoc", 223456789L));
assertTrue(db().query(PlayerFetchQueries.isPlayerRegistered(playerUUID)));
@ -883,7 +879,7 @@ public interface DatabaseTest {
OptionalAssert.equals(1, container.getValue(PlayerKeys.KICK_COUNT));
List<GeoInfo> expectedGeoInfo =
Collections.singletonList(new GeoInfo("", "TestLoc", 223456789));
Collections.singletonList(new GeoInfo("TestLoc", 223456789));
OptionalAssert.equals(expectedGeoInfo, container.getValue(PlayerKeys.GEO_INFO));
List<Nickname> expectedNicknames = Collections.singletonList(new Nickname("TestNick", -1, serverUUID()));

14
Plan/common/src/test/java/com/djrapitops/plan/storage/database/MySQLTest.java
View File

@ -100,13 +100,13 @@ class MySQLTest implements DatabaseTest {
database.executeTransaction(new PlayerRegisterTransaction(secondUuid, () -> 0L, ""));
database.executeTransaction(new PlayerRegisterTransaction(thirdUuid, () -> 0L, ""));
saveGeoInfo(firstUuid, new GeoInfo("-", "Norway", 0));
saveGeoInfo(firstUuid, new GeoInfo("-", "Finland", 5));
saveGeoInfo(secondUuid, new GeoInfo("-", "Sweden", 0));
saveGeoInfo(thirdUuid, new GeoInfo("-", "Denmark", 0));
saveGeoInfo(fourthUuid, new GeoInfo("-", "Denmark", 0));
saveGeoInfo(fifthUuid, new GeoInfo("-", "Not Known", 0));
saveGeoInfo(sixthUuid, new GeoInfo("-", "Local Machine", 0));
saveGeoInfo(firstUuid, new GeoInfo("Norway", 0));
saveGeoInfo(firstUuid, new GeoInfo("Finland", 5));
saveGeoInfo(secondUuid, new GeoInfo("Sweden", 0));
saveGeoInfo(thirdUuid, new GeoInfo("Denmark", 0));
saveGeoInfo(fourthUuid, new GeoInfo("Denmark", 0));
saveGeoInfo(fifthUuid, new GeoInfo("Not Known", 0));
saveGeoInfo(sixthUuid, new GeoInfo("Local Machine", 0));
Map<String, Integer> got = database.query(GeoInfoQueries.networkGeolocationCounts());

14
Plan/common/src/test/java/com/djrapitops/plan/storage/database/SQLiteTest.java
View File

@ -128,13 +128,13 @@ public class SQLiteTest implements DatabaseTest {
database.executeTransaction(new PlayerRegisterTransaction(secondUuid, () -> 0L, ""));
database.executeTransaction(new PlayerRegisterTransaction(thirdUuid, () -> 0L, ""));
saveGeoInfo(firstUuid, new GeoInfo("-", "Norway", 0));
saveGeoInfo(firstUuid, new GeoInfo("-", "Finland", 5));
saveGeoInfo(secondUuid, new GeoInfo("-", "Sweden", 0));
saveGeoInfo(thirdUuid, new GeoInfo("-", "Denmark", 0));
saveGeoInfo(fourthUuid, new GeoInfo("-", "Denmark", 0));
saveGeoInfo(fifthUuid, new GeoInfo("-", "Not Known", 0));
saveGeoInfo(sixthUuid, new GeoInfo("-", "Local Machine", 0));
saveGeoInfo(firstUuid, new GeoInfo("Norway", 0));
saveGeoInfo(firstUuid, new GeoInfo("Finland", 5));
saveGeoInfo(secondUuid, new GeoInfo("Sweden", 0));
saveGeoInfo(thirdUuid, new GeoInfo("Denmark", 0));
saveGeoInfo(fourthUuid, new GeoInfo("Denmark", 0));
saveGeoInfo(fifthUuid, new GeoInfo("Not Known", 0));
saveGeoInfo(sixthUuid, new GeoInfo("Local Machine", 0));
Map<String, Integer> got = database.query(GeoInfoQueries.networkGeolocationCounts());

2
Plan/common/src/test/java/utilities/RandomData.java
View File

@ -106,7 +106,7 @@ public class RandomData {
public static List<GeoInfo> randomGeoInfo() {
List<GeoInfo> test = new ArrayList<>();
for (int i = 0; i < 20; i++) {
GeoInfo geoInfo = new GeoInfo(randomString(10), randomString(10), r.nextLong());
GeoInfo geoInfo = new GeoInfo(randomString(10), r.nextLong());
test.add(geoInfo);
}
return test;

16
Plan/common/src/test/java/utilities/TestData.java
View File

@ -22,13 +22,9 @@ import com.djrapitops.plan.storage.database.transactions.StoreServerInformationT
import com.djrapitops.plan.storage.database.transactions.Transaction;
import com.djrapitops.plan.storage.database.transactions.events.*;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Class for saving test data to a database.
@ -66,14 +62,10 @@ public class TestData {
private static List<GeoInfo> createGeoInfoForPlayer() {
List<GeoInfo> geoInfos = new ArrayList<>();
try {
geoInfos.add(new GeoInfo(InetAddress.getByName("1.2.3.4"), "Not Known", playerFirstJoin));
geoInfos.add(new GeoInfo(InetAddress.getByName("43b9:416b:3cb2:649d:ebaf:872:d89a:343d"), "Not Known", playerFirstJoin));
geoInfos.add(new GeoInfo(InetAddress.getByName("127.0.0.1"), "Local Machine", playerFirstJoin));
geoInfos.add(new GeoInfo(InetAddress.getByName("181.103.227.78"), "Argentina", playerFirstJoin));
} catch (UnknownHostException e) {
Logger.getGlobal().log(Level.WARNING, e, () -> "Failed to create GeoInfo");
}
geoInfos.add(new GeoInfo("Not Known", playerFirstJoin));
geoInfos.add(new GeoInfo("Not Known", playerFirstJoin));
geoInfos.add(new GeoInfo("Local Machine", playerFirstJoin));
geoInfos.add(new GeoInfo("Argentina", playerFirstJoin));
return geoInfos;
}