GeoInfo table structure optimization

- Replaced user_id with uuid
- Replaced server_id with server_uuid
This commit is contained in:
Rsl1122 2018-12-13 20:52:03 +02:00
parent 1830e71157
commit e9a2a3b1e2
5 changed files with 112 additions and 72 deletions

View File

@ -208,7 +208,8 @@ public abstract class SQLDB extends Database {
new SessionsOptimizationPatch(this), new SessionsOptimizationPatch(this),
new PingOptimizationPatch(this), new PingOptimizationPatch(this),
new NicknamesOptimizationPatch(this), new NicknamesOptimizationPatch(this),
new UserInfoOptimizationPatch(this) new UserInfoOptimizationPatch(this),
new GeoInfoOptimizationPatch(this)
}; };
try { try {

View File

@ -0,0 +1,75 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.system.database.databases.sql.patches;
import com.djrapitops.plan.api.exceptions.database.DBOpException;
import com.djrapitops.plan.system.database.databases.sql.SQLDB;
import com.djrapitops.plan.system.database.databases.sql.tables.GeoInfoTable;
import com.djrapitops.plan.system.database.databases.sql.tables.GeoInfoTable.Col;
public class GeoInfoOptimizationPatch extends Patch {
private String tempTableName;
private String tableName;
public GeoInfoOptimizationPatch(SQLDB db) {
super(db);
tableName = GeoInfoTable.TABLE_NAME;
tempTableName = "temp_ips";
}
@Override
public boolean hasBeenApplied() {
return hasColumn(tableName, Col.ID.get())
&& hasColumn(tableName, Col.UUID.get())
&& !hasColumn(tableName, "user_id")
&& !hasTable(tempTableName); // If this table exists the patch has failed to finish.
}
@Override
public void apply() {
try {
tempOldTable();
db.getGeoInfoTable().createTable();
db.execute("INSERT INTO " + tableName + " (" +
Col.UUID + ", " +
Col.IP + ", " +
Col.IP_HASH + ", " +
Col.LAST_USED + ", " +
Col.GEOLOCATION +
") SELECT " +
"(SELECT plan_users.uuid FROM plan_users WHERE plan_users.id = " + tempTableName + ".user_id LIMIT 1), " +
Col.IP + ", " +
Col.IP_HASH + ", " +
Col.LAST_USED + ", " +
Col.GEOLOCATION +
" FROM " + tempTableName
);
dropTable(tempTableName);
} catch (Exception e) {
throw new DBOpException(GeoInfoOptimizationPatch.class.getSimpleName() + " failed.", e);
}
}
private void tempOldTable() {
if (!hasTable(tempTableName)) {
renameTable(tableName, tempTableName);
}
}
}

View File

@ -23,7 +23,6 @@ import com.djrapitops.plan.system.database.databases.sql.SQLDB;
import com.djrapitops.plan.system.database.databases.sql.processing.ExecStatement; import com.djrapitops.plan.system.database.databases.sql.processing.ExecStatement;
import com.djrapitops.plan.system.database.databases.sql.processing.QueryStatement; import com.djrapitops.plan.system.database.databases.sql.processing.QueryStatement;
import com.djrapitops.plan.system.database.databases.sql.tables.GeoInfoTable; import com.djrapitops.plan.system.database.databases.sql.tables.GeoInfoTable;
import com.djrapitops.plan.system.database.databases.sql.tables.move.Version18TransferTable;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
@ -106,7 +105,21 @@ public class IPAnonPatch extends Patch {
private void groupHashedIPs() { private void groupHashedIPs() {
try { try {
new Version18TransferTable(db).alterTableV18(); String tempTableName = "plan_ips_temp";
String ipTableName = "plan_ips";
try {
renameTable(ipTableName, tempTableName);
} catch (DBOpException e) {
// Temp table already exists
if (!e.getMessage().contains("plan_ips_temp")) {
throw e;
}
}
db.getGeoInfoTable().createTable();
db.execute("INSERT INTO plan_ips (" +
"id, uuid, ip, ip_hash, geolocation, last_used" +
") SELECT id, uuid, ip, ip_hash, geolocation, MAX(last_used) FROM plan_ips_temp GROUP BY ip_hash, uuid, ip, geolocation");
dropTable(tempTableName);
} catch (DBInitException e) { } catch (DBInitException e) {
throw new DBOpException(e.getMessage(), e); throw new DBOpException(e.getMessage(), e);
} }

View File

@ -39,25 +39,28 @@ import java.util.*;
* <p> * <p>
* Table Name: plan_ips * Table Name: plan_ips
* <p> * <p>
* For contained columns {@see Col} * Patches related to this table:
* {@link com.djrapitops.plan.system.database.databases.sql.patches.Version10Patch}
* {@link com.djrapitops.plan.system.database.databases.sql.patches.GeoInfoLastUsedPatch}
* {@link com.djrapitops.plan.system.database.databases.sql.patches.IPAnonPatch}
* {@link com.djrapitops.plan.system.database.databases.sql.patches.IPHashPatch}
* {@link com.djrapitops.plan.system.database.databases.sql.patches.GeoInfoOptimizationPatch}
* *
* @author Rsl1122 * @author Rsl1122
*/ */
public class GeoInfoTable extends UserIDTable { public class GeoInfoTable extends UserUUIDTable {
public static final String TABLE_NAME = "plan_ips"; public static final String TABLE_NAME = "plan_ips";
public GeoInfoTable(SQLDB db) { public GeoInfoTable(SQLDB db) {
super(TABLE_NAME, db); super(TABLE_NAME, db);
insertStatement = "INSERT INTO " + tableName + " (" insertStatement = "INSERT INTO " + tableName + " ("
+ Col.USER_ID + ", " + Col.UUID + ", "
+ Col.IP + ", " + Col.IP + ", "
+ Col.IP_HASH + ", " + Col.IP_HASH + ", "
+ Col.GEOLOCATION + ", " + Col.GEOLOCATION + ", "
+ Col.LAST_USED + Col.LAST_USED
+ ") VALUES (" + ") VALUES (?, ?, ?, ?, ?)";
+ usersTable.statementSelectID + ", "
+ "?, ?, ?, ?)";
} }
private String insertStatement; private String insertStatement;
@ -65,19 +68,20 @@ public class GeoInfoTable extends UserIDTable {
@Override @Override
public void createTable() throws DBInitException { public void createTable() throws DBInitException {
createTable(TableSqlParser.createTable(tableName) createTable(TableSqlParser.createTable(tableName)
.column(Col.USER_ID, Sql.INT).notNull() .primaryKeyIDColumn(supportsMySQLQueries, Col.ID)
.column(Col.UUID, Sql.varchar(36)).notNull()
.column(Col.IP, Sql.varchar(39)).notNull() .column(Col.IP, Sql.varchar(39)).notNull()
.column(Col.GEOLOCATION, Sql.varchar(50)).notNull() .column(Col.GEOLOCATION, Sql.varchar(50)).notNull()
.column(Col.IP_HASH, Sql.varchar(200)) .column(Col.IP_HASH, Sql.varchar(200))
.column(Col.LAST_USED, Sql.LONG).notNull().defaultValue("0") .column(Col.LAST_USED, Sql.LONG).notNull().defaultValue("0")
.foreignKey(Col.USER_ID, usersTable.getTableName(), UsersTable.Col.ID) .primaryKey(supportsMySQLQueries, Col.ID)
.toString() .toString()
); );
} }
public List<GeoInfo> getGeoInfo(UUID uuid) { public List<GeoInfo> getGeoInfo(UUID uuid) {
String sql = "SELECT DISTINCT * FROM " + tableName + String sql = "SELECT DISTINCT * FROM " + tableName +
" WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID; " WHERE " + Col.UUID + "=?";
return query(new QueryStatement<List<GeoInfo>>(sql, 100) { return query(new QueryStatement<List<GeoInfo>>(sql, 100) {
@Override @Override
@ -103,7 +107,7 @@ public class GeoInfoTable extends UserIDTable {
private void updateGeoInfo(UUID uuid, GeoInfo info) { private void updateGeoInfo(UUID uuid, GeoInfo info) {
String sql = "UPDATE " + tableName + " SET " String sql = "UPDATE " + tableName + " SET "
+ Col.LAST_USED + "=?" + + Col.LAST_USED + "=?" +
" WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID + " WHERE " + Col.UUID + "=?" +
" AND " + Col.IP_HASH + "=?" + " AND " + Col.IP_HASH + "=?" +
" AND " + Col.GEOLOCATION + "=?"; " AND " + Col.GEOLOCATION + "=?";
@ -162,23 +166,20 @@ public class GeoInfoTable extends UserIDTable {
} }
public Map<UUID, List<GeoInfo>> getAllGeoInfo() { public Map<UUID, List<GeoInfo>> getAllGeoInfo() {
String usersIDColumn = usersTable + "." + UsersTable.Col.ID;
String usersUUIDColumn = usersTable + "." + UsersTable.Col.UUID + " as uuid";
String sql = "SELECT " + String sql = "SELECT " +
Col.IP + ", " + Col.IP + ", " +
Col.GEOLOCATION + ", " + Col.GEOLOCATION + ", " +
Col.LAST_USED + ", " + Col.LAST_USED + ", " +
Col.IP_HASH + ", " + Col.IP_HASH + ", " +
usersUUIDColumn + Col.UUID +
" FROM " + tableName + " FROM " + tableName;
" INNER JOIN " + usersTable + " on " + usersIDColumn + "=" + Col.USER_ID;
return query(new QueryAllStatement<Map<UUID, List<GeoInfo>>>(sql, 50000) { return query(new QueryAllStatement<Map<UUID, List<GeoInfo>>>(sql, 50000) {
@Override @Override
public Map<UUID, List<GeoInfo>> processResults(ResultSet set) throws SQLException { public Map<UUID, List<GeoInfo>> processResults(ResultSet set) throws SQLException {
Map<UUID, List<GeoInfo>> geoLocations = new HashMap<>(); Map<UUID, List<GeoInfo>> geoLocations = new HashMap<>();
while (set.next()) { while (set.next()) {
UUID uuid = UUID.fromString(set.getString("uuid")); UUID uuid = UUID.fromString(set.getString(Col.UUID.get()));
List<GeoInfo> userGeoInfo = geoLocations.getOrDefault(uuid, new ArrayList<>()); List<GeoInfo> userGeoInfo = geoLocations.getOrDefault(uuid, new ArrayList<>());
@ -241,7 +242,10 @@ public class GeoInfoTable extends UserIDTable {
} }
public enum Col implements Column { public enum Col implements Column {
ID("id"),
@Deprecated
USER_ID(UserIDTable.Col.USER_ID.get()), USER_ID(UserIDTable.Col.USER_ID.get()),
UUID(UserUUIDTable.Col.UUID.get()),
IP("ip"), IP("ip"),
IP_HASH("ip_hash"), IP_HASH("ip_hash"),
GEOLOCATION("geolocation"), GEOLOCATION("geolocation"),

View File

@ -1,53 +0,0 @@
/*
* This file is part of Player Analytics (Plan).
*
* Plan is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License v3 as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Plan is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Plan. If not, see <https://www.gnu.org/licenses/>.
*/
package com.djrapitops.plan.system.database.databases.sql.tables.move;
import com.djrapitops.plan.api.exceptions.database.DBInitException;
import com.djrapitops.plan.api.exceptions.database.DBOpException;
import com.djrapitops.plan.system.database.databases.sql.SQLDB;
/**
* DB Schema v17 -> 18 table.
* <p>
* Required due to a bug where duplicate rows were inserted.
*
* @author Rsl1122
*/
public class Version18TransferTable extends TransferTable {
public Version18TransferTable(SQLDB db) {
super(db);
}
public void alterTableV18() throws DBInitException {
String tempTableName = "plan_ips_temp";
String ipTableName = "plan_ips";
try {
renameTable(ipTableName, tempTableName);
} catch (DBOpException e) {
// Temp table already exists
if (!e.getMessage().contains("plan_ips_temp")) {
throw e;
}
}
db.getGeoInfoTable().createTable();
execute("INSERT INTO plan_ips (" +
"user_id, ip, ip_hash, geolocation, last_used" +
") SELECT user_id, ip, ip_hash, geolocation, MAX(last_used) FROM plan_ips_temp GROUP BY ip_hash, user_id, ip, geolocation");
dropTable(tempTableName);
}
}