2020-05-06 11:48:49 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2020-04-08 09:09:05 +02:00
From: Aikar <aikar@aikar.co>
Date: Wed, 8 Apr 2020 03:06:30 -0400
Subject: [PATCH] Optimize PlayerChunkMap memory use for visibleChunks
No longer clones visible chunks which is causing massive memory
allocation issues, likely the source of Humongous Objects on large servers.
Instead we just synchronize, clear and rebuild, reusing the same object buffers
as before with only 2 small objects created (FastIterator/MapEntry)
This should result in siginificant memory use reduction and improved GC behavior.
2020-04-10 20:11:46 +02:00
diff --git a/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
new file mode 100644
2020-05-06 11:48:49 +02:00
index 0000000000000000000000000000000000000000..e0ad725b2e63ffd329fc4725d15290cbf494b790
2020-04-10 20:11:46 +02:00
--- /dev/null
+++ b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
@@ -0,0 +1,32 @@
+package com.destroystokyo.paper.util.map;
+
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
+
+public class Long2ObjectLinkedOpenHashMapFastCopy<V> extends Long2ObjectLinkedOpenHashMap<V> {
+
+ public void copyFrom(Long2ObjectLinkedOpenHashMapFastCopy<V> map) {
+ if (key.length < map.key.length) {
+ key = null;
+ key = new long[map.key.length];
+ }
+ if (value.length < map.value.length) {
+ value = null;
+ //noinspection unchecked
+ value = (V[]) new Object[map.value.length];
+ }
+ if (link.length < map.link.length) {
+ link = null;
+ link = new long[map.link.length];
+ }
+ System.arraycopy(map.key, 0, this.key, 0, map.key.length);
+ System.arraycopy(map.value, 0, this.value, 0, map.value.length);
+ System.arraycopy(map.link, 0, this.link, 0, map.link.length);
+ this.size = map.size;
+ this.mask = map.mask;
+ this.first = map.first;
+ this.last = map.last;
+ this.n = map.n;
+ this.maxFill = map.maxFill;
+ this.containsNullKey = map.containsNullKey;
+ }
+}
2020-04-08 09:09:05 +02:00
diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
2020-05-14 10:58:47 +02:00
index fd998e4fb1534690a2ef8c1bca55e0ae9fe855f9..8f849d83d08b39f1cd9184f484a2089a7a3124ef 100644
2020-04-08 09:09:05 +02:00
--- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
+++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
2020-05-14 10:58:47 +02:00
@@ -755,7 +755,7 @@ public class ChunkProviderServer extends IChunkProvider {
2020-04-08 09:09:05 +02:00
entityPlayer.playerNaturallySpawnedEvent.callEvent();
};
// Paper end
- this.playerChunkMap.f().forEach((playerchunk) -> {
2020-04-09 03:07:59 +02:00
+ this.playerChunkMap.forEachVisibleChunk((playerchunk) -> { // Paper - safe iterator incase chunk loads, also no wrapping
2020-04-08 09:09:05 +02:00
Optional<Chunk> optional = ((Either) playerchunk.b().getNow(PlayerChunk.UNLOADED_CHUNK)).left();
if (optional.isPresent()) {
diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
2020-05-06 11:48:49 +02:00
index d9941b38ca037a31f520784b3706080f1d322fb4..71ab65e00fe31ea4047cf8a5921c6deba13de6b9 100644
2020-04-08 09:09:05 +02:00
--- a/src/main/java/net/minecraft/server/MCUtil.java
+++ b/src/main/java/net/minecraft/server/MCUtil.java
2020-04-19 12:01:07 +02:00
@@ -529,7 +529,7 @@ public final class MCUtil {
2020-04-08 09:09:05 +02:00
WorldServer world = ((org.bukkit.craftbukkit.CraftWorld)bukkitWorld).getHandle();
PlayerChunkMap chunkMap = world.getChunkProvider().playerChunkMap;
- Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = chunkMap.visibleChunks;
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = chunkMap.getVisibleChunks();
ChunkMapDistance chunkMapDistance = chunkMap.getChunkMapDistanceManager();
List<PlayerChunk> allChunks = new ArrayList<>(visibleChunks.values());
List<EntityPlayer> players = world.players;
diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
2020-05-06 11:48:49 +02:00
index 4beae504c875767ff00e26461fe7240498750e27..00f26ae23da65453073fc06ffec8a349ef28dd7e 100644
2020-04-08 09:09:05 +02:00
--- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
+++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
2020-04-10 20:11:46 +02:00
@@ -55,8 +55,10 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
private static final Logger LOGGER = LogManager.getLogger();
public static final int GOLDEN_TICKET = 33 + ChunkStatus.b();
2020-04-10 20:11:46 +02:00
- public final Long2ObjectLinkedOpenHashMap<PlayerChunk> updatingChunks = new Long2ObjectLinkedOpenHashMap();
2020-04-08 09:09:05 +02:00
- public volatile Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks;
2020-04-10 20:11:46 +02:00
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> updatingChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> visibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> pendingVisibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - this is used if the visible chunks is updated while iterating only
+ public transient Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunksClone; // Paper - used for async access of visible chunks, clone and cache only when needed
2020-04-08 09:09:05 +02:00
private final Long2ObjectLinkedOpenHashMap<PlayerChunk> pendingUnload;
final LongSet loadedChunks; // Paper - private -> package
public final WorldServer world;
2020-05-06 09:44:47 +02:00
@@ -130,7 +132,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
public PlayerChunkMap(WorldServer worldserver, File file, DataFixer datafixer, DefinedStructureManager definedstructuremanager, Executor executor, IAsyncTaskHandler<Runnable> iasynctaskhandler, ILightAccess ilightaccess, ChunkGenerator<?> chunkgenerator, WorldLoadListener worldloadlistener, Supplier<WorldPersistentData> supplier, int i) {
super(new File(worldserver.getWorldProvider().getDimensionManager().a(file), "region"), datafixer);
- this.visibleChunks = this.updatingChunks.clone();
+ //this.visibleChunks = this.updatingChunks.clone(); // Paper - no more cloning
this.pendingUnload = new Long2ObjectLinkedOpenHashMap();
this.loadedChunks = new LongOpenHashSet();
this.unloadQueue = new LongOpenHashSet();
2020-05-06 09:44:47 +02:00
@@ -221,9 +223,52 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
return (PlayerChunk) this.updatingChunks.get(i);
}
+ // Paper start - remove cloning of visible chunks unless accessed as a collection async
+ private static final boolean DEBUG_ASYNC_VISIBLE_CHUNKS = Boolean.getBoolean("paper.debug-async-visible-chunks");
2020-04-09 03:07:59 +02:00
+ private boolean isIterating = false;
2020-04-10 20:11:46 +02:00
+ private boolean hasPendingVisibleUpdate = false;
2020-04-09 03:07:59 +02:00
+ public void forEachVisibleChunk(java.util.function.Consumer<PlayerChunk> consumer) {
+ org.spigotmc.AsyncCatcher.catchOp("forEachVisibleChunk");
+ boolean prev = isIterating;
+ isIterating = true;
+ try {
+ for (PlayerChunk value : this.visibleChunks.values()) {
+ consumer.accept(value);
+ }
+ } finally {
+ this.isIterating = prev;
2020-04-10 20:11:46 +02:00
+ if (!this.isIterating && this.hasPendingVisibleUpdate) {
+ this.visibleChunks.copyFrom(this.pendingVisibleChunks);
+ this.pendingVisibleChunks.clear();
+ this.hasPendingVisibleUpdate = false;
2020-04-09 03:07:59 +02:00
+ }
+ }
+ }
2020-04-08 09:09:05 +02:00
+ public Long2ObjectLinkedOpenHashMap<PlayerChunk> getVisibleChunks() {
+ if (Thread.currentThread() == this.world.serverThread) {
+ return this.visibleChunks;
+ } else {
+ synchronized (this.visibleChunks) {
+ if (DEBUG_ASYNC_VISIBLE_CHUNKS) new Throwable("Async getVisibleChunks").printStackTrace();
+ if (this.visibleChunksClone == null) {
2020-04-10 20:11:46 +02:00
+ this.visibleChunksClone = this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.clone() : this.visibleChunks.clone();
2020-04-08 09:09:05 +02:00
+ }
+ return this.visibleChunksClone;
+ }
+ }
+ }
+ // Paper end
+
@Nullable
public PlayerChunk getVisibleChunk(long i) { // Paper - protected -> public
2020-04-10 20:11:46 +02:00
- return (PlayerChunk) this.visibleChunks.get(i);
2020-04-08 09:09:05 +02:00
+ // Paper start - mt safe get
+ if (Thread.currentThread() != this.world.serverThread) {
+ synchronized (this.visibleChunks) {
2020-04-10 20:11:46 +02:00
+ return (PlayerChunk) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(i) : this.visibleChunks.get(i));
2020-04-08 09:09:05 +02:00
+ }
+ }
2020-04-10 20:11:46 +02:00
+ return (PlayerChunk) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(i) : this.visibleChunks.get(i));
2020-04-08 09:09:05 +02:00
+ // Paper end
}
2020-04-10 20:11:46 +02:00
protected IntSupplier c(long i) {
2020-05-06 09:44:47 +02:00
@@ -411,8 +456,9 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
// Paper end
protected void save(boolean flag) {
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = this.getVisibleChunks(); // Paper remove clone of visible Chunks unless saving off main thread (watchdog kill)
if (flag) {
- List<PlayerChunk> list = (List) this.visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).peek(PlayerChunk::m).collect(Collectors.toList());
+ List<PlayerChunk> list = (List) visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).peek(PlayerChunk::m).collect(Collectors.toList()); // Paper - remove cloning of visible chunks
MutableBoolean mutableboolean = new MutableBoolean();
do {
2020-05-06 09:44:47 +02:00
@@ -440,7 +486,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
// this.i(); // Paper - nuke IOWorker
PlayerChunkMap.LOGGER.info("ThreadedAnvilChunkStorage ({}): All chunks are saved", this.w.getName());
} else {
- this.visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).forEach((playerchunk) -> {
+ visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).forEach((playerchunk) -> {
IChunkAccess ichunkaccess = (IChunkAccess) playerchunk.getChunkSave().getNow(null); // CraftBukkit - decompile error
if (ichunkaccess instanceof ProtoChunkExtension || ichunkaccess instanceof Chunk) {
2020-05-06 09:44:47 +02:00
@@ -610,7 +656,20 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-10 20:11:46 +02:00
if (!this.updatingChunksModified) {
2020-04-08 09:09:05 +02:00
return false;
} else {
- this.visibleChunks = this.updatingChunks.clone();
+ // Paper start - stop cloning visibleChunks
+ synchronized (this.visibleChunks) {
2020-04-10 20:11:46 +02:00
+ if (isIterating) {
+ hasPendingVisibleUpdate = true;
+ this.pendingVisibleChunks.copyFrom(this.updatingChunks);
+ } else {
+ hasPendingVisibleUpdate = false;
+ this.pendingVisibleChunks.clear();
+ this.visibleChunks.copyFrom(this.updatingChunks);
+ this.visibleChunksClone = null;
+ }
2020-04-08 09:09:05 +02:00
+ }
+ // Paper end
+
this.updatingChunksModified = false;
return true;
}
2020-05-06 09:44:47 +02:00
@@ -1076,12 +1135,12 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
2020-04-08 09:09:05 +02:00
}
protected Iterable<PlayerChunk> f() {
- return Iterables.unmodifiableIterable(this.visibleChunks.values());
+ return Iterables.unmodifiableIterable(this.getVisibleChunks().values()); // Paper
}
void a(Writer writer) throws IOException {
CSVWriter csvwriter = CSVWriter.a().a("x").a("z").a("level").a("in_memory").a("status").a("full_status").a("accessible_ready").a("ticking_ready").a("entity_ticking_ready").a("ticket").a("spawning").a("entity_count").a("block_entity_count").a(writer);
- ObjectBidirectionalIterator objectbidirectionaliterator = this.visibleChunks.long2ObjectEntrySet().iterator();
+ ObjectBidirectionalIterator objectbidirectionaliterator = this.getVisibleChunks().long2ObjectEntrySet().iterator(); // Paper
while (objectbidirectionaliterator.hasNext()) {
Entry<PlayerChunk> entry = (Entry) objectbidirectionaliterator.next();
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
2020-05-10 04:30:28 +02:00
index 5e84bbad1ba75cb2b541a29d38d3f96832cff853..2393c8715603478142a1980a9a769de58f8bb3c4 100644
2020-04-08 09:09:05 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
2020-04-09 03:07:59 +02:00
@@ -74,6 +74,7 @@ import net.minecraft.server.GameRules;
import net.minecraft.server.GroupDataEntity;
import net.minecraft.server.IBlockData;
import net.minecraft.server.IChunkAccess;
+import net.minecraft.server.MCUtil;
import net.minecraft.server.MinecraftKey;
2020-05-10 04:30:28 +02:00
import net.minecraft.server.MinecraftServer;
2020-04-09 03:07:59 +02:00
import net.minecraft.server.MovingObjectPosition;
2020-05-10 04:30:28 +02:00
@@ -291,6 +292,7 @@ public class CraftWorld implements World {
2020-04-08 09:09:05 +02:00
return ret;
}
public int getTileEntityCount() {
2020-04-09 03:07:59 +02:00
+ return MCUtil.ensureMain(() -> {
2020-04-08 09:09:05 +02:00
// We don't use the full world tile entity list, so we must iterate chunks
Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
int size = 0;
2020-05-10 04:30:28 +02:00
@@ -302,11 +304,13 @@ public class CraftWorld implements World {
2020-04-09 03:07:59 +02:00
size += chunk.tileEntities.size();
}
return size;
+ });
}
public int getTickableTileEntityCount() {
2020-04-08 09:09:05 +02:00
return world.tileEntityListTick.size();
}
public int getChunkCount() {
2020-04-09 03:07:59 +02:00
+ return MCUtil.ensureMain(() -> {
2020-04-08 09:09:05 +02:00
int ret = 0;
for (PlayerChunk chunkHolder : world.getChunkProvider().playerChunkMap.visibleChunks.values()) {
2020-05-10 04:30:28 +02:00
@@ -315,7 +319,7 @@ public class CraftWorld implements World {
2020-04-09 03:07:59 +02:00
}
}
- return ret;
+ return ret; });
}
public int getPlayerCount() {
return world.players.size();
2020-05-10 04:30:28 +02:00
@@ -435,6 +439,14 @@ public class CraftWorld implements World {
2020-04-08 09:09:05 +02:00
@Override
public Chunk[] getLoadedChunks() {
2020-04-09 03:07:59 +02:00
+ // Paper start
+ if (Thread.currentThread() != world.getMinecraftWorld().serverThread) {
+ synchronized (world.getChunkProvider().playerChunkMap.visibleChunks) {
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
+ return chunks.values().stream().map(PlayerChunk::getFullChunk).filter(Objects::nonNull).map(net.minecraft.server.Chunk::getBukkitChunk).toArray(Chunk[]::new);
+ }
+ }
+ // Paper end
2020-04-08 09:09:05 +02:00
Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
return chunks.values().stream().map(PlayerChunk::getFullChunk).filter(Objects::nonNull).map(net.minecraft.server.Chunk::getBukkitChunk).toArray(Chunk[]::new);
}