Paper/patches/server/0434-Stop-copy-on-write-operations-for-updating-light-dat.patch

298 lines
17 KiB
Diff
Raw Normal View History

2021-06-11 14:02:28 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
Rewrite chunk system (#8177) Patch documentation to come Issues with the old system that are fixed now: - World generation does not scale with cpu cores effectively. - Relies on the main thread for scheduling and maintaining chunk state, dropping chunk load/generate rates at lower tps. - Unreliable prioritisation of chunk gen/load calls that block the main thread. - Shutdown logic is utterly unreliable, as it has to wait for all chunks to unload - is it guaranteed that the chunk system is in a state on shutdown that it can reliably do this? Watchdog shutdown also typically failed due to thread checks, which is now resolved. - Saving of data is not unified (i.e can save chunk data without saving entity data, poses problems for desync if shutdown is really abnormal. - Entities are not loaded with chunks. This caused quite a bit of headache for Chunk#getEntities API, but now the new chunk system loads entities with chunks so that they are ready whenever the chunk loads in. Effectively brings the behavior back to 1.16 era, but still storing entities in their own separate regionfiles. The above list is not complete. The patch documentation will complete it. New chunk system hard relies on starlight and dataconverter, and most importantly the new concurrent utilities in ConcurrentUtil. Some of the old async chunk i/o interface (i.e the old file io thread reroutes _some_ calls to the new file io thread) is kept for plugin compat reasons. It will be removed in the next major version of minecraft. The old legacy chunk system patches have been moved to the removed folder in case we need them again.
2022-09-26 10:02:51 +02:00
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
2021-06-11 14:02:28 +02:00
Date: Mon, 27 Apr 2020 04:05:38 -0700
Subject: [PATCH] Stop copy-on-write operations for updating light data
Causes huge memory allocations + gc issues
diff --git a/src/main/java/net/minecraft/world/level/lighting/BlockLightSectionStorage.java b/src/main/java/net/minecraft/world/level/lighting/BlockLightSectionStorage.java
index 573cdb0897978eef8f5fc906ed4928293f4b2ab9..314b46f0becd088d26956b45981217b128d539cb 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/lighting/BlockLightSectionStorage.java
+++ b/src/main/java/net/minecraft/world/level/lighting/BlockLightSectionStorage.java
@@ -9,7 +9,7 @@ import net.minecraft.world.level.chunk.LightChunkGetter;
2021-06-11 14:02:28 +02:00
public class BlockLightSectionStorage extends LayerLightSectionStorage<BlockLightSectionStorage.BlockDataLayerStorageMap> {
2021-06-11 14:02:28 +02:00
protected BlockLightSectionStorage(LightChunkGetter chunkProvider) {
- super(LightLayer.BLOCK, chunkProvider, new BlockLightSectionStorage.BlockDataLayerStorageMap(new Long2ObjectOpenHashMap<>()));
2021-06-11 14:02:28 +02:00
+ super(LightLayer.BLOCK, chunkProvider, new BlockLightSectionStorage.BlockDataLayerStorageMap(new com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<>(), false)); // Paper - avoid copying light data
}
@Override
@@ -20,13 +20,13 @@ public class BlockLightSectionStorage extends LayerLightSectionStorage<BlockLigh
}
2021-06-11 14:02:28 +02:00
protected static final class BlockDataLayerStorageMap extends DataLayerStorageMap<BlockLightSectionStorage.BlockDataLayerStorageMap> {
2021-06-11 14:02:28 +02:00
- public BlockDataLayerStorageMap(Long2ObjectOpenHashMap<DataLayer> arrays) {
- super(arrays);
+ public BlockDataLayerStorageMap(com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<DataLayer> long2objectopenhashmap, boolean isVisible) { // Paper - avoid copying light data
+ super(long2objectopenhashmap, isVisible); // Paper - avoid copying light data
}
@Override
public BlockLightSectionStorage.BlockDataLayerStorageMap copy() {
- return new BlockLightSectionStorage.BlockDataLayerStorageMap(this.map.clone());
+ return new BlockDataLayerStorageMap(this.data, true); // Paper - avoid copying light data
}
}
}
diff --git a/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java b/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java
index 67ff66e232592203cf8dad605ad01eabc4dded89..f357a3473682c2d37a20fb862522c67b9979402a 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java
+++ b/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java
@@ -9,10 +9,23 @@ public abstract class DataLayerStorageMap<M extends DataLayerStorageMap<M>> {
private final long[] lastSectionKeys = new long[2];
private final DataLayer[] lastSections = new DataLayer[2];
private boolean cacheEnabled;
- protected final Long2ObjectOpenHashMap<DataLayer> map;
+ protected final com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<DataLayer> data; // Paper - avoid copying light data
+ protected final boolean isVisible; // Paper - avoid copying light data
+ java.util.function.Function<Long, DataLayer> lookup; // Paper - faster branchless lookup
- protected DataLayerStorageMap(Long2ObjectOpenHashMap<DataLayer> arrays) {
- this.map = arrays;
2021-06-11 14:02:28 +02:00
+ // Paper start - avoid copying light data
+ protected DataLayerStorageMap(com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<DataLayer> data, boolean isVisible) {
+ if (isVisible) {
+ data.performUpdatesLockMap();
+ }
+ this.data = data;
+ this.isVisible = isVisible;
+ if (isVisible) {
+ lookup = data::getVisibleAsync;
+ } else {
+ lookup = data::getUpdating;
+ }
+ // Paper end - avoid copying light data
this.clearCache();
this.cacheEnabled = true;
}
@@ -20,16 +33,17 @@ public abstract class DataLayerStorageMap<M extends DataLayerStorageMap<M>> {
public abstract M copy();
public void copyDataLayer(long pos) {
- this.map.put(pos, this.map.get(pos).copy());
2021-06-11 14:02:28 +02:00
+ if (this.isVisible) { throw new IllegalStateException("writing to visible data"); } // Paper - avoid copying light data
+ this.data.queueUpdate(pos, ((DataLayer) this.data.getUpdating(pos)).copy()); // Paper - avoid copying light data
this.clearCache();
}
public boolean hasLayer(long chunkPos) {
- return this.map.containsKey(chunkPos);
+ return lookup.apply(chunkPos) != null; // Paper - avoid copying light data
}
@Nullable
- public DataLayer getLayer(long chunkPos) {
+ public final DataLayer getLayer(long chunkPos) { // Paper - final
if (this.cacheEnabled) {
for(int i = 0; i < 2; ++i) {
if (chunkPos == this.lastSectionKeys[i]) {
2021-06-11 14:02:28 +02:00
@@ -38,7 +52,7 @@ public abstract class DataLayerStorageMap<M extends DataLayerStorageMap<M>> {
}
}
- DataLayer dataLayer = this.map.get(chunkPos);
+ DataLayer dataLayer = lookup.apply(chunkPos); // Paper - avoid copying light data
if (dataLayer == null) {
2021-06-11 14:02:28 +02:00
return null;
} else {
@@ -58,11 +72,13 @@ public abstract class DataLayerStorageMap<M extends DataLayerStorageMap<M>> {
2021-06-11 14:02:28 +02:00
@Nullable
public DataLayer removeLayer(long chunkPos) {
- return this.map.remove(chunkPos);
2021-06-11 14:02:28 +02:00
+ if (this.isVisible) { throw new IllegalStateException("writing to visible data"); } // Paper - avoid copying light data
+ return (DataLayer) this.data.queueRemove(chunkPos); // Paper - avoid copying light data
}
public void setLayer(long pos, DataLayer data) {
- this.map.put(pos, data);
+ if (this.isVisible) { throw new IllegalStateException("writing to visible data"); } // Paper - avoid copying light data
+ this.data.queueUpdate(pos, data); // Paper - avoid copying light data
}
public void clearCache() {
diff --git a/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java b/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java
2022-05-29 21:54:25 +02:00
index c899674ee24167ee3abdf1588d7396df0ab4f0aa..85175b01b1623b3bc66c65805cec26eaead48265 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java
+++ b/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java
2022-05-29 21:54:25 +02:00
@@ -27,7 +27,7 @@ public abstract class LayerLightSectionStorage<M extends DataLayerStorageMap<M>>
2021-06-11 14:02:28 +02:00
protected final LongSet dataSectionSet = new LongOpenHashSet();
protected final LongSet toMarkNoData = new LongOpenHashSet();
protected final LongSet toMarkData = new LongOpenHashSet();
- protected volatile M visibleSectionData;
+ protected volatile M e_visible; protected final Object visibleUpdateLock = new Object(); // Paper - diff on change, should be "visible" - force compile fail on usage change
protected final M updatingSectionData;
2021-06-11 14:02:28 +02:00
protected final LongSet changedSections = new LongOpenHashSet();
protected final LongSet sectionsAffectedByLightUpdates = new LongOpenHashSet();
2022-05-29 21:54:25 +02:00
@@ -42,8 +42,8 @@ public abstract class LayerLightSectionStorage<M extends DataLayerStorageMap<M>>
2021-06-11 14:02:28 +02:00
this.layer = lightType;
this.chunkSource = chunkProvider;
this.updatingSectionData = lightData;
- this.visibleSectionData = lightData.copy();
- this.visibleSectionData.disableCache();
+ this.e_visible = lightData.copy(); // Paper - avoid copying light dat
+ this.e_visible.disableCache(); // Paper - avoid copying light dat
2021-06-11 14:02:28 +02:00
}
protected boolean storingLightForSection(long sectionPos) {
2022-05-29 21:54:25 +02:00
@@ -52,7 +52,15 @@ public abstract class LayerLightSectionStorage<M extends DataLayerStorageMap<M>>
2021-06-11 14:02:28 +02:00
@Nullable
protected DataLayer getDataLayer(long sectionPos, boolean cached) {
- return this.getDataLayer((M)(cached ? this.updatingSectionData : this.visibleSectionData), sectionPos);
2021-06-11 14:02:28 +02:00
+ // Paper start - avoid copying light data
+ if (cached) {
+ return this.getDataLayer(this.updatingSectionData, sectionPos);
+ } else {
+ synchronized (this.visibleUpdateLock) {
+ return this.getDataLayer(this.e_visible, sectionPos);
+ }
+ }
+ // Paper end - avoid copying light data
}
@Nullable
2022-05-29 21:54:25 +02:00
@@ -342,9 +350,11 @@ public abstract class LayerLightSectionStorage<M extends DataLayerStorageMap<M>>
2021-06-11 14:02:28 +02:00
protected void swapSectionMap() {
if (!this.changedSections.isEmpty()) {
+ synchronized (this.visibleUpdateLock) { // Paper - avoid copying light data
M dataLayerStorageMap = this.updatingSectionData.copy();
dataLayerStorageMap.disableCache();
- this.visibleSectionData = dataLayerStorageMap;
+ this.e_visible = dataLayerStorageMap; // Paper - avoid copying light data
2021-06-11 14:02:28 +02:00
+ } // Paper - avoid copying light data
this.changedSections.clear();
}
diff --git a/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java b/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java
2022-05-29 21:54:25 +02:00
index 59d2af9f883541518c203302257f03dbe957aa0b..e6c857c8b4e4e65e3cf6a75ce6d844ff61acb566 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java
+++ b/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java
@@ -21,7 +21,7 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
2021-06-11 14:02:28 +02:00
private volatile boolean hasSourceInconsistencies;
protected SkyLightSectionStorage(LightChunkGetter chunkProvider) {
- super(LightLayer.SKY, chunkProvider, new SkyLightSectionStorage.SkyDataLayerStorageMap(new Long2ObjectOpenHashMap<>(), new Long2IntOpenHashMap(), Integer.MAX_VALUE));
2021-06-11 14:02:28 +02:00
+ super(LightLayer.SKY, chunkProvider, new SkyLightSectionStorage.SkyDataLayerStorageMap(new com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<>(), new com.destroystokyo.paper.util.map.QueuedChangesMapLong2Int(), Integer.MAX_VALUE, false)); // Paper - avoid copying light data
}
@Override
@@ -32,8 +32,9 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
protected int getLightValue(long blockPos, boolean cached) {
long l = SectionPos.blockToSection(blockPos);
int i = SectionPos.y(l);
- SkyLightSectionStorage.SkyDataLayerStorageMap skyDataLayerStorageMap = cached ? this.updatingSectionData : this.visibleSectionData;
- int j = skyDataLayerStorageMap.topSections.get(SectionPos.getZeroNode(l));
2021-06-11 14:02:28 +02:00
+ synchronized (this.visibleUpdateLock) { // Paper - avoid copying light data
+ SkyLightSectionStorage.SkyDataLayerStorageMap skyDataLayerStorageMap = (SkyLightSectionStorage.SkyDataLayerStorageMap) this.e_visible; // Paper - avoid copying light data - must be after lock acquire
+ int j = skyDataLayerStorageMap.otherData.getVisibleAsync(SectionPos.getZeroNode(l)); // Paper - avoid copying light data
if (j != skyDataLayerStorageMap.currentLowestY && i < j) {
DataLayer dataLayer = this.getDataLayer(skyDataLayerStorageMap, l);
if (dataLayer == null) {
2021-06-11 14:02:28 +02:00
@@ -52,6 +53,7 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
} else {
return cached && !this.lightOnInSection(l) ? 0 : 15;
2021-06-11 14:02:28 +02:00
}
+ } // Paper - avoid copying light data
}
@Override
@@ -59,13 +61,13 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
int i = SectionPos.y(sectionPos);
if ((this.updatingSectionData).currentLowestY > i) {
(this.updatingSectionData).currentLowestY = i;
- (this.updatingSectionData).topSections.defaultReturnValue((this.updatingSectionData).currentLowestY);
+ (this.updatingSectionData).otherData.queueDefaultReturnValue((this.updatingSectionData).currentLowestY); // Paper - avoid copying light data
2021-06-11 14:02:28 +02:00
}
long l = SectionPos.getZeroNode(sectionPos);
- int j = (this.updatingSectionData).topSections.get(l);
+ int j = (this.updatingSectionData).otherData.getUpdating(l); // Paper - avoid copying light data
if (j < i + 1) {
- (this.updatingSectionData).topSections.put(l, i + 1);
+ (this.updatingSectionData).otherData.queueUpdate(l, i + 1); // Paper - avoid copying light data
if (this.columnsWithSkySources.contains(l)) {
2021-06-11 14:02:28 +02:00
this.queueAddSource(sectionPos);
if (j > (this.updatingSectionData).currentLowestY) {
@@ -102,19 +104,19 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
}
2021-06-11 14:02:28 +02:00
int i = SectionPos.y(sectionPos);
- if ((this.updatingSectionData).topSections.get(l) == i + 1) {
+ if ((this.updatingSectionData).otherData.getUpdating(l) == i + 1) { // Paper - avoid copying light data
long m;
for(m = sectionPos; !this.storingLightForSection(m) && this.hasSectionsBelow(i); m = SectionPos.offset(m, Direction.DOWN)) {
--i;
2021-06-11 14:02:28 +02:00
}
if (this.storingLightForSection(m)) {
- (this.updatingSectionData).topSections.put(l, i + 1);
+ (this.updatingSectionData).otherData.queueUpdate(l, i + 1); // Paper - avoid copying light data
if (bl) {
this.queueAddSource(m);
2021-06-11 14:02:28 +02:00
}
} else {
- (this.updatingSectionData).topSections.remove(l);
+ (this.updatingSectionData).otherData.queueRemove(l); // Paper - avoid copying light data
2021-06-11 14:02:28 +02:00
}
}
@@ -128,7 +130,7 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
2021-06-11 14:02:28 +02:00
protected void enableLightSources(long columnPos, boolean enabled) {
this.runAllUpdates();
if (enabled && this.columnsWithSkySources.add(columnPos)) {
- int i = (this.updatingSectionData).topSections.get(columnPos);
+ int i = (this.updatingSectionData).otherData.getUpdating(columnPos); // Paper - avoid copying light data
if (i != (this.updatingSectionData).currentLowestY) {
long l = SectionPos.asLong(SectionPos.x(columnPos), i - 1, SectionPos.z(columnPos));
this.queueAddSource(l);
@@ -152,7 +154,7 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
return dataLayer;
2021-06-11 14:02:28 +02:00
} else {
long l = SectionPos.offset(sectionPos, Direction.UP);
- int i = (this.updatingSectionData).topSections.get(SectionPos.getZeroNode(sectionPos));
+ int i = (this.updatingSectionData).otherData.getUpdating(SectionPos.getZeroNode(sectionPos)); // Paper - avoid copying light data
if (i != (this.updatingSectionData).currentLowestY && SectionPos.y(l) < i) {
DataLayer dataLayer2;
while((dataLayer2 = this.getDataLayer(l, true)) == null) {
2021-07-07 08:52:40 +02:00
@@ -275,7 +277,7 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
2021-06-11 14:02:28 +02:00
protected boolean isAboveData(long sectionPos) {
long l = SectionPos.getZeroNode(sectionPos);
- int i = (this.updatingSectionData).topSections.get(l);
+ int i = (this.updatingSectionData).otherData.getUpdating(l); // Paper - avoid copying light data
return i == (this.updatingSectionData).currentLowestY || SectionPos.y(sectionPos) >= i;
2021-06-11 14:02:28 +02:00
}
2021-07-07 08:52:40 +02:00
@@ -286,18 +288,21 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage<SkyLightSec
protected static final class SkyDataLayerStorageMap extends DataLayerStorageMap<SkyLightSectionStorage.SkyDataLayerStorageMap> {
int currentLowestY;
- final Long2IntOpenHashMap topSections;
2021-06-11 14:02:28 +02:00
-
- public SkyDataLayerStorageMap(Long2ObjectOpenHashMap<DataLayer> arrays, Long2IntOpenHashMap columnToTopSection, int minSectionY) {
- super(arrays);
- this.topSections = columnToTopSection;
- columnToTopSection.defaultReturnValue(minSectionY);
+ private final com.destroystokyo.paper.util.map.QueuedChangesMapLong2Int otherData; // Paper - avoid copying light data
+
+ // Paper start - avoid copying light data
+ public SkyDataLayerStorageMap(com.destroystokyo.paper.util.map.QueuedChangesMapLong2Object<DataLayer> arrays, com.destroystokyo.paper.util.map.QueuedChangesMapLong2Int columnToTopSection, int minSectionY, boolean isVisible) {
+ super(arrays, isVisible);
+ this.otherData = columnToTopSection;
+ otherData.queueDefaultReturnValue(minSectionY);
+ // Paper end
this.currentLowestY = minSectionY;
2021-06-11 14:02:28 +02:00
}
@Override
public SkyLightSectionStorage.SkyDataLayerStorageMap copy() {
- return new SkyLightSectionStorage.SkyDataLayerStorageMap(this.map.clone(), this.topSections.clone(), this.currentLowestY);
+ this.otherData.performUpdatesLockMap(); // Paper - avoid copying light data
+ return new SkyLightSectionStorage.SkyDataLayerStorageMap(this.data, this.otherData, this.currentLowestY, true); // Paper - avoid copying light data
}
}
}