Paper/CraftBukkit-Patches/0014-Compressed-Nibble-Arrays.patch

From 004bae243fac61ac72370588bbbe248a767f80c7 Mon Sep 17 00:00:00 2001
From: Mike Primm <mike@primmhome.com>
Date: Sun, 13 Jan 2013 03:49:07 -0800
Subject: [PATCH] Compressed Nibble Arrays

Implement 'lightening' of NibbleArrays - only allocate
 buffers when non-trivial value Saving from 40-45% of memory use by chunk
 section data.

Finish up NibbleArray lightening work - use for Snapshots, reduce copies

Fix nibble handling with NBT - arrays aren't copied by NBTByteArray

diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
index ba6d32f..0b086a2 100644
--- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java
+++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
@@ -218,15 +218,15 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
                 nbttagcompound1.setByte("Y", (byte) (chunksection.getYPosition() >> 4 & 255));
                 nbttagcompound1.setByteArray("Blocks", chunksection.getIdArray());
                 if (chunksection.getExtendedIdArray() != null) {
-                    nbttagcompound1.setByteArray("Add", chunksection.getExtendedIdArray().a);
+                    nbttagcompound1.setByteArray("Add", chunksection.getExtendedIdArray().getValueArray()); // Spigot
                 }
 
-                nbttagcompound1.setByteArray("Data", chunksection.getDataArray().a);
-                nbttagcompound1.setByteArray("BlockLight", chunksection.getEmittedLightArray().a);
+                nbttagcompound1.setByteArray("Data", chunksection.getDataArray().getValueArray()); // Spigot
+                nbttagcompound1.setByteArray("BlockLight", chunksection.getEmittedLightArray().getValueArray()); // Spigot
                 if (flag) {
-                    nbttagcompound1.setByteArray("SkyLight", chunksection.getSkyLightArray().a);
+                    nbttagcompound1.setByteArray("SkyLight", chunksection.getSkyLightArray().getValueArray()); // Spigot
                 } else {
-                    nbttagcompound1.setByteArray("SkyLight", new byte[chunksection.getEmittedLightArray().a.length]);
+                    nbttagcompound1.setByteArray("SkyLight", new byte[chunksection.getEmittedLightArray().getValueArray().length]); // Spigot
                 }
 
                 nbttaglist.add(nbttagcompound1);
diff --git a/src/main/java/net/minecraft/server/ChunkSection.java b/src/main/java/net/minecraft/server/ChunkSection.java
index 3f67a19..e69afba 100644
--- a/src/main/java/net/minecraft/server/ChunkSection.java
+++ b/src/main/java/net/minecraft/server/ChunkSection.java
@@ -134,7 +134,8 @@ public class ChunkSection {
                 }
             }
         } else {
-            byte[] ext = this.extBlockIds.a;
+            this.extBlockIds.forceToNonTrivialArray(); // Spigot
+            byte[] ext = this.extBlockIds.getValueArray();
             for (int off = 0, off2 = 0; off < blkIds.length;) {
                 byte extid = ext[off2];
                 int l = (blkIds[off] & 0xFF) | ((extid & 0xF) << 8); // Even data
@@ -165,6 +166,12 @@ public class ChunkSection {
                 off++;
                 off2++;
             }
+            // Spigot start
+            this.extBlockIds.detectAndProcessTrivialArray();
+            if (this.extBlockIds.isTrivialArray() && (this.extBlockIds.getTrivialArrayValue() == 0)) {
+                this.extBlockIds = null;
+            }
+            // Spigot end
         }
         this.nonEmptyBlockCount = cntNonEmpty;
         this.tickingBlockCount = cntTicking;
@@ -225,12 +232,11 @@ public class ChunkSection {
     public void setExtendedIdArray(NibbleArray nibblearray) {
         // CraftBukkit start - Don't hang on to an empty nibble array
         boolean empty = true;
-        for (int i = 0; i < nibblearray.a.length; i++) {
-            if (nibblearray.a[i] != 0) {
-                empty = false;
-                break;
-            }
+        // Spigot start
+        if ((!nibblearray.isTrivialArray()) || (nibblearray.getTrivialArrayValue() != 0)) {
+            empty = false;
         }
+        // Spigot end
 
         if (empty) {
             return;
@@ -254,11 +260,11 @@ public class ChunkSection {
 
     // CraftBukkit start - Validate array lengths
     private NibbleArray validateNibbleArray(NibbleArray nibbleArray) {
-        if (nibbleArray != null && nibbleArray.a.length < 2048) {
-            byte[] newArray = new byte[2048];
-            System.arraycopy(nibbleArray.a, 0, newArray, 0, ((nibbleArray.a.length > 2048) ? 2048 : nibbleArray.a.length));
-            nibbleArray = new NibbleArray(newArray, 4);
+        // Spigot start - fix for more awesome nibble arrays
+        if (nibbleArray != null && nibbleArray.getByteLength() < 2048) {
+            nibbleArray.resizeArray(2048);
         }
+        // Spigot end
 
         return nibbleArray;
     }
diff --git a/src/main/java/net/minecraft/server/NibbleArray.java b/src/main/java/net/minecraft/server/NibbleArray.java
index 5d75a54..c9bc20c 100644
--- a/src/main/java/net/minecraft/server/NibbleArray.java
+++ b/src/main/java/net/minecraft/server/NibbleArray.java
@@ -1,13 +1,117 @@
 package net.minecraft.server;
 
+import java.util.Arrays; // Spigot
+
 public class NibbleArray {
 
-    public final byte[] a;
+    private byte[] a; // Spigot - remove final, make private (anyone directly accessing this is broken already)
     private final int b;
     private final int c;
+    // Spigot start
+    private byte trivialValue;
+    private byte trivialByte;
+    private int length;
+    private static final int LEN2K = 2048; // Universal length used right now - optimize around this
+    private static final byte[][] TrivLen2k;
+
+    static {
+        TrivLen2k = new byte[16][];
+        for (int i = 0; i < 16; i++) {
+            TrivLen2k[i] = new byte[LEN2K];
+            Arrays.fill(TrivLen2k[i], (byte) (i | (i << 4)));
+        }
+    }
+
+    // Try to convert array to trivial array
+    public void detectAndProcessTrivialArray() {
+        trivialValue = (byte) (a[0] & 0xF);
+        trivialByte = (byte) (trivialValue | (trivialValue << 4));
+        for (int i = 0; i < a.length; i++) {
+            if (a[i] != trivialByte) return;
+        }
+        // All values matches, so array is trivial
+        this.length = a.length;
+        this.a = null;
+    }
+
+    // Force array to non-trivial state
+    public void forceToNonTrivialArray() {
+        if (this.a == null) {
+            this.a = new byte[this.length];
+            if (this.trivialByte != 0) {
+                Arrays.fill(this.a, this.trivialByte);
+            }
+        }
+    }
+
+    // Test if array is in trivial state
+    public boolean isTrivialArray() {
+        return (this.a == null);
+    }
+
+    // Get value of all elements (only valid if array is in trivial state)
+    public int getTrivialArrayValue() {
+        return this.trivialValue;
+    }
+
+    // Get logical length of byte array for nibble data (whether trivial or non-trivial)
+    public int getByteLength() {
+        if (this.a == null) {
+            return this.length;
+        } else {
+            return this.a.length;
+        }
+    }
+
+    // Return byte encoding of array (whether trivial or non-trivial) - returns read-only array if trivial (do not modify!)
+    public byte[] getValueArray() {
+        if (this.a != null) {
+            return this.a;
+        } else {
+            byte[] rslt;
+
+            if (this.length == LEN2K) {  // All current uses are 2k long, but be safe
+                rslt = TrivLen2k[this.trivialValue];
+            } else {
+                rslt = new byte[this.length];
+                if (this.trivialByte != 0) {
+                    Arrays.fill(rslt, this.trivialByte);
+                }
+            }
+            return rslt;
+        }
+    }
+
+    // Copy byte representation of array to given offset in given byte array
+    public int copyToByteArray(byte[] dest, int off) {
+        if (this.a == null) {
+            Arrays.fill(dest, off, off + this.length, this.trivialByte);
+            return off + this.length;
+        } else {
+            System.arraycopy(this.a, 0, dest, off, this.a.length);
+            return off + this.a.length;
+        }
+    }
+
+    // Resize array to given byte length
+    public void resizeArray(int len) {
+        if (this.a == null) {
+            this.length = len;
+        } else if (this.a.length != len) {
+            byte[] newa = new byte[len];
+            System.arraycopy(this.a, 0, newa, 0, ((this.a.length > len) ? len : this.a.length));
+            this.a = newa;
+        }
+    }
+    // Spigot end
 
     public NibbleArray(int i, int j) {
-        this.a = new byte[i >> 1];
+        // Spigot start
+        //this.a = new byte[i >> 1];
+        this.a = null; // Start off as trivial value (all same zero value)
+        this.length = i >> 1;
+        this.trivialByte = this.trivialValue = 0;
+        // Spigot end
         this.b = j;
         this.c = j + 4;
     }
@@ -16,9 +120,11 @@ public class NibbleArray {
         this.a = abyte;
         this.b = i;
         this.c = i + 4;
+        detectAndProcessTrivialArray(); // Spigot
     }
 
     public int a(int i, int j, int k) {
+        if (this.a == null) return this.trivialValue; // Spigot
         int l = j << this.c | k << this.b | i;
         int i1 = l >> 1;
         int j1 = l & 1;
@@ -27,6 +133,18 @@ public class NibbleArray {
     }
 
     public void a(int i, int j, int k, int l) {
+        // Spigot start
+        if (this.a == null) {
+            if (l != this.trivialValue) { // Not same as trivial value, array no longer trivial
+                this.a = new byte[this.length];
+                if (this.trivialByte != 0) {
+                    Arrays.fill(this.a, this.trivialByte);
+                }
+            } else {
+                return;
+            }
+        }
+        // Spigot end
         int i1 = j << this.c | k << this.b | i;
         int j1 = i1 >> 1;
         int k1 = i1 & 1;
diff --git a/src/main/java/net/minecraft/server/OldChunkLoader.java b/src/main/java/net/minecraft/server/OldChunkLoader.java
index 53c1cb5..9a9e20f 100644
--- a/src/main/java/net/minecraft/server/OldChunkLoader.java
+++ b/src/main/java/net/minecraft/server/OldChunkLoader.java
@@ -94,9 +94,11 @@ public class OldChunkLoader {
 
                 nbttagcompound1.setByte("Y", (byte) (k & 255));
                 nbttagcompound1.setByteArray("Blocks", abyte);
-                nbttagcompound1.setByteArray("Data", nibblearray.a);
-                nbttagcompound1.setByteArray("SkyLight", nibblearray1.a);
-                nbttagcompound1.setByteArray("BlockLight", nibblearray2.a);
+                // Spigot start - a -> getValueArray() accessor
+                nbttagcompound1.setByteArray("Data", nibblearray.getValueArray());
+                nbttagcompound1.setByteArray("SkyLight", nibblearray1.getValueArray());
+                nbttagcompound1.setByteArray("BlockLight", nibblearray2.getValueArray());
+                // Spigot end
                 nbttaglist.add(nbttagcompound1);
             }
         }
diff --git a/src/main/java/net/minecraft/server/Packet51MapChunk.java b/src/main/java/net/minecraft/server/Packet51MapChunk.java
new file mode 100644
index 0000000..b7b4fd6
--- /dev/null
+++ b/src/main/java/net/minecraft/server/Packet51MapChunk.java
@@ -0,0 +1,198 @@
+package net.minecraft.server;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+
+public class Packet51MapChunk extends Packet {
+
+    public int a;
+    public int b;
+    public int c;
+    public int d;
+    private byte[] buffer;
+    private byte[] inflatedBuffer;
+    public boolean e;
+    private int size;
+    private static byte[] buildBuffer = new byte[196864];
+
+    public Packet51MapChunk() {
+        this.lowPriority = true;
+    }
+
+    public Packet51MapChunk(Chunk chunk, boolean flag, int i) {
+        this.lowPriority = true;
+        this.a = chunk.x;
+        this.b = chunk.z;
+        this.e = flag;
+        ChunkMap chunkmap = a(chunk, flag, i);
+        Deflater deflater = new Deflater(-1);
+
+        this.d = chunkmap.c;
+        this.c = chunkmap.b;
+
+        try {
+            this.inflatedBuffer = chunkmap.a;
+            deflater.setInput(chunkmap.a, 0, chunkmap.a.length);
+            deflater.finish();
+            this.buffer = new byte[chunkmap.a.length];
+            this.size = deflater.deflate(this.buffer);
+        } finally {
+            deflater.end();
+        }
+    }
+
+    public void a(DataInput datainput) throws IOException { // CraftBukkit - throws IOException
+        this.a = datainput.readInt();
+        this.b = datainput.readInt();
+        this.e = datainput.readBoolean();
+        this.c = datainput.readShort();
+        this.d = datainput.readShort();
+        this.size = datainput.readInt();
+        if (buildBuffer.length < this.size) {
+            buildBuffer = new byte[this.size];
+        }
+
+        datainput.readFully(buildBuffer, 0, this.size);
+        int i = 0;
+
+        int j;
+
+        for (j = 0; j < 16; ++j) {
+            i += this.c >> j & 1;
+        }
+
+        j = 12288 * i;
+        if (this.e) {
+            j += 256;
+        }
+
+        this.inflatedBuffer = new byte[j];
+        Inflater inflater = new Inflater();
+
+        inflater.setInput(buildBuffer, 0, this.size);
+
+        try {
+            inflater.inflate(this.inflatedBuffer);
+        } catch (DataFormatException dataformatexception) {
+            throw new IOException("Bad compressed data format");
+        } finally {
+            inflater.end();
+        }
+    }
+
+    public void a(DataOutput dataoutput) throws IOException { // CraftBukkit - throws IOException
+        dataoutput.writeInt(this.a);
+        dataoutput.writeInt(this.b);
+        dataoutput.writeBoolean(this.e);
+        dataoutput.writeShort((short) (this.c & '\uffff'));
+        dataoutput.writeShort((short) (this.d & '\uffff'));
+        dataoutput.writeInt(this.size);
+        dataoutput.write(this.buffer, 0, this.size);
+    }
+
+    public void handle(Connection connection) {
+        connection.a(this);
+    }
+
+    public int a() {
+        return 17 + this.size;
+    }
+
+    public static ChunkMap a(Chunk chunk, boolean flag, int i) {
+        int j = 0;
+        ChunkSection[] achunksection = chunk.i();
+        int k = 0;
+        ChunkMap chunkmap = new ChunkMap();
+        byte[] abyte = buildBuffer;
+
+        if (flag) {
+            chunk.seenByPlayer = true;
+        }
+
+        int l;
+
+        for (l = 0; l < achunksection.length; ++l) {
+            if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && (i & 1 << l) != 0) {
+                chunkmap.b |= 1 << l;
+                if (achunksection[l].getExtendedIdArray() != null) {
+                    chunkmap.c |= 1 << l;
+                    ++k;
+                }
+            }
+        }
+
+        for (l = 0; l < achunksection.length; ++l) {
+            if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && (i & 1 << l) != 0) {
+                byte[] abyte1 = achunksection[l].getIdArray();
+
+                System.arraycopy(abyte1, 0, abyte, j, abyte1.length);
+                j += abyte1.length;
+            }
+        }
+
+        NibbleArray nibblearray;
+
+        for (l = 0; l < achunksection.length; ++l) {
+            if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && (i & 1 << l) != 0) {
+                nibblearray = achunksection[l].getDataArray();
+                // Spigot start
+                // System.arraycopy(nibblearray.a, 0, abyte, j, nibblearray.a.length);
+                // j += nibblearray.a.length;
+                j = nibblearray.copyToByteArray(abyte, j);
+                // Spigot end
+            }
+        }
+
+        for (l = 0; l < achunksection.length; ++l) {
+            if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && (i & 1 << l) != 0) {
+                nibblearray = achunksection[l].getEmittedLightArray();
+                // Spigot start
+                // System.arraycopy(nibblearray.a, 0, abyte, j, nibblearray.a.length);
+                // j += nibblearray.a.length;
+                j = nibblearray.copyToByteArray(abyte, j);
+                // Spigot end
+            }
+        }
+
+        if (!chunk.world.worldProvider.f) {
+            for (l = 0; l < achunksection.length; ++l) {
+                if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && (i & 1 << l) != 0) {
+                    nibblearray = achunksection[l].getSkyLightArray();
+                    // Spigot start
+                    // System.arraycopy(nibblearray.a, 0, abyte, j, nibblearray.a.length);
+                    // j += nibblearray.a.length;
+                    j = nibblearray.copyToByteArray(abyte, j);
+                    // Spigot end
+                }
+            }
+        }
+
+        if (k > 0) {
+            for (l = 0; l < achunksection.length; ++l) {
+                if (achunksection[l] != null && (!flag || !achunksection[l].isEmpty()) && achunksection[l].getExtendedIdArray() != null && (i & 1 << l) != 0) {
+                    nibblearray = achunksection[l].getExtendedIdArray();
+                    // Spigot start
+                    //System.arraycopy(nibblearray.a, 0, abyte, j, nibblearray.a.length);
+                    //j += nibblearray.a.length;
+                    j = nibblearray.copyToByteArray(abyte, j);
+                    // Spigot end
+                }
+            }
+        }
+
+        if (flag) {
+            byte[] abyte2 = chunk.m();
+
+            System.arraycopy(abyte2, 0, abyte, j, abyte2.length);
+            j += abyte2.length;
+        }
+
+        chunkmap.a = new byte[j];
+        System.arraycopy(abyte, 0, chunkmap.a, 0, j);
+        return chunkmap;
+    }
+}
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftChunk.java b/src/main/java/org/bukkit/craftbukkit/CraftChunk.java
index 1a21516..6921206 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftChunk.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftChunk.java
@@ -174,7 +174,18 @@ public class CraftChunk implements Chunk {
                 }
 
                 if (cs[i].getExtendedIdArray() != null) { /* If we've got extended IDs */
-                    byte[] extids = cs[i].getExtendedIdArray().a;
+                    // Spigot start
+                    if (cs[i].getExtendedIdArray().isTrivialArray()) {
+                        int tval = cs[i].getExtendedIdArray().getTrivialArrayValue();
+                        if (tval != 0) {
+                            tval = tval << 8;
+                            for (int j = 0; j < 4096; j++) {
+                                blockids[j] |= tval;
+                            }
+                        }
+                  } else {
+                    byte[] extids = cs[i].getExtendedIdArray().getValueArray();
+                    // Spigot end
 
                     for (int j = 0; j < 2048; j++) {
                         short b = (short) (extids[j] & 0xFF);
@@ -186,21 +197,42 @@ public class CraftChunk implements Chunk {
                         blockids[j<<1] |= (b & 0x0F) << 8;
                         blockids[(j<<1)+1] |= (b & 0xF0) << 4;
                     }
+                    } // Spigot
                 }
 
                 sectionBlockIDs[i] = blockids;
 
                 /* Get block data nibbles */
-                sectionBlockData[i] = new byte[2048];
-                System.arraycopy(cs[i].getDataArray().a, 0, sectionBlockData[i], 0, 2048);
+                // Spigot start
+                if (cs[i].getDataArray().isTrivialArray() && (cs[i].getDataArray().getTrivialArrayValue() == 0)) {
+                    sectionBlockData[i] = emptyData;
+                } else {
+                    sectionBlockData[i] = new byte[2048];
+                    cs[i].getDataArray().copyToByteArray(sectionBlockData[i], 0);
+                }
                 if (cs[i].getSkyLightArray() == null) {
                     sectionSkyLights[i] = emptyData;
+                }
+                else if (cs[i].getSkyLightArray().isTrivialArray()) {
+                    if (cs[i].getSkyLightArray().getTrivialArrayValue() == 0) {
+                        sectionSkyLights[i] = emptyData;
+                    } else if (cs[i].getSkyLightArray().getTrivialArrayValue() == 15) {
+                        sectionSkyLights[i] = emptySkyLight;
+                    } else {
+                        sectionSkyLights[i] = new byte[2048];
+                        cs[i].getSkyLightArray().copyToByteArray(sectionSkyLights[i], 0);
+                    }
                 } else {
                     sectionSkyLights[i] = new byte[2048];
-                    System.arraycopy(cs[i].getSkyLightArray().a, 0, sectionSkyLights[i], 0, 2048);
+                    cs[i].getSkyLightArray().copyToByteArray(sectionSkyLights[i], 0);
+                }
+                if (cs[i].getEmittedLightArray().isTrivialArray() && (cs[i].getEmittedLightArray().getTrivialArrayValue() == 0)) {
+                    sectionEmitLights[i] = emptyData;
+                } else {
+                    sectionEmitLights[i] = new byte[2048];
+                    cs[i].getEmittedLightArray().copyToByteArray(sectionEmitLights[i], 0);
                 }
-                sectionEmitLights[i] = new byte[2048];
-                System.arraycopy(cs[i].getEmittedLightArray().a, 0, sectionEmitLights[i], 0, 2048);
+                // Spigot end
             }
         }
 
-- 
1.8.1.2