SerializableData data types can now be indexed

src/main/java/fr/themode/demo/PlayerInit.java

@@ -30,12 +30,13 @@ import net.minestom.server.item.metadata.MapMeta;
 import net.minestom.server.network.ConnectionManager;
 import net.minestom.server.ping.ResponseDataConsumer;
 import net.minestom.server.scoreboard.Sidebar;
+import net.minestom.server.storage.StorageFolder;
+import net.minestom.server.storage.StorageOptions;
 import net.minestom.server.utils.BlockPosition;
 import net.minestom.server.utils.MathUtils;
 import net.minestom.server.utils.Position;
 import net.minestom.server.utils.Vector;
 import net.minestom.server.utils.time.TimeUnit;
-import net.minestom.server.world.DimensionType;
 
 import java.util.Map;
 import java.util.UUID;
@@ -47,11 +48,11 @@ public class PlayerInit {
     private static volatile Inventory inventory;
 
     static {
-        //StorageFolder storageFolder = MinecraftServer.getStorageManager().getFolder("instance_data", new StorageOptions().setCompression(true));
+        StorageFolder storageFolder = MinecraftServer.getStorageManager().getFolder("instance_data", new StorageOptions().setCompression(true));
         ChunkGeneratorDemo chunkGeneratorDemo = new ChunkGeneratorDemo();
         NoiseTestGenerator noiseTestGenerator = new NoiseTestGenerator();
-        //instanceContainer = MinecraftServer.getInstanceManager().createInstanceContainer(storageFolder);
+        instanceContainer = MinecraftServer.getInstanceManager().createInstanceContainer(storageFolder);
-        instanceContainer = MinecraftServer.getInstanceManager().createInstanceContainer(DimensionType.OVERWORLD);
+        //instanceContainer = MinecraftServer.getInstanceManager().createInstanceContainer(DimensionType.OVERWORLD);
         instanceContainer.enableAutoChunkLoad(true);
         //instanceContainer.setChunkDecider((x,y) -> (pos) -> pos.getY()>40?(short)0:(short)1);
         instanceContainer.setChunkGenerator(noiseTestGenerator);

src/main/java/fr/themode/demo/commands/SimpleCommand.java

@@ -69,7 +69,7 @@ public class SimpleCommand implements CommandProcessor {
 
         System.gc();
 
-        //player.getInstance().saveChunksToStorageFolder(() -> System.out.println("end save"));
+        player.getInstance().saveChunksToStorageFolder(() -> System.out.println("end save"));
 
         return true;
     }

src/main/java/fr/themode/demo/generator/NoiseTestGenerator.java

@@ -3,6 +3,7 @@ package fr.themode.demo.generator;
 import de.articdive.jnoise.JNoise;
 import de.articdive.jnoise.interpolation.InterpolationType;
 import net.minestom.server.MinecraftServer;
+import net.minestom.server.data.SerializableData;
 import net.minestom.server.instance.Chunk;
 import net.minestom.server.instance.ChunkGenerator;
 import net.minestom.server.instance.ChunkPopulator;
@@ -11,7 +12,10 @@ import net.minestom.server.instance.block.Block;
 import net.minestom.server.utils.BlockPosition;
 import net.minestom.server.world.biomes.Biome;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
 
 public class NoiseTestGenerator extends ChunkGenerator {
 
@@ -29,7 +33,7 @@ public class NoiseTestGenerator extends ChunkGenerator {
     public void generateChunkData(ChunkBatch batch, int chunkX, int chunkZ) {
         for (int x = 0; x < Chunk.CHUNK_SIZE_X; x++) {
             for (int z = 0; z < Chunk.CHUNK_SIZE_Z; z++) {
-				int height = getHeight(x + chunkX * 16, z + chunkZ * 16);
+                final int height = getHeight(x + chunkX * 16, z + chunkZ * 16);
                 for (int y = 0; y < height; y++) {
                     //if (random.nextInt(100) > 10) {
                     //    batch.setBlock(x, y, z, Block.DIAMOND_BLOCK);
@@ -41,7 +45,10 @@ public class NoiseTestGenerator extends ChunkGenerator {
                     } else if (y == height - 1) {
                         batch.setBlock(x, y, z, Block.GRASS_BLOCK);
                     } else if (y > height - 7) {
-						batch.setBlock(x, y, z, Block.DIRT);
+                        // Data for debugging purpose
+                        SerializableData serializableData = new SerializableData();
+                        serializableData.set("test", 55, Integer.class);
+                        batch.setBlockStateId(x, y, z, Block.DIRT.getBlockId(), serializableData);
                     } else {
                         batch.setBlock(x, y, z, Block.STONE);
                     }

src/main/java/net/minestom/server/data/SerializableData.java

@@ -1,9 +1,16 @@
 package net.minestom.server.data;
 
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import it.unimi.dsi.fastutil.objects.Object2ShortMap;
+import it.unimi.dsi.fastutil.objects.Object2ShortOpenHashMap;
 import net.minestom.server.MinecraftServer;
+import net.minestom.server.reader.DataReader;
 import net.minestom.server.utils.PrimitiveConversion;
+import net.minestom.server.utils.binary.BinaryReader;
 import net.minestom.server.utils.binary.BinaryWriter;
 
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class SerializableData extends Data {
@@ -44,32 +51,105 @@ public class SerializableData extends Data {
     /**
      * Serialize the data into an array of bytes
      * <p>
-     * Use {@link net.minestom.server.reader.DataReader#readData(byte[])}
+     * Use {@link DataReader#readIndexedData(BinaryReader)} if {@code indexed} is true,
-     * to convert it back
+     * {@link DataReader#readData(Object2ShortMap, BinaryReader)} otherwise with the index map
+     * to convert it back to a {@link SerializableData}
      *
+     * @param typeToIndexMap the type to index map, will create entries if new types are discovered.
+     *                       The map is not thread-safe
+     * @param indexed        true to add the types index in the header
      * @return the array representation of this data object
      */
-    public byte[] getSerializedData() {
+    public byte[] getSerializedData(Object2ShortMap<String> typeToIndexMap, boolean indexed) {
+        // Get the current max index, it supposes that the index keep being incremented by 1
+        short lastIndex = (short) typeToIndexMap.size();
+
+        // Main buffer containing the data
         BinaryWriter binaryWriter = new BinaryWriter();
+        for (Map.Entry<String, Object> entry : data.entrySet()) {
+            final String key = entry.getKey();
+            final Object value = entry.getValue();
 
-        data.forEach((key, value) -> {
             final Class type = dataType.get(key);
-            final DataType dataType = DATA_MANAGER.getDataType(type);
+            final short typeIndex;
-
+            {
-            // Write the data type
+                // Find the type name
                 final String encodedType = PrimitiveConversion.getObjectClassString(type.getName()); // Data type (fix for primitives)
-            binaryWriter.writeSizedString(encodedType);
+
+                // Find the type index
+                if (typeToIndexMap.containsKey(encodedType)) {
+                    // Get index
+                    typeIndex = typeToIndexMap.getShort(encodedType);
+                } else {
+                    // Create new index
+                    typeToIndexMap.put(encodedType, ++lastIndex);
+                    // Set index
+                    typeIndex = lastIndex;
+                }
+            }
+
+
+            // Write the data type index
+            binaryWriter.writeShort(typeIndex);
 
             // Write the data key
             binaryWriter.writeSizedString(key);
 
             // Write the data (no length)
+            final DataType dataType = DATA_MANAGER.getDataType(type);
             dataType.encode(binaryWriter, value);
-        });
+        }
 
-        binaryWriter.writeVarInt(0); // End of data object
+        binaryWriter.writeShort((short) 0); // End of data object
+
+        // Header for type indexes
+        if (indexed) {
+            // The buffer containing all the index info (class name to class index)
+            BinaryWriter indexWriter = new BinaryWriter();
+            writeDataIndexHeader(indexWriter, typeToIndexMap);
+            // Merge the index buffer & the main data buffer
+            final ByteBuf finalBuffer = Unpooled.wrappedBuffer(indexWriter.getBuffer(), binaryWriter.getBuffer());
+            // Change the main writer buffer, so it contains both the indexes and the data
+            binaryWriter.setBuffer(finalBuffer);
+        }
 
         return binaryWriter.toByteArray();
     }
 
+    /**
+     * Serialize the data into an array of bytes
+     * <p>
+     * Use {@link net.minestom.server.reader.DataReader#readIndexedData(BinaryReader)}
+     * to convert it back to a {@link SerializableData}
+     * <p>
+     * This will create a type index map which will be present in the header
+     *
+     * @return the array representation of this data object
+     */
+    public byte[] getIndexedSerializedData() {
+        return getSerializedData(new Object2ShortOpenHashMap<>(), true);
+    }
+
+    /**
+     * Get the index info (class name -> class index)
+     * <p>
+     * Sized by a var-int
+     *
+     * @param typeToIndexMap the data index map
+     */
+    public static void writeDataIndexHeader(BinaryWriter indexWriter, Object2ShortMap<String> typeToIndexMap) {
+        // Write the size of the following index list (class name-> class index)
+        indexWriter.writeVarInt(typeToIndexMap.size());
+
+        for (Object2ShortMap.Entry<String> entry : typeToIndexMap.object2ShortEntrySet()) {
+            final String className = entry.getKey();
+            final short classIndex = entry.getShortValue();
+
+            // Write className -> class index
+            indexWriter.writeSizedString(className);
+            indexWriter.writeShort(classIndex);
+
+        }
+    }
+
 }

src/main/java/net/minestom/server/data/type/SerializableDataData.java

@@ -11,11 +11,11 @@ public class SerializableDataData extends DataType<SerializableData> {
 
     @Override
     public void encode(BinaryWriter writer, SerializableData value) {
-        writer.writeBytes(value.getSerializedData());
+        writer.writeBytes(value.getIndexedSerializedData());
     }
 
     @Override
     public SerializableData decode(BinaryReader reader) {
-        return DataReader.readData(reader);
+        return DataReader.readIndexedData(reader);
     }
 }

src/main/java/net/minestom/server/instance/DynamicChunk.java

@@ -1,7 +1,11 @@
 package net.minestom.server.instance;
 
 import com.extollit.gaming.ai.path.model.ColumnarOcclusionFieldList;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2ShortMap;
+import it.unimi.dsi.fastutil.objects.Object2ShortOpenHashMap;
 import net.minestom.server.data.Data;
 import net.minestom.server.data.SerializableData;
 import net.minestom.server.entity.pathfinding.PFBlockDescription;
@@ -145,6 +149,10 @@ public class DynamicChunk extends Chunk {
 
     @Override
     protected byte[] getSerializedData() {
+
+        // Used for blocks data
+        Object2ShortMap<String> typeToIndexMap = new Object2ShortOpenHashMap<>();
+
         BinaryWriter binaryWriter = new BinaryWriter();
 
         // Write the biomes id
@@ -179,13 +187,28 @@ public class DynamicChunk extends Chunk {
                     final boolean hasData = data instanceof SerializableData;
                     binaryWriter.writeBoolean(hasData);
                     if (hasData) {
-                        final byte[] serializedData = ((SerializableData) data).getSerializedData();
+                        // Get the un-indexed data
+                        final byte[] serializedData = ((SerializableData) data).getSerializedData(typeToIndexMap, false);
                         binaryWriter.writeBytes(serializedData);
                     }
                 }
             }
         }
 
+        // If the chunk data contains SerializableData type, it needs to be added in the header
+        BinaryWriter indexWriter = new BinaryWriter();
+        final boolean hasIndex = !typeToIndexMap.isEmpty();
+        indexWriter.writeBoolean(hasIndex);
+        if (hasIndex) {
+            // Get the index buffer (prefixed by true to say that the chunk contains data indexes)
+            SerializableData.writeDataIndexHeader(indexWriter, typeToIndexMap);
+        }
+
+        // Create the final buffer (data index buffer followed by the chunk buffer)
+        final ByteBuf finalBuffer = Unpooled.wrappedBuffer(indexWriter.getBuffer(), binaryWriter.getBuffer());
+        // Change the main writer buffer
+        binaryWriter.setBuffer(finalBuffer);
+
         return binaryWriter.toByteArray();
     }
 

src/main/java/net/minestom/server/reader/ChunkReader.java

@@ -1,5 +1,6 @@
 package net.minestom.server.reader;
 
+import it.unimi.dsi.fastutil.objects.Object2ShortMap;
 import net.minestom.server.MinecraftServer;
 import net.minestom.server.data.Data;
 import net.minestom.server.instance.Chunk;
@@ -19,9 +20,17 @@ public class ChunkReader {
     public static void readChunk(byte[] b, Instance instance, int chunkX, int chunkZ, Consumer<Chunk> callback) {
         BinaryReader binaryReader = new BinaryReader(b);
 
+        // Used for blocks data
+        Object2ShortMap<String> typeToIndexMap = null;
+
         ChunkBatch chunkBatch = null;
         try {
 
+            final boolean hasIndex = binaryReader.readBoolean();
+            if (hasIndex) {
+                typeToIndexMap = DataReader.readDataIndexes(binaryReader);
+            }
+
             Biome[] biomes = new Biome[Chunk.BIOME_COUNT];
             for (int i = 0; i < biomes.length; i++) {
                 final byte id = binaryReader.readByte();
@@ -48,7 +57,8 @@ public class ChunkReader {
                     final boolean hasData = binaryReader.readBoolean();
                     // Data deserializer
                     if (hasData) {
-                        data = DataReader.readData(binaryReader);
+                        // Read the data with the deserialized index map
+                        data = DataReader.readData(typeToIndexMap, binaryReader);
                     }
                 }
 

src/main/java/net/minestom/server/reader/DataReader.java

@@ -1,10 +1,16 @@
 package net.minestom.server.reader;
 
+import it.unimi.dsi.fastutil.objects.Object2ShortMap;
+import it.unimi.dsi.fastutil.objects.Object2ShortOpenHashMap;
+import it.unimi.dsi.fastutil.shorts.Short2ObjectMap;
+import it.unimi.dsi.fastutil.shorts.Short2ObjectOpenHashMap;
 import net.minestom.server.MinecraftServer;
 import net.minestom.server.data.DataManager;
 import net.minestom.server.data.SerializableData;
 import net.minestom.server.utils.binary.BinaryReader;
 
+import java.util.concurrent.ConcurrentHashMap;
+
 /**
  * Class used to convert an array of bytes to a {@link SerializableData}
  * <p>
@@ -14,33 +20,50 @@ public class DataReader {
 
     private static final DataManager DATA_MANAGER = MinecraftServer.getDataManager();
 
+    private static ConcurrentHashMap<String, Class> nameToClassMap = new ConcurrentHashMap<>();
+
     /**
-     * Convert a buffer into a {@link SerializableData}
+     * Convert a buffer into a {@link SerializableData}, this will not read the data index header.
+     * Use {@link #readIndexedData(BinaryReader)} to read the whole data object (if your data contains the indexes)
      * <p>
      * WARNING: the {@link DataManager} needs to have all the required types as the {@link SerializableData} has
      *
+     * @param typeToIndexMap the map which index all the type contained in the data (className->classIndex)
      * @param reader         the reader
      * @return a {@link SerializableData} based on the data input
      */
-    public static SerializableData readData(BinaryReader reader) {
+    public static SerializableData readData(Object2ShortMap<String> typeToIndexMap, BinaryReader reader) {
-        SerializableData data = new SerializableData();
+        final Short2ObjectMap<String> indexToTypeMap = new Short2ObjectOpenHashMap<>(typeToIndexMap.size());
-        try {
+        {
-            while (true) {
+            // Fill the indexToType map
-                final int typeLength = reader.readVarInt();
+            for (Object2ShortMap.Entry<String> entry : typeToIndexMap.object2ShortEntrySet()) {
+                final String type = entry.getKey();
+                final short index = entry.getShortValue();
+                indexToTypeMap.put(index, type);
+            }
+        }
 
-                if (typeLength == 0) {
+        SerializableData data = new SerializableData();
+        while (true) {
+            // Get the class index
+            final short typeIndex = reader.readShort();
+
+            if (typeIndex == 0) {
                 // End of data
                 break;
             }
 
-                // Get the class type
             final Class type;
             {
-                    final byte[] typeCache = reader.readBytes(typeLength);
+                final String className = indexToTypeMap.get(typeIndex);
-
+                type = nameToClassMap.computeIfAbsent(className, s -> {
-                    final String className = new String(typeCache);
+                    try {
-
+                        return Class.forName(className);
-                    type = Class.forName(className);
+                    } catch (ClassNotFoundException e) {
+                        e.printStackTrace();
+                        return null;
+                    }
+                });
             }
 
             // Get the key
@@ -52,22 +75,38 @@ public class DataReader {
             // Set the data
             data.set(name, value, type);
         }
-        } catch (ClassNotFoundException e) {
-            e.printStackTrace();
-        }
 
         return data;
     }
 
     /**
-     * Convert a bytes array to a {@link SerializableData}
+     * Read the indexes of the data + the data
      *
-     * @param data the data
+     * @param reader the reader
-     * @return a {@link SerializableData} based on the data input
+     * @return the deserialized {@link SerializableData}
-     * @see #readData(BinaryReader)
      */
-    public static SerializableData readData(byte[] data) {
+    public static SerializableData readIndexedData(BinaryReader reader) {
-        return readData(new BinaryReader(data));
+        final Object2ShortMap<String> typeToIndexMap = readDataIndexes(reader);
+        return readData(typeToIndexMap, reader);
+    }
+
+    /**
+     * Get a map containing the indexes of your data (type name -> type index)
+     *
+     * @param binaryReader the reader
+     * @return a map containing the indexes of your data
+     */
+    public static Object2ShortMap<String> readDataIndexes(BinaryReader binaryReader) {
+        Object2ShortMap<String> typeToIndexMap = new Object2ShortOpenHashMap<>();
+        {
+            final int dataIndexSize = binaryReader.readVarInt();
+            for (int i = 0; i < dataIndexSize; i++) {
+                final String className = binaryReader.readSizedString();
+                final short classIndex = binaryReader.readShort();
+                typeToIndexMap.put(className, classIndex);
+            }
+        }
+        return typeToIndexMap;
     }
 
 }

src/main/java/net/minestom/server/storage/StorageFolder.java

@@ -96,7 +96,7 @@ public class StorageFolder {
             SerializableData data;
 
             if (bytes != null) {
-                data = DataReader.readData(new BinaryReader(bytes));
+                data = DataReader.readIndexedData(new BinaryReader(bytes));
             } else {
                 data = new SerializableData();
             }
@@ -129,7 +129,7 @@ public class StorageFolder {
             SerializableData data;
 
             if (bytes != null) {
-                data = DataReader.readData(new BinaryReader(bytes));
+                data = DataReader.readIndexedData(new BinaryReader(bytes));
             } else {
                 data = new SerializableData();
             }
@@ -153,7 +153,7 @@ public class StorageFolder {
                 return;
 
             // Save the data
-            set(key, serializableData.getSerializedData());
+            set(key, serializableData.getIndexedSerializedData());
 
             // Remove from map
             this.cachedData.remove(key);
@@ -166,7 +166,7 @@ public class StorageFolder {
     public void saveCachedData() {
         synchronized (cachedData) {
             cachedData.forEach((key, data) -> {
-                set(key, data.getSerializedData());
+                set(key, data.getIndexedSerializedData());
             });
         }
     }
@@ -179,7 +179,7 @@ public class StorageFolder {
     public void saveCachedData(String key) {
         synchronized (cachedData) {
             final SerializableData data = cachedData.get(key);
-            set(key, data.getSerializedData());
+            set(key, data.getIndexedSerializedData());
         }
     }
 

src/main/java/net/minestom/server/utils/binary/BinaryWriter.java

@@ -18,7 +18,7 @@ import java.util.function.Consumer;
 
 public class BinaryWriter extends OutputStream {
 
-    private final ByteBuf buffer;
+    private ByteBuf buffer;
     private final NBTWriter nbtWriter = new NBTWriter(this, false);
 
     /**
@@ -261,6 +261,15 @@ public class BinaryWriter extends OutputStream {
         return buffer;
     }
 
+    /**
+     * Change the buffer used by this binary writer
+     *
+     * @param buffer the new buffer used by this binary writer
+     */
+    public void setBuffer(ByteBuf buffer) {
+        this.buffer = buffer;
+    }
+
     @Override
     public void write(int b) {
         writeByte((byte) b);