Add items to identifiers and compact identifier files

Instead of adding all string identifier for every single version, they're now collected in a global index file whose ids are then referenced in the individual identifier files

.gitignore

@@ -1,6 +1,7 @@
 libraries/
 logs/
 generated/
+output/
 
 ### Java files ###
 *.class

README.md

@@ -55,8 +55,10 @@ own [ViaNBT](https://github.com/ViaVersion/ViaNBT) as the NBT reader/writer. Com
 
 ### Identifier files
 
-Next to a standardized compact format, there are extra files per version to store the identifier arrays of certain
-registries, as their full keys might be required. These files simply contain any number of json arrays.
+Next to a standardized compact format for int id mappings, the full identifiers of some registries are also required.
+For this, we generate a list of *all* identifiers in the registry across all versions, so that their names only need to
+be stored once, as opposed to storing them again in every new version they are still in. Wherever needed, these
+identifiers are then referred to via their index in the global list.
 
 ### Mapping files
 
@@ -81,26 +83,26 @@ The direct storage simply stores an array of ints exactly as they can be used in
 * `id` (byte tag) is `0`
 * `val` (int array tag) contains the mapped ids, where their array index corresponds to the unmapped id
 
-### Changed value storage
-
-The changed value storage stores two int arrays: One containing the changed unmapped ids, and one their corresponding
-mapped ids in a simple int→int mapping over the two arrays.
-
-* `id` (byte tag) is `1`
-* `at` (int array tag) contains the unmapped ids that have been changed
-* `val` (int array tag) contains the mapped ids, indexed by the same index as the unmapped id in `at`
-* Optional: `nofill` (byte tag): Unless present, all ids between the ones found in `at` are mapped to their identity
-
 ### Shifted value storage
 
 The shifted value storage stores two int arrays: One containing the unmapped ids that end a sequence of mapped ids. For
 an index `i`, all unmapped ids between `at[i] + sequence` (inclusive) and `at[i + 1]` (exclusive) are mapped
 to `to[i] + sequence`.
 
-* `id` (byte tag) is `2`
+* `id` (byte tag) is `1`
 * `at` (int array tag) contains the unmapped ids, where their mapped is is *not* simply the last mapped id + 1
 * `to` (int array tag) contains the mapped ids, indexed by the same index as the unmapped id in `at`
 
+### Changed value storage
+
+The changed value storage stores two int arrays: One containing the changed unmapped ids, and one their corresponding
+mapped ids in a simple int→int mapping over the two arrays.
+
+* `id` (byte tag) is `2`
+* `at` (int array tag) contains the unmapped ids that have been changed
+* `val` (int array tag) contains the mapped ids, indexed by the same index as the unmapped id in `at`
+* Optional: `nofill` (byte tag): Unless present, all ids between the ones found in `at` are mapped to their identity
+
 ### Identity storage
 
 The identity storage signifies that every id between `0` and `size` is mapped to itself. This is sometimes used over

build.gradle.kts

@@ -22,7 +22,7 @@ dependencies {
 }
 
 group = "com.viaversion"
-version = "3.4.0"
+version = "4.0.0"
 description = "MappingsGenerator"
 java.sourceCompatibility = JavaVersion.VERSION_21
 

output_hashes.json

@@ -0,0 +1,3 @@
+{
+
+}

src/main/java/com/viaversion/mappingsgenerator/MappingsOptimizer.java

@@ -20,8 +20,6 @@ package com.viaversion.mappingsgenerator;
 
 import com.github.steveice10.opennbt.tag.builtin.CompoundTag;
 import com.github.steveice10.opennbt.tag.builtin.IntArrayTag;
-import com.github.steveice10.opennbt.tag.builtin.ListTag;
-import com.github.steveice10.opennbt.tag.builtin.StringTag;
 import com.github.steveice10.opennbt.tag.builtin.Tag;
 import com.github.steveice10.opennbt.tag.io.NBTIO;
 import com.github.steveice10.opennbt.tag.io.TagWriter;
@@ -33,7 +31,8 @@ import com.viaversion.mappingsgenerator.util.JsonConverter;
 import com.viaversion.mappingsgenerator.util.Version;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
-
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -41,7 +40,6 @@ import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,10 +56,12 @@ public final class MappingsOptimizer {
     public static final Path MAPPINGS_DIR = Path.of("mappings");
     public static final Path OUTPUT_DIR = Path.of("output");
     public static final Path OUTPUT_BACKWARDS_DIR = OUTPUT_DIR.resolve("backwards");
+    public static final String GLOBAL_IDENTIFIERS_FILE = "identifier-table.json";
     public static final String DIFF_FILE_FORMAT = "mapping-%sto%s.json";
     public static final String MAPPING_FILE_FORMAT = "mapping-%s.json";
     public static final String OUTPUT_FILE_FORMAT = "mappings-%sto%s.nbt";
     public static final String OUTPUT_IDENTIFIERS_FILE_FORMAT = "identifiers-%s.nbt";
+    public static final String OUTPUT_GLOBAL_IDENTIFIERS_FILE = "identifier-table.nbt";
 
     private static final Logger LOGGER = LoggerFactory.getLogger(MappingsOptimizer.class.getSimpleName());
     private static final TagWriter TAG_WRITER = NBTIO.writer().named();
@@ -81,7 +81,8 @@ public final class MappingsOptimizer {
             "tags",
             "attributes"
     );
-    private static final Set<String> SAVED_IDENTIFIER_FILES = new HashSet<>();
+    private static final Set<String> savedIdentifierFiles = new HashSet<>();
+    private static JsonObject globalIdentifiersObject;
 
     private final Set<String> ignoreMissing = new HashSet<>(Arrays.asList("blocks", "statistics"));
     private final CompoundTag output = new CompoundTag();
@@ -94,6 +95,7 @@ public final class MappingsOptimizer {
     private ErrorStrategy errorStrategy = ErrorStrategy.WARN;
     private JsonObject diffObject;
     private boolean keepUnknownFields;
+    private boolean updatedGlobalIdentifiers;
 
     public static void main(final String[] args) throws IOException {
         if (args.length < 2) {
@@ -118,6 +120,18 @@ public final class MappingsOptimizer {
         optimizer.optimizeAndWrite();
     }
 
+    private JsonObject loadGlobalIdentifiers() {
+        // Load and reuse identifiers file, being a global table across all versions
+        if (globalIdentifiersObject == null) {
+            try {
+                return MappingsLoader.load(MAPPINGS_DIR, GLOBAL_IDENTIFIERS_FILE);
+            } catch (final IOException e) {
+                throw new RuntimeException("Failed to load global identifiers", e);
+            }
+        }
+        return globalIdentifiersObject;
+    }
+
     public MappingsOptimizer(final String from, final String to) throws IOException {
         this(from, to, false);
     }
@@ -157,6 +171,8 @@ public final class MappingsOptimizer {
         }
 
         diffObject = MappingsLoader.load(getDiffDir(), DIFF_FILE_FORMAT.formatted(from, to));
+
+        globalIdentifiersObject = loadGlobalIdentifiers();
     }
 
     /**
@@ -189,8 +205,8 @@ public final class MappingsOptimizer {
             names("items", "itemnames");
             names("enchantments", "enchantmentnames");
             fullNames("entitynames", "entitynames");
-
             if (backwards) {
+                // No need to put sounds into the identifier files, so just use full names
                 fullNames("sounds", "soundnames");
             }
 
@@ -252,14 +268,27 @@ public final class MappingsOptimizer {
     public void saveIdentifierFiles(final String version, final JsonObject object) throws IOException {
         final CompoundTag identifiers = new CompoundTag();
         storeIdentifiers(identifiers, object, "entities");
+        storeIdentifiers(identifiers, object, "items");
         storeIdentifiers(identifiers, object, "particles");
         storeIdentifiers(identifiers, object, "argumenttypes");
         storeIdentifiers(identifiers, object, "recipe_serializers");
         storeIdentifiers(identifiers, object, "data_component_type");
-        if (SAVED_IDENTIFIER_FILES.add(version)) {
+
+        // No need to save the same identifiers multiple times if one version appears in multiple runs
+        if (savedIdentifierFiles.add(version) && !identifiers.isEmpty()) {
             final Path outputDir = special ? OUTPUT_DIR.resolve("special") : OUTPUT_DIR;
             write(identifiers, outputDir.resolve(OUTPUT_IDENTIFIERS_FILE_FORMAT.formatted(version)));
         }
+
+        // Update global identifiers file if necessary
+        if (updatedGlobalIdentifiers) {
+            try (final BufferedWriter writer = Files.newBufferedWriter(MAPPINGS_DIR.resolve(GLOBAL_IDENTIFIERS_FILE))) {
+                MappingsGenerator.GSON.toJson(globalIdentifiersObject, writer);
+            }
+            write((CompoundTag) JsonConverter.toTag(globalIdentifiersObject), OUTPUT_DIR.resolve(OUTPUT_GLOBAL_IDENTIFIERS_FILE));
+            updatedGlobalIdentifiers = false;
+            LOGGER.info("Updated global identifiers file");
+        }
     }
 
     /**
@@ -434,28 +463,63 @@ public final class MappingsOptimizer {
     }
 
     /**
-     * Stores a list of string identifiers in the given tag.
+     * Stores a list of global identifier indexes in the given tag.
      *
      * @param tag    tag to write to
      * @param object object to read identifiers from
      * @param key    to read from and write to
      */
-    private static void storeIdentifiers(
+    private void storeIdentifiers(
             final CompoundTag tag,
             final JsonObject object,
             final String key
     ) {
-        final JsonArray identifiers = object.getAsJsonArray(key);
-        if (identifiers == null) {
+        final JsonElement identifiersElement = object.get(key);
+        if (identifiersElement == null) {
             return;
         }
 
-        final ListTag<StringTag> list = new ListTag<>(StringTag.class);
-        for (final JsonElement identifier : identifiers) {
-            list.add(new StringTag(identifier.getAsString()));
+        if (identifiersElement.isJsonObject()) {
+            // Pre 1.13
+            LOGGER.debug("Identifiers for {} are not an array", key);
+            return;
         }
 
-        tag.put(key, list);
+        // Add to global identifiers if not already present
+        final JsonArray identifiers = identifiersElement.getAsJsonArray();
+        JsonArray globalIdentifiersArray = globalIdentifiersObject.getAsJsonArray(key);
+        if (globalIdentifiersArray == null) {
+            globalIdentifiersArray = new JsonArray();
+            globalIdentifiersObject.add(key, globalIdentifiersArray);
+        }
+
+        final Object2IntMap<String> globalIdentifiers = new Object2IntOpenHashMap<>(globalIdentifiersArray.size());
+        globalIdentifiers.defaultReturnValue(-1);
+        for (int globalId = 0; globalId < globalIdentifiersArray.size(); globalId++) {
+            final String identifier = globalIdentifiersArray.get(globalId).getAsString();
+            globalIdentifiers.put(identifier, globalId);
+        }
+
+        for (int id = 0; id < identifiers.size(); id++) {
+            final JsonElement entry = identifiers.get(id);
+            if (entry.isJsonNull()) {
+                continue;
+            }
+
+            final String identifier = entry.getAsString();
+            if (globalIdentifiers.containsKey(identifier)) {
+                continue;
+            }
+
+            final int addedGlobalIndex = globalIdentifiersArray.size();
+            globalIdentifiersArray.add(identifier);
+            globalIdentifiers.put(identifier, addedGlobalIndex);
+            updatedGlobalIdentifiers = true;
+        }
+
+        // Use the same compact storage on the identifier->global identifier files, just about halves the size
+        final MappingsResult result = MappingsLoader.map(identifiers, globalIdentifiersArray, null, errorStrategy);
+        serialize(result, tag, key, true);
     }
 
     /**

src/main/java/com/viaversion/mappingsgenerator/extra/ItemsAndBlocks1_20_3.java

@@ -36,7 +36,6 @@ public final class ItemsAndBlocks1_20_3 {
     public static void main(final String[] args) throws IOException {
         final JsonObject mappings = MappingsLoader.load("mapping-1.20.3.json");
         final CompoundTag tag = new CompoundTag();
-        tag.put("items", collectStringList(mappings.getAsJsonArray("items")));
         tag.put("blocks", collectStringList(mappings.getAsJsonArray("blocks")));
         tag.put("sounds", collectStringList(mappings.getAsJsonArray("sounds")));
         MappingsOptimizer.write(tag, MappingsOptimizer.OUTPUT_DIR.resolve("extra/extra-identifiers-1.20.3.nbt"));