Write file data file instead of keeping nbt files in the git tree

output_hashes.json

@@ -1,3 +1,202 @@
 {
-
+  "1.13": {
+    "object-hash": 1835999558,
+    "size": 94
+  },
+  "1.13.2": {
+    "object-hash": 214381667,
+    "size": 192
+  },
+  "1.14": {
+    "object-hash": 1631090930,
+    "size": 570
+  },
+  "1.15": {
+    "object-hash": 667164485,
+    "size": 594
+  },
+  "1.16": {
+    "object-hash": 978512762,
+    "size": 1210
+  },
+  "1.16.2": {
+    "object-hash": 1064891088,
+    "size": 1394
+  },
+  "1.17": {
+    "object-hash": -1238441101,
+    "size": 2594
+  },
+  "1.18": {
+    "object-hash": 521012559,
+    "size": 2707
+  },
+  "1.19": {
+    "object-hash": -929747200,
+    "size": 3444
+  },
+  "1.19.3": {
+    "object-hash": 537457037,
+    "size": 3583
+  },
+  "1.19.4": {
+    "object-hash": -2120264184,
+    "size": 4215
+  },
+  "1.20": {
+    "object-hash": 1019999243,
+    "size": 4287
+  },
+  "1.20.2": {
+    "object-hash": 1019999243,
+    "size": 4287
+  },
+  "1.20.3": {
+    "object-hash": -898988321,
+    "size": 4568
+  },
+  "1.20.5": {
+    "object-hash": 123791910,
+    "size": 4840
+  },
+  "3D_Shareware": {
+    "object-hash": -639482332,
+    "size": 972
+  },
+  "20w14infinite": {
+    "object-hash": 682358467,
+    "size": 1522
+  },
+  "1.10:1.9.4": {
+    "object-hash": 31287637,
+    "size": 1165
+  },
+  "1.11:1.10": {
+    "object-hash": -1578970102,
+    "size": 2050
+  },
+  "1.12:1.11": {
+    "object-hash": 1247217955,
+    "size": 4441
+  },
+  "1.13:1.13.2": {
+    "object-hash": -1354203704,
+    "size": 378
+  },
+  "1.13.2:1.13": {
+    "object-hash": 11515691,
+    "size": 814
+  },
+  "1.13.2:1.14": {
+    "object-hash": 2028478608,
+    "size": 2103
+  },
+  "1.14:1.13.2": {
+    "object-hash": 54259356,
+    "size": 15127
+  },
+  "1.14:1.15": {
+    "object-hash": 1174694439,
+    "size": 456
+  },
+  "1.15:1.14": {
+    "object-hash": -392279077,
+    "size": 5293
+  },
+  "1.15:1.16": {
+    "object-hash": -1634129381,
+    "size": 8553
+  },
+  "1.16:1.15": {
+    "object-hash": 681874607,
+    "size": 59760
+  },
+  "1.16:1.16.2": {
+    "object-hash": 969359550,
+    "size": 560
+  },
+  "1.16.2:1.16": {
+    "object-hash": 771013598,
+    "size": 1087
+  },
+  "1.16.2:1.17": {
+    "object-hash": -1050831736,
+    "size": 4688
+  },
+  "1.17:1.16.2": {
+    "object-hash": -1923510301,
+    "size": 22976
+  },
+  "1.17:1.18": {
+    "object-hash": 1536701233,
+    "size": 351
+  },
+  "1.18:1.17": {
+    "object-hash": 1191025590,
+    "size": 1086
+  },
+  "1.18:1.19": {
+    "object-hash": 624447254,
+    "size": 2519
+  },
+  "1.19:1.18": {
+    "object-hash": -527846779,
+    "size": 12214
+  },
+  "1.19:1.19.3": {
+    "object-hash": -1757215187,
+    "size": 1024
+  },
+  "1.19.3:1.19": {
+    "object-hash": -803581937,
+    "size": 11964
+  },
+  "1.19.3:1.19.4": {
+    "object-hash": -1220416263,
+    "size": 1617
+  },
+  "1.19.4:1.19.3": {
+    "object-hash": -1507041123,
+    "size": 7639
+  },
+  "1.19.4:1.20": {
+    "object-hash": -809213508,
+    "size": 1136
+  },
+  "1.20:1.19.4": {
+    "object-hash": -308457178,
+    "size": 3726
+  },
+  "1.20:1.20.2": {
+    "object-hash": -707055172,
+    "size": 451
+  },
+  "1.20.2:1.20": {
+    "object-hash": 446121302,
+    "size": 819
+  },
+  "1.20.2:1.20.3": {
+    "object-hash": 620178790,
+    "size": 897
+  },
+  "1.20.3:1.20.2": {
+    "object-hash": 280433401,
+    "size": 7630
+  },
+  "1.20.3:1.20.5": {
+    "object-hash": 1354539065,
+    "size": 1021
+  },
+  "1.20.5:1.20.3": {
+    "object-hash": 1855931855,
+    "size": 10411
+  },
+  "3D_Shareware:1.14": {
+    "object-hash": -224826872,
+    "size": 7046
+  },
+  "20w14infinite:1.16": {
+    "object-hash": 1666812931,
+    "size": 9803
+  }
 }

src/main/java/com/viaversion/mappingsgenerator/CursedMappings.java

@@ -26,7 +26,7 @@ public final class CursedMappings {
         final MappingsOptimizer optimizer = create("1.12", "1.13");
         optimizer.cursedMappings("blocks", "blockstates", "blockstates", 4084);
         optimizer.cursedMappings("legacy_enchantments", "enchantments", "enchantments", 72);
-        optimizer.write(MappingsOptimizer.OUTPUT_DIR);
+        optimizer.writeToDir(MappingsOptimizer.OUTPUT_DIR);
     }
 
     public static void optimizeAndSaveOhSoSpecial1_12AsNBTBackwards() throws IOException {
@@ -36,7 +36,7 @@ public final class CursedMappings {
         optimizer.names("items", "itemnames");
         optimizer.fullNames("entitynames", "entitynames");
         optimizer.fullNames("sounds", "soundnames");
-        optimizer.write(MappingsOptimizer.OUTPUT_BACKWARDS_DIR);
+        optimizer.writeToDir(MappingsOptimizer.OUTPUT_BACKWARDS_DIR);
     }
 
     private static MappingsOptimizer create(final String from, final String to) throws IOException {

src/main/java/com/viaversion/mappingsgenerator/ManualRunner.java

@@ -23,6 +23,7 @@ import com.viaversion.mappingsgenerator.util.Version;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -31,11 +32,11 @@ public final class ManualRunner {
 
     private static final Set<String> SPECIAL_BACKWARDS_ONLY = Set.of("1.9.4", "1.10", "1.11");
 
-    // April Fool version -> Release version
-    private static final Map<String, String> SPECIAL_VERSIONS = Map.of(
+    // April Fool version -> Release version. Linked map to keep order during mapping writing
+    private static final Map<String, String> SPECIAL_VERSIONS = new LinkedHashMap<>(Map.of(
             "3D_Shareware", "1.14",
             "20w14infinite", "1.16"
-    );
+    ));
 
     private static final boolean ALL = true;
 

src/main/java/com/viaversion/mappingsgenerator/MappingsLoader.java

@@ -27,13 +27,12 @@ import it.unimi.dsi.fastutil.ints.Int2IntLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
 import java.util.Set;
-
 import org.jetbrains.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,8 +60,9 @@ public final class MappingsLoader {
             return null;
         }
 
-        final String content = Files.readString(path);
-        return GSON.fromJson(content, JsonObject.class);
+        try (final BufferedReader reader = Files.newBufferedReader(path)) {
+            return GSON.fromJson(reader, JsonObject.class);
+        }
     }
 
     /**
@@ -287,6 +287,7 @@ public final class MappingsLoader {
      * @param identityMappings number of identity mappings
      * @param shiftChanges     number of shift changes where a mapped id is not the last mapped id + 1
      */
-    public record MappingsResult(int[] mappings, int mappedSize, int emptyMappings, int identityMappings, int shiftChanges) {
+    public record MappingsResult(int[] mappings, int mappedSize, int emptyMappings, int identityMappings,
+                                 int shiftChanges) {
     }
 }

src/main/java/com/viaversion/mappingsgenerator/MappingsOptimizer.java

@@ -32,6 +32,7 @@ import com.viaversion.mappingsgenerator.util.Version;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -56,7 +57,6 @@ public final class MappingsOptimizer {
     public static final Path MAPPINGS_DIR = Path.of("mappings");
     public static final Path OUTPUT_DIR = Path.of("output");
     public static final Path OUTPUT_BACKWARDS_DIR = OUTPUT_DIR.resolve("backwards");
-    public static final String GLOBAL_IDENTIFIERS_FILE = "identifier-table.json";
     public static final String DIFF_FILE_FORMAT = "mapping-%sto%s.json";
     public static final String MAPPING_FILE_FORMAT = "mapping-%s.json";
     public static final String OUTPUT_FILE_FORMAT = "mappings-%sto%s.nbt";
@@ -83,6 +83,7 @@ public final class MappingsOptimizer {
     );
     private static final Set<String> savedIdentifierFiles = new HashSet<>();
     private static JsonObject globalIdentifiersObject;
+    private static JsonObject fileHashesObject;
 
     private final Set<String> ignoreMissing = new HashSet<>(Arrays.asList("blocks", "statistics"));
     private final CompoundTag output = new CompoundTag();
@@ -120,16 +121,16 @@ public final class MappingsOptimizer {
         optimizer.optimizeAndWrite();
     }
 
-    private JsonObject loadGlobalIdentifiers() {
+    private void loadGlobalFiles() throws IOException {
         // Load and reuse identifiers file, being a global table across all versions
         if (globalIdentifiersObject == null) {
-            try {
-                return MappingsLoader.load(MAPPINGS_DIR, GLOBAL_IDENTIFIERS_FILE);
-            } catch (final IOException e) {
-                throw new RuntimeException("Failed to load global identifiers", e);
+            globalIdentifiersObject = MappingsLoader.load(MAPPINGS_DIR, "identifier-table.json");
+        }
+        if (fileHashesObject == null) {
+            try (final BufferedReader reader = Files.newBufferedReader(Path.of("output_hashes.json"))) {
+                fileHashesObject = MappingsGenerator.GSON.fromJson(reader, JsonObject.class);
             }
         }
-        return globalIdentifiersObject;
     }
 
     public MappingsOptimizer(final String from, final String to) throws IOException {
@@ -172,7 +173,7 @@ public final class MappingsOptimizer {
 
         diffObject = MappingsLoader.load(getDiffDir(), DIFF_FILE_FORMAT.formatted(from, to));
 
-        globalIdentifiersObject = loadGlobalIdentifiers();
+        loadGlobalFiles();
     }
 
     /**
@@ -215,12 +216,20 @@ public final class MappingsOptimizer {
             }
         }
 
-        final Path outputDir = backwards ? OUTPUT_BACKWARDS_DIR : OUTPUT_DIR;
-        write(special ? outputDir.resolve("special") : outputDir);
+        Path outputDir = backwards ? OUTPUT_BACKWARDS_DIR : OUTPUT_DIR;
+        if (special) {
+            outputDir = outputDir.resolve("special");
+        }
+
+        final Path outputPath = outputDir.resolve(OUTPUT_FILE_FORMAT.formatted(fromVersion, toVersion));
+        write(output, outputPath);
 
         // Save full identifiers to a separate file per version
         saveIdentifierFiles(fromVersion, unmappedObject);
         saveIdentifierFiles(toVersion, mappedObject);
+
+        // Store object/file data to keep track of changes
+        addFileData(fromVersion + ":" + toVersion, output.hashCode(), outputPath);
     }
 
     /**
@@ -261,36 +270,61 @@ public final class MappingsOptimizer {
      *
      * @param directory directory to write the output file to
      */
-    public void write(final Path directory) throws IOException {
+    public void writeToDir(final Path directory) throws IOException {
         write(output, directory.resolve(OUTPUT_FILE_FORMAT.formatted(fromVersion, toVersion)));
     }
 
     public void saveIdentifierFiles(final String version, final JsonObject object) throws IOException {
         final CompoundTag identifiers = new CompoundTag();
-        storeIdentifiers(identifiers, object, "entities");
-        storeIdentifiers(identifiers, object, "items");
-        storeIdentifiers(identifiers, object, "particles");
-        storeIdentifiers(identifiers, object, "argumenttypes");
-        storeIdentifiers(identifiers, object, "recipe_serializers");
-        storeIdentifiers(identifiers, object, "data_component_type");
+        storeIdentifierIndexes(identifiers, object, "entities");
+        storeIdentifierIndexes(identifiers, object, "items");
+        storeIdentifierIndexes(identifiers, object, "particles");
+        storeIdentifierIndexes(identifiers, object, "argumenttypes");
+        storeIdentifierIndexes(identifiers, object, "recipe_serializers");
+        storeIdentifierIndexes(identifiers, object, "data_component_type");
 
         // No need to save the same identifiers multiple times if one version appears in multiple runs
         if (savedIdentifierFiles.add(version) && !identifiers.isEmpty()) {
             final Path outputDir = special ? OUTPUT_DIR.resolve("special") : OUTPUT_DIR;
-            write(identifiers, outputDir.resolve(OUTPUT_IDENTIFIERS_FILE_FORMAT.formatted(version)));
+            final Path outputPath = outputDir.resolve(OUTPUT_IDENTIFIERS_FILE_FORMAT.formatted(version));
+
+            write(identifiers, outputPath);
+            addFileData(version, identifiers.hashCode(), outputPath);
         }
 
         // Update global identifiers file if necessary
         if (updatedGlobalIdentifiers) {
-            try (final BufferedWriter writer = Files.newBufferedWriter(MAPPINGS_DIR.resolve(GLOBAL_IDENTIFIERS_FILE))) {
-                MappingsGenerator.GSON.toJson(globalIdentifiersObject, writer);
-            }
-            write((CompoundTag) JsonConverter.toTag(globalIdentifiersObject), OUTPUT_DIR.resolve(OUTPUT_GLOBAL_IDENTIFIERS_FILE));
+            // Also keep a json file around for easier viewing
+            writeJson(globalIdentifiersObject, MAPPINGS_DIR.resolve("identifier-table.json"));
+
+            final Path outputPath = OUTPUT_DIR.resolve(OUTPUT_GLOBAL_IDENTIFIERS_FILE);
+            final CompoundTag globalIdentifiersTag = (CompoundTag) JsonConverter.toTag(globalIdentifiersObject);
+            write(globalIdentifiersTag, outputPath);
             updatedGlobalIdentifiers = false;
             LOGGER.info("Updated global identifiers file");
         }
     }
 
+    private static void addFileData(final String key, final int hash, final Path path) throws IOException {
+        JsonObject fileData = fileHashesObject.getAsJsonObject(key);
+        if (fileData == null) {
+            fileData = new JsonObject();
+            fileHashesObject.add(key, fileData);
+        }
+
+        // The object hash is good enough
+        fileData.addProperty("object-hash", hash);
+        fileData.addProperty("size", Files.size(path));
+
+        writeJson(fileHashesObject, Path.of("output_hashes.json"));
+    }
+
+    private static void writeJson(final JsonObject object, final Path path) throws IOException {
+        try (final BufferedWriter writer = Files.newBufferedWriter(path)) {
+            MappingsGenerator.GSON.toJson(object, writer);
+        }
+    }
+
     /**
      * Checks for unknown fields in the unmapped object and writes them to the tag unchanged.
      */
@@ -469,7 +503,7 @@ public final class MappingsOptimizer {
      * @param object object to read identifiers from
      * @param key    to read from and write to
      */
-    private void storeIdentifiers(
+    private void storeIdentifierIndexes(
             final CompoundTag tag,
             final JsonObject object,
             final String key