Upstream
/
ViaVersion
mirror of https://github.com/ViaVersion/ViaVersion.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Store item identifiers, make identifier files more compact via global table

This commit is contained in:
Nassim Jahnke 2024-04-21 20:47:38 +02:00
parent d7e66260de
commit f4e9225f85
No known key found for this signature in database
GPG Key ID: EF6771C01F6EF02F
28 changed files with 107 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/main/java/com/viaversion/viaversion/api/data/FullMappings.java
View File

@ -27,7 +27,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
/**
* Mappings containing the full string identifier mappings.
*/
public interface FullMappings extends Mappings {
public interface FullMappings extends BiMappings {
/**
* Returns the unmapped integer id for the given identifier, or -1 if not found.

2
api/src/main/java/com/viaversion/viaversion/api/data/FullMappingsBase.java
View File

@ -22,6 +22,7 @@
*/
package com.viaversion.viaversion.api.data;
import com.google.common.base.Preconditions;
import com.viaversion.viaversion.util.Key;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
@ -37,6 +38,7 @@ public class FullMappingsBase implements FullMappings {
private final Mappings mappings;
public FullMappingsBase(final List<String> unmappedIdentifiers, final List<String> mappedIdentifiers, final Mappings mappings) {
Preconditions.checkNotNull(mappings, "Mappings cannot be null");
this.mappings = mappings;
this.stringToId = toInverseMap(unmappedIdentifiers);
this.mappedStringToId = toInverseMap(mappedIdentifiers);

13
api/src/main/java/com/viaversion/viaversion/api/data/MappingData.java
View File

@ -89,8 +89,21 @@ public interface MappingData {
*/
@Nullable List<TagData> getTags(RegistryType type);
/**
* Returns item mappings.
*
* @return item mappings
*/
@Nullable BiMappings getItemMappings();
/**
* Returns item mappings if they also have identifier data present.
*
* @return item mappings if they also have identifier data present
* @see #getItemMappings()
*/
@Nullable FullMappings getFullItemMappings();
@Nullable ParticleMappings getParticleMappings();
@Nullable Mappings getBlockMappings();

30
api/src/main/java/com/viaversion/viaversion/api/data/MappingDataBase.java
View File

@ -24,8 +24,6 @@ package com.viaversion.viaversion.api.data;
import com.github.steveice10.opennbt.tag.builtin.CompoundTag;
import com.github.steveice10.opennbt.tag.builtin.IntArrayTag;
import com.github.steveice10.opennbt.tag.builtin.ListTag;
import com.github.steveice10.opennbt.tag.builtin.StringTag;
import com.github.steveice10.opennbt.tag.builtin.Tag;
import com.viaversion.viaversion.api.Via;
import com.viaversion.viaversion.api.minecraft.RegistryType;
@ -35,7 +33,6 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.checkerframework.checker.nullness.qual.Nullable;
public class MappingDataBase implements MappingData {
@ -80,28 +77,29 @@ public class MappingDataBase implements MappingData {
enchantmentMappings = loadMappings(data, "enchantments");
paintingMappings = loadMappings(data, "paintings");
attributeMappings = loadMappings(data, "attributes");
itemMappings = loadBiMappings(data, "items");
final CompoundTag unmappedIdentifierData = readUnmappedIdentifiersFile("identifiers-" + unmappedVersion + ".nbt");
final CompoundTag mappedIdentifierData = readMappedIdentifiersFile("identifiers-" + mappedVersion + ".nbt");
if (unmappedIdentifierData != null && mappedIdentifierData != null) {
itemMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "items");
entityMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "entities");
argumentTypeMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "argumenttypes");
recipeSerializerMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "recipe_serializers");
itemDataSerializerMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "data_component_type");
final ListTag<StringTag> unmappedParticles = unmappedIdentifierData.getListTag("particles", StringTag.class);
final ListTag<StringTag> mappedParticles = mappedIdentifierData.getListTag("particles", StringTag.class);
final List<String> unmappedParticles = identifiersFromGlobalIds(unmappedIdentifierData, "particles");
final List<String> mappedParticles = identifiersFromGlobalIds(mappedIdentifierData, "particles");
if (unmappedParticles != null && mappedParticles != null) {
Mappings particleMappings = loadMappings(data, "particles");
if (particleMappings == null) {
particleMappings = new IdentityMappings(unmappedParticles.size(), mappedParticles.size());
}
final List<String> identifiers = unmappedParticles.stream().map(StringTag::getValue).collect(Collectors.toList());
final List<String> mappedIdentifiers = mappedParticles.stream().map(StringTag::getValue).collect(Collectors.toList());
this.particleMappings = new ParticleMappings(identifiers, mappedIdentifiers, particleMappings);
this.particleMappings = new ParticleMappings(unmappedParticles, mappedParticles, particleMappings);
}
} else {
// Might not have identifiers in older versions
itemMappings = loadBiMappings(data, "items");
}
final CompoundTag tagsTag = data.getCompoundTag("tags");
@ -114,6 +112,10 @@ public class MappingDataBase implements MappingData {
loadExtras(data);
}
protected @Nullable List<String> identifiersFromGlobalIds(final CompoundTag mappingsTag, final String key) {
return MappingDataLoader.INSTANCE.identifiersFromGlobalIds(mappingsTag, key);
}
protected @Nullable CompoundTag readMappingsFile(final String name) {
return MappingDataLoader.INSTANCE.loadNBT(name);
}
@ -194,6 +196,14 @@ public class MappingDataBase implements MappingData {
return itemMappings;
}
@Override
public @Nullable FullMappings getFullItemMappings() {
if (itemMappings instanceof FullMappings) {
return (FullMappings) itemMappings;
}
return null;
}
@Override
public @Nullable ParticleMappings getParticleMappings() {
return particleMappings;

73
api/src/main/java/com/viaversion/viaversion/api/data/MappingDataLoader.java
View File

@ -28,6 +28,7 @@ import com.github.steveice10.opennbt.tag.builtin.IntArrayTag;
import com.github.steveice10.opennbt.tag.builtin.IntTag;
import com.github.steveice10.opennbt.tag.builtin.ListTag;
import com.github.steveice10.opennbt.tag.builtin.StringTag;
import com.github.steveice10.opennbt.tag.builtin.Tag;
import com.github.steveice10.opennbt.tag.io.NBTIO;
import com.github.steveice10.opennbt.tag.io.TagReader;
import com.google.common.annotations.Beta;
@ -40,13 +41,16 @@ import com.viaversion.viaversion.api.Via;
import com.viaversion.viaversion.util.GsonUtil;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@ -54,14 +58,14 @@ import org.checkerframework.checker.nullness.qual.Nullable;
public class MappingDataLoader {
public static final MappingDataLoader INSTANCE = new MappingDataLoader(MappingDataLoader.class, "assets/viaversion/data/");
public static final TagReader<CompoundTag> MAPPINGS_READER = NBTIO.reader(CompoundTag.class).named();
private static final Map<String, String[]> GLOBAL_IDENTIFIER_INDEXES = new HashMap<>();
private static final byte DIRECT_ID = 0;
private static final byte SHIFTS_ID = 1;
private static final byte CHANGES_ID = 2;
private static final byte IDENTITY_ID = 3;
public static final MappingDataLoader INSTANCE = new MappingDataLoader(MappingDataLoader.class, "assets/viaversion/data/");
private final Map<String, CompoundTag> mappingsCache = new HashMap<>();
private final Class<?> dataLoaderClass;
private final String dataPath;
@ -72,6 +76,40 @@ public class MappingDataLoader {
this.dataPath = dataPath;
}
public static void loadGlobalIdentifiers() {
// Load in a file with all the identifiers we need, so that we don't need to duplicate them
// for every single new version with only a couple of changes in them.
final CompoundTag globalIdentifiers = INSTANCE.loadNBT("identifier-table.nbt");
for (final Map.Entry<String, Tag> entry : globalIdentifiers.entrySet()) {
//noinspection unchecked
final ListTag<StringTag> value = (ListTag<StringTag>) entry.getValue();
final String[] array = new String[value.size()];
for (int i = 0, size = value.size(); i < size; i++) {
array[i] = value.get(i).getValue();
}
GLOBAL_IDENTIFIER_INDEXES.put(entry.getKey(), array);
}
}
/**
* Returns the global id of the identifier in the registry.
*
* @param registry registry key
* @param globalId global id
* @return identifier
* @throws IllegalArgumentException if the registry key is invalid
*/
public @Nullable String identifierFromGlobalId(final String registry, final int globalId) {
final String[] array = GLOBAL_IDENTIFIER_INDEXES.get(registry);
if (array == null) {
throw new IllegalArgumentException("Unknown global identifier key: " + registry);
}
if (globalId < 0 || globalId >= array.length) {
throw new IllegalArgumentException("Unknown global identifier index: " + globalId);
}
return array[globalId];
}
public void clearCache() {
mappingsCache.clear();
cacheValid = false;
@ -146,7 +184,7 @@ public class MappingDataLoader {
return null;
}
try (final InputStream stream = resource) {
try (final InputStream stream = new BufferedInputStream(resource)) {
return MAPPINGS_READER.read(stream);
} catch (final IOException e) {
throw new RuntimeException(e);
@ -238,23 +276,32 @@ public class MappingDataLoader {
return mappingsSupplier.create(mappings, mappedSizeTag.asInt());
}
public FullMappings loadFullMappings(final CompoundTag mappingsTag, final CompoundTag unmappedIdentifiers, final CompoundTag mappedIdentifiers, final String key) {
final ListTag<StringTag> unmappedElements = unmappedIdentifiers.getListTag(key, StringTag.class);
final ListTag<StringTag> mappedElements = mappedIdentifiers.getListTag(key, StringTag.class);
if (unmappedElements == null || mappedElements == null) {
public FullMappings loadFullMappings(final CompoundTag mappingsTag, final CompoundTag unmappedIdentifiersTag, final CompoundTag mappedIdentifiersTag, final String key) {
if (!unmappedIdentifiersTag.contains(key) || !mappedIdentifiersTag.contains(key)) {
return null;
}
final List<String> unmappedIdentifiers = identifiersFromGlobalIds(unmappedIdentifiersTag, key);
final List<String> mappedIdentifiers = identifiersFromGlobalIds(mappedIdentifiersTag, key);
Mappings mappings = loadMappings(mappingsTag, key);
if (mappings == null) {
mappings = new IdentityMappings(unmappedElements.size(), mappedElements.size());
mappings = new IdentityMappings(unmappedIdentifiers.size(), mappedIdentifiers.size());
}
return new FullMappingsBase(
unmappedElements.stream().map(StringTag::getValue).collect(Collectors.toList()),
mappedElements.stream().map(StringTag::getValue).collect(Collectors.toList()),
mappings
);
return new FullMappingsBase(unmappedIdentifiers, mappedIdentifiers, mappings);
}
public @Nullable List<String> identifiersFromGlobalIds(final CompoundTag mappingsTag, final String key) {
final Mappings mappings = loadMappings(mappingsTag, key);
if (mappings == null) {
return null;
}
final List<String> identifiers = new ArrayList<>(mappings.size());
for (int i = 0; i < mappings.size(); i++) {
identifiers.add(identifierFromGlobalId(key, mappings.getNewId(i)));
}
return identifiers;
}
/**

6
api/src/main/java/com/viaversion/viaversion/api/data/Mappings.java
View File

@ -84,10 +84,4 @@ public interface Mappings {
* @return mappings with keys and values swapped
*/
Mappings inverse();
@FunctionalInterface
interface MappingsSupplier<T extends Mappings> {
T supply(int[] mappings, int mappedIds);
}
}

3
common/src/main/java/com/viaversion/viaversion/ViaManagerImpl.java
View File

@ -21,6 +21,7 @@ import com.viaversion.viaversion.api.Via;
import com.viaversion.viaversion.api.ViaManager;
import com.viaversion.viaversion.api.configuration.ConfigurationProvider;
import com.viaversion.viaversion.api.connection.ConnectionManager;
import com.viaversion.viaversion.api.data.MappingDataLoader;
import com.viaversion.viaversion.api.debug.DebugHandler;
import com.viaversion.viaversion.api.platform.PlatformTask;
import com.viaversion.viaversion.api.platform.UnsupportedSoftware;
@ -93,6 +94,8 @@ public class ViaManagerImpl implements ViaManager {
loadServerProtocol();
}
MappingDataLoader.loadGlobalIdentifiers();
// Register protocols
protocolManager.registerProtocols();

10
common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/data/MappingData.java
View File

@ -26,7 +26,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
public class MappingData extends MappingDataBase {
private KeyMappings items;
private KeyMappings blocks;
private KeyMappings sounds;
@ -39,19 +38,10 @@ public class MappingData extends MappingDataBase {
super.loadExtras(data);
final CompoundTag extraMappings = MappingDataLoader.INSTANCE.loadNBT("extra-identifiers-1.20.3.nbt");
items = new KeyMappings(extraMappings.getListTag("items", StringTag.class));
blocks = new KeyMappings(extraMappings.getListTag("blocks", StringTag.class));
sounds = new KeyMappings(extraMappings.getListTag("sounds", StringTag.class));
}
public int itemId(final String name) {
return items.keyToId(name);
}
public @Nullable String itemName(final int id) {
return items.idToKey(id);
}
public int blockId(final String name) {
return blocks.keyToId(name);
}

2
common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java
View File

@ -544,7 +544,7 @@ public final class BlockItemPacketRewriter1_20_5 extends ItemRewriter<Clientboun
}
private int unmappedItemId(final String name) {
return protocol.getMappingData().itemId(name);
return protocol.getMappingData().getFullItemMappings().id(name);
}
private int toMappedItemId(final String name) {

12
common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/ComponentRewriter1_20_5.java
View File

@ -185,7 +185,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
return;
}
final int itemId = Protocol1_20_5To1_20_3.MAPPINGS.itemId(idTag.getValue());
final int itemId = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().id(idTag.getValue());
if (itemId == -1) {
return;
}
@ -216,7 +216,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
return;
}
final String itemName = Protocol1_20_5To1_20_3.MAPPINGS.itemName(newItem.identifier());
final String itemName = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(newItem.identifier());
if (itemName != null) {
contentsTag.putString("id", itemName);
}
@ -606,7 +606,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
final ArmorTrimMaterial armorTrimMaterial = material.value();
final CompoundTag materialTag = new CompoundTag();
materialTag.putString("asset_name", armorTrimMaterial.assetName());
final String ingredient = Protocol1_20_5To1_20_3.MAPPINGS.itemName(armorTrimMaterial.itemId());
final String ingredient = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(armorTrimMaterial.itemId());
if (ingredient == null) {
throw new IllegalArgumentException("Unknown item: " + armorTrimMaterial.itemId());
}
@ -628,7 +628,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
final ArmorTrimPattern armorTrimPattern = pattern.value();
final CompoundTag patternTag = new CompoundTag();
patternTag.put("asset_id", convertIdentifier(armorTrimPattern.assetName()));
final String templateItem = Protocol1_20_5To1_20_3.MAPPINGS.itemName(armorTrimPattern.itemId());
final String templateItem = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(armorTrimPattern.itemId());
if (templateItem == null) {
throw new IllegalArgumentException("Unknown item: " + armorTrimPattern.itemId());
}
@ -769,7 +769,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
protected ListTag<StringTag> convertPotDecorations(final PotDecorations value) {
final ListTag<StringTag> tag = new ListTag<>(StringTag.class);
for (final int decoration : value.itemIds()) {
final String item = Protocol1_20_5To1_20_3.MAPPINGS.itemName(decoration);
final String item = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(decoration);
if (item == null) {
throw new IllegalArgumentException("Unknown item: " + decoration);
}
@ -877,7 +877,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
}
protected void convertItem(final CompoundTag tag, final Item item) {
final String name = Protocol1_20_5To1_20_3.MAPPINGS.itemName(item.identifier());
final String name = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(item.identifier());
if (name == null) {
throw new IllegalArgumentException("Unknown item: " + item.identifier());
}

2
common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/StructuredDataConverter.java
View File

@ -627,7 +627,7 @@ public final class StructuredDataConverter {
private String toMappedItemName(final int id) {
final int mappedId = unmappedItemId(id);
return mappedId != -1 ? Protocol1_20_5To1_20_3.MAPPINGS.itemName(mappedId) : "";
return mappedId != -1 ? Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(mappedId) : "";
}
private static CompoundTag getBlockEntityTag(final CompoundTag tag) {

BIN
common/src/main/resources/assets/viaversion/data/identifier-table.nbt Normal file
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.13.2.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.13.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.14.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.15.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.16.2.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.16.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.17.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.18.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.19.3.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.19.4.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.19.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.20.2.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.20.3.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.20.5.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/identifiers-1.20.nbt
View File

Binary file not shown.

BIN
common/src/main/resources/assets/viaversion/data/mappings-1.12to1.13.nbt
View File

Binary file not shown.