From 6f71b7c4a92cb28a43a7da4b12f8f19ce9be61a7 Mon Sep 17 00:00:00 2001 From: Jason <11360596+jpenilla@users.noreply.github.com> Date: Thu, 4 Nov 2021 17:23:06 -0700 Subject: [PATCH] Deduplicate strings in ObfHelper (#6841) --- build.gradle.kts | 24 ++- ...ktraces-in-log-messages-crash-report.patch | 139 +++++++++--------- .../server/0430-Implement-Mob-Goal-API.patch | 6 +- ...pport-for-hex-color-codes-in-console.patch | 4 +- ...dd-git-branch-and-commit-to-manifest.patch | 6 +- ...ocity-compression-and-cipher-natives.patch | 6 +- 6 files changed, 94 insertions(+), 91 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 5e325d0b6b..9a44dad124 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,14 +56,14 @@ paperweight { minecraftVersion.set(providers.gradleProperty("mcVersion")) serverProject.set(project(":Paper-Server")) + paramMappingsRepo.set("https://maven.quiltmc.org/repository/release/") + remapRepo.set("https://maven.fabricmc.net/") + decompileRepo.set("https://files.minecraftforge.net/maven/") + paper { spigotApiPatchDir.set(layout.projectDirectory.dir("patches/api")) spigotServerPatchDir.set(layout.projectDirectory.dir("patches/server")) - paramMappingsRepo.set("https://maven.quiltmc.org/repository/release/") - remapRepo.set("https://maven.fabricmc.net/") - decompileRepo.set("https://files.minecraftforge.net/maven/") - mappingsPatch.set(layout.projectDirectory.file("build-data/mappings-patch.tiny")) reobfMappingsPatch.set(layout.projectDirectory.file("build-data/reobf-mappings-patch.tiny")) @@ -86,18 +86,15 @@ paperweight { tasks.generateDevelopmentBundle { apiCoordinates.set("io.papermc.paper:paper-api") mojangApiCoordinates.set("io.papermc.paper:paper-mojangapi") - libraryRepositories.set( - listOf( - "https://repo.maven.apache.org/maven2/", - "https://libraries.minecraft.net/", - "https://papermc.io/repo/repository/maven-public/", - "https://maven.quiltmc.org/repository/release/", - ) + libraryRepositories.addAll( + "https://repo.maven.apache.org/maven2/", + "https://libraries.minecraft.net/", + "https://papermc.io/repo/repository/maven-public/", ) } publishing { - if (project.hasProperty("publishDevBundle")) { + if (project.providers.gradleProperty("publishDevBundle").forUseAtConfigurationTime().isPresent) { publications.create("devBundle") { artifact(tasks.generateDevelopmentBundle) { artifactId = "dev-bundle" @@ -109,9 +106,8 @@ publishing { allprojects { publishing { repositories { - maven { + maven("https://papermc.io/repo/repository/maven-snapshots/") { name = "paperSnapshots" - url = uri("https://papermc.io/repo/repository/maven-snapshots/") credentials(PasswordCredentials::class) } } diff --git a/patches/server/0429-Deobfuscate-stacktraces-in-log-messages-crash-report.patch b/patches/server/0429-Deobfuscate-stacktraces-in-log-messages-crash-report.patch index ba8bb60f08..40e86c7146 100644 --- a/patches/server/0429-Deobfuscate-stacktraces-in-log-messages-crash-report.patch +++ b/patches/server/0429-Deobfuscate-stacktraces-in-log-messages-crash-report.patch @@ -6,7 +6,7 @@ Subject: [PATCH] Deobfuscate stacktraces in log messages, crash reports, and diff --git a/build.gradle.kts b/build.gradle.kts -index a482ad74b9a5d06a982ac2a6d9b6c5dc9f664f46..7a7a6a1a1bdfecb619dd6ba6f6865eb0bb74d0d9 100644 +index a482ad74b9a5d06a982ac2a6d9b6c5dc9f664f46..974f0bbbd3d271d28ce884490dc68b68849a07c3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,9 @@ @@ -19,31 +19,36 @@ index a482ad74b9a5d06a982ac2a6d9b6c5dc9f664f46..7a7a6a1a1bdfecb619dd6ba6f6865eb0 import java.util.Locale plugins { -@@ -12,6 +14,14 @@ plugins { +@@ -12,6 +14,11 @@ plugins { repositories { maven("https://libraries.minecraft.net/") + // Paper start -+ maven("https://maven.quiltmc.org/repository/release/") { -+ mavenContent { -+ releasesOnly() -+ includeModule("org.quiltmc", "tiny-mappings-parser") -+ } ++ maven("https://maven.fabricmc.net/") { ++ mavenContent { includeModule("net.fabricmc", "mapping-io") } + } + // Paper end } dependencies { -@@ -46,6 +56,8 @@ dependencies { +@@ -46,6 +53,8 @@ dependencies { implementation("io.netty:netty-all:4.1.65.Final") // Paper -+ implementation("org.quiltmc:tiny-mappings-parser:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation ++ implementation("net.fabricmc:mapping-io:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + testImplementation("junit:junit:4.13.1") testImplementation("org.hamcrest:hamcrest-library:1.3") } -@@ -139,6 +151,45 @@ tasks.shadowJar { +@@ -95,6 +104,7 @@ relocation { + + relocate("org.jline:jline-terminal-jansi", "org.jline" to cb("jline")) + sequenceOf( ++ "net.fabricmc:mapping-io" to "net.fabricmc.mappingio", // Paper + "commons-codec:commons-codec" to "org.apache.commons.codec", + "commons-io:commons-io" to "org.apache.commons.io", + //"it.unimi.dsi:fastutil" to "it.unimi", // Paper - don't relocate fastutil +@@ -139,6 +149,45 @@ tasks.shadowJar { transform(ModifiedLog4j2PluginsCacheFileTransformer::class.java) } @@ -162,26 +167,26 @@ index 0000000000000000000000000000000000000000..c701ef3c287f62aa0ebfbdbd6da6ed82 +} diff --git a/src/main/java/io/papermc/paper/util/ObfHelper.java b/src/main/java/io/papermc/paper/util/ObfHelper.java new file mode 100644 -index 0000000000000000000000000000000000000000..5ee513c0e2436e6afa3432f9f835d4028b37aed0 +index 0000000000000000000000000000000000000000..b8b17d046f836c8652ab094db00ab1af84971b2c --- /dev/null +++ b/src/main/java/io/papermc/paper/util/ObfHelper.java -@@ -0,0 +1,141 @@ +@@ -0,0 +1,146 @@ +package io.papermc.paper.util; + -+import com.google.common.base.Charsets; -+import com.google.common.collect.ImmutableMap; -+import com.google.common.collect.ImmutableSet; -+import com.mojang.datafixers.util.Pair; -+import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; ++import java.nio.charset.StandardCharsets; ++import java.util.HashMap; ++import java.util.HashSet; +import java.util.Map; +import java.util.Set; -+import net.fabricmc.mapping.tree.ClassDef; -+import net.fabricmc.mapping.tree.MethodDef; -+import net.fabricmc.mapping.tree.TinyMappingFactory; -+import net.fabricmc.mapping.tree.TinyTree; ++import java.util.function.Function; ++import java.util.stream.Collectors; ++import net.fabricmc.mappingio.MappingReader; ++import net.fabricmc.mappingio.format.MappingFormat; ++import net.fabricmc.mappingio.tree.MappingTree; ++import net.fabricmc.mappingio.tree.MemoryMappingTree; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.framework.qual.DefaultQualifier; @@ -199,8 +204,8 @@ index 0000000000000000000000000000000000000000..5ee513c0e2436e6afa3432f9f835d402 + ObfHelper() { + final @Nullable Set maps = loadMappingsIfPresent(); + if (maps != null) { -+ this.mappingsByObfName = maps.stream().collect(ImmutableMap.toImmutableMap(ClassMapping::obfName, map -> map)); -+ this.mappingsByMojangName = maps.stream().collect(ImmutableMap.toImmutableMap(ClassMapping::mojangName, map -> map)); ++ this.mappingsByObfName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::obfName, map -> map)); ++ this.mappingsByMojangName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::mojangName, map -> map)); + } else { + this.mappingsByObfName = null; + this.mappingsByMojangName = null; @@ -260,34 +265,33 @@ index 0000000000000000000000000000000000000000..5ee513c0e2436e6afa3432f9f835d402 + if (mappingsInputStream == null) { + return null; + } -+ final TinyTree tree = TinyMappingFactory.loadWithDetection(new BufferedReader(new InputStreamReader(mappingsInputStream, Charsets.UTF_8))); -+ final var builder = ImmutableSet.builder(); ++ final MemoryMappingTree tree = new MemoryMappingTree(); ++ MappingReader.read(new InputStreamReader(mappingsInputStream, StandardCharsets.UTF_8), MappingFormat.TINY_2, tree); ++ final Set classes = new HashSet<>(); + -+ for (final ClassDef classDef : tree.getClasses()) { -+ final String obfClassName = classDef.getName(SPIGOT_NAMESPACE).replace('/', '.'); -+ final var methodMappings = ImmutableMap., MethodMapping>builder(); ++ final StringPool pool = new StringPool(); ++ for (final MappingTree.ClassMapping cls : tree.getClasses()) { ++ final Map methods = new HashMap<>(); + -+ for (final MethodDef methodDef : classDef.getMethods()) { -+ final MethodMapping method = new MethodMapping( -+ methodDef.getName(SPIGOT_NAMESPACE), -+ methodDef.getName(MOJANG_PLUS_YARN_NAMESPACE), -+ methodDef.getDescriptor(SPIGOT_NAMESPACE) -+ ); -+ methodMappings.put( -+ new Pair<>(method.obfName(), method.descriptor()), -+ method ++ for (final MappingTree.MethodMapping methodMapping : cls.getMethods()) { ++ methods.put( ++ pool.string(methodKey( ++ methodMapping.getName(SPIGOT_NAMESPACE), ++ methodMapping.getDesc(SPIGOT_NAMESPACE) ++ )), ++ pool.string(methodMapping.getName(MOJANG_PLUS_YARN_NAMESPACE)) + ); + } + + final ClassMapping map = new ClassMapping( -+ obfClassName, -+ classDef.getName(MOJANG_PLUS_YARN_NAMESPACE).replace('/', '.'), -+ methodMappings.build() ++ cls.getName(SPIGOT_NAMESPACE).replace('/', '.'), ++ cls.getName(MOJANG_PLUS_YARN_NAMESPACE).replace('/', '.'), ++ Map.copyOf(methods) + ); -+ builder.add(map); ++ classes.add(map); + } + -+ return builder.build(); ++ return Set.copyOf(classes); + } catch (final IOException ex) { + System.err.println("Failed to load mappings for stacktrace deobfuscation."); + ex.printStackTrace(); @@ -295,28 +299,33 @@ index 0000000000000000000000000000000000000000..5ee513c0e2436e6afa3432f9f835d402 + } + } + ++ public static String methodKey(final String obfName, final String obfDescriptor) { ++ return obfName + obfDescriptor; ++ } ++ ++ private static final class StringPool { ++ private final Map pool = new HashMap<>(); ++ ++ public String string(final String string) { ++ return this.pool.computeIfAbsent(string, Function.identity()); ++ } ++ } ++ + public record ClassMapping( + String obfName, + String mojangName, -+ Map, MethodMapping> methodMappings -+ ) {} -+ -+ public record MethodMapping( -+ String obfName, -+ String mojangName, -+ String descriptor ++ Map methodsByObf + ) {} +} diff --git a/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java new file mode 100644 -index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aaa4465244 +index 0000000000000000000000000000000000000000..5019d22fa2ab8d72fac5ffcf5dc2a3715ae63760 --- /dev/null +++ b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java -@@ -0,0 +1,156 @@ +@@ -0,0 +1,153 @@ +package io.papermc.paper.util; + +import com.destroystokyo.paper.PaperConfig; -+import com.mojang.datafixers.util.Pair; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import java.io.IOException; @@ -337,9 +346,9 @@ index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aa +public enum StacktraceDeobfuscator { + INSTANCE; + -+ private final Map, Map, IntList>> lineMapCache = Collections.synchronizedMap(new LinkedHashMap<>(128, 0.75f, true) { ++ private final Map, Map> lineMapCache = Collections.synchronizedMap(new LinkedHashMap<>(128, 0.75f, true) { + @Override -+ protected boolean removeEldestEntry(final Map.Entry, Map, IntList>> eldest) { ++ protected boolean removeEldestEntry(final Map.Entry, Map> eldest) { + return this.size() > 127; + } + }); @@ -387,17 +396,15 @@ index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aa + } catch (final ClassNotFoundException ex) { + throw new RuntimeException(ex); + } -+ final @Nullable Pair nameDescriptorPair = this.determineMethodForLine(clazz, element.getLineNumber()); -+ final ObfHelper.@Nullable MethodMapping methodMapping = nameDescriptorPair == null -+ ? null -+ : classMapping.methodMappings().get(nameDescriptorPair); ++ final @Nullable String methodKey = this.determineMethodForLine(clazz, element.getLineNumber()); ++ final @Nullable String mappedMethodName = methodKey == null ? null : classMapping.methodsByObf().get(methodKey); + + result[i] = new StackTraceElement( + element.getClassLoaderName(), + element.getModuleName(), + element.getModuleVersion(), + classMapping.mojangName(), -+ methodMapping != null ? methodMapping.mojangName() : methodName, ++ mappedMethodName != null ? mappedMethodName : methodName, + sourceFileName(classMapping.mojangName()), + element.getLineNumber() + ); @@ -405,15 +412,15 @@ index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aa + return result; + } + -+ private @Nullable Pair determineMethodForLine(final Class clazz, final int lineNumber) { -+ final Map, IntList> lineMap = this.lineMapCache.computeIfAbsent(clazz, StacktraceDeobfuscator::buildLineMap); ++ private @Nullable String determineMethodForLine(final Class clazz, final int lineNumber) { ++ final Map lineMap = this.lineMapCache.computeIfAbsent(clazz, StacktraceDeobfuscator::buildLineMap); + for (final var entry : lineMap.entrySet()) { -+ final Pair pair = entry.getKey(); ++ final String methodKey = entry.getKey(); + final IntList lines = entry.getValue(); + for (int i = 0, linesSize = lines.size(); i < linesSize; i++) { + final int num = lines.getInt(i); + if (num == lineNumber) { -+ return pair; ++ return methodKey; + } + } + } @@ -429,8 +436,8 @@ index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aa + return rootClassName + ".java"; + } + -+ private static Map, IntList> buildLineMap(final Class key) { -+ final Map, IntList> lineMap = new HashMap<>(); ++ private static Map buildLineMap(final Class key) { ++ final Map lineMap = new HashMap<>(); + final class LineCollectingMethodVisitor extends MethodVisitor { + private final IntList lines = new IntArrayList(); + private final String name; @@ -451,7 +458,7 @@ index 0000000000000000000000000000000000000000..f3099d3cfa4ea8ad56b1abdd0e7b77aa + @Override + public void visitEnd() { + super.visitEnd(); -+ lineMap.put(new Pair<>(this.name, this.descriptor), this.lines); ++ lineMap.put(ObfHelper.methodKey(this.name, this.descriptor), this.lines); + } + } + final ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { diff --git a/patches/server/0430-Implement-Mob-Goal-API.patch b/patches/server/0430-Implement-Mob-Goal-API.patch index ed06d89048..9c72336d8c 100644 --- a/patches/server/0430-Implement-Mob-Goal-API.patch +++ b/patches/server/0430-Implement-Mob-Goal-API.patch @@ -5,12 +5,12 @@ Subject: [PATCH] Implement Mob Goal API diff --git a/build.gradle.kts b/build.gradle.kts -index 7a7a6a1a1bdfecb619dd6ba6f6865eb0bb74d0d9..58c6e444ee8c8cabf8449b0e3b77970edf1d499b 100644 +index f802d9e803b406cb9b5b4bce7de6858eba0fae7e..27f8a99f35a22f334e8023c1c9048e3a88a63d6f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts -@@ -58,6 +58,7 @@ dependencies { +@@ -55,6 +55,7 @@ dependencies { - implementation("org.quiltmc:tiny-mappings-parser:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + implementation("net.fabricmc:mapping-io:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + testImplementation("io.github.classgraph:classgraph:4.8.47") // Paper - mob goal test testImplementation("junit:junit:4.13.1") diff --git a/patches/server/0623-Add-support-for-hex-color-codes-in-console.patch b/patches/server/0623-Add-support-for-hex-color-codes-in-console.patch index cf21b606ab..c56b164a8e 100644 --- a/patches/server/0623-Add-support-for-hex-color-codes-in-console.patch +++ b/patches/server/0623-Add-support-for-hex-color-codes-in-console.patch @@ -6,10 +6,10 @@ Subject: [PATCH] Add support for hex color codes in console Converts upstream's hex color code legacy format into actual hex color codes in the console. diff --git a/build.gradle.kts b/build.gradle.kts -index de4f525573f222426a693b4d28e15caf4a87a210..149607e8c080b79a20d204382db69fde58c6b513 100644 +index 57edac376bcbd056fb8fc7f0a433e946bce4ff2e..f3db86346c5e7bf446707a746f3094f7f27e09c4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts -@@ -37,6 +37,7 @@ dependencies { +@@ -34,6 +34,7 @@ dependencies { Scanning takes about 1-2 seconds so adding this speeds up the server start. */ implementation("org.apache.logging.log4j:log4j-core:2.14.1") // Paper - implementation diff --git a/patches/server/0710-Add-git-branch-and-commit-to-manifest.patch b/patches/server/0710-Add-git-branch-and-commit-to-manifest.patch index 9f55d195ae..50009c0457 100644 --- a/patches/server/0710-Add-git-branch-and-commit-to-manifest.patch +++ b/patches/server/0710-Add-git-branch-and-commit-to-manifest.patch @@ -5,10 +5,10 @@ Subject: [PATCH] Add git branch and commit to manifest diff --git a/build.gradle.kts b/build.gradle.kts -index 3dc4631c9a595c50c0f281897c6953d5a505a8e3..c42f192c5a5557ba4b541e9bd1769b353115d7e8 100644 +index 1e4e3c6fbac820fa03e2a6f5d76c008d5879d2e6..c7ebad49e04c36c27341cabc4a024f170d762f3f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts -@@ -72,6 +72,7 @@ tasks.jar { +@@ -69,6 +69,7 @@ tasks.jar { val gitHash = git("rev-parse", "--short=7", "HEAD").getText().trim() val implementationVersion = System.getenv("BUILD_NUMBER") ?: "\"$gitHash\"" val date = git("show", "-s", "--format=%ci", gitHash).getText().trim() // Paper @@ -16,7 +16,7 @@ index 3dc4631c9a595c50c0f281897c6953d5a505a8e3..c42f192c5a5557ba4b541e9bd1769b35 attributes( "Main-Class" to "org.bukkit.craftbukkit.Main", "Implementation-Title" to "CraftBukkit", -@@ -81,6 +82,8 @@ tasks.jar { +@@ -78,6 +79,8 @@ tasks.jar { "Specification-Version" to project.version, "Specification-Vendor" to "Bukkit Team", "Multi-Release" to "true", // Paper diff --git a/patches/server/0797-Use-Velocity-compression-and-cipher-natives.patch b/patches/server/0797-Use-Velocity-compression-and-cipher-natives.patch index 9b8effa915..2d7569820c 100644 --- a/patches/server/0797-Use-Velocity-compression-and-cipher-natives.patch +++ b/patches/server/0797-Use-Velocity-compression-and-cipher-natives.patch @@ -5,13 +5,13 @@ Subject: [PATCH] Use Velocity compression and cipher natives diff --git a/build.gradle.kts b/build.gradle.kts -index c42f192c5a5557ba4b541e9bd1769b353115d7e8..d78267e02a705ffa9083815da0df042d3a184837 100644 +index c7ebad49e04c36c27341cabc4a024f170d762f3f..f46f11c32560da6f4955284d93da3dd7e22555a3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts -@@ -58,6 +58,7 @@ dependencies { +@@ -55,6 +55,7 @@ dependencies { implementation("io.netty:netty-all:4.1.65.Final") // Paper - implementation("org.quiltmc:tiny-mappings-parser:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + implementation("net.fabricmc:mapping-io:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + implementation("com.velocitypowered:velocity-native:1.1.0-SNAPSHOT") // Paper testImplementation("io.github.classgraph:classgraph:4.8.47") // Paper - mob goal test