Paper/patches/server/0400-Deobfuscate-stacktraces-in-log-messages-crash-report.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Jason Penilla <11360596+jpenilla@users.noreply.github.com>
Date: Sun, 20 Jun 2021 18:19:09 -0700
Subject: [PATCH] Deobfuscate stacktraces in log messages, crash reports, and
 etc.


diff --git a/build.gradle.kts b/build.gradle.kts
index e260462933a9f7065b2360e6bf9e4ee56069a705..a1a8c4778742584125d6084fa761b1bc86f6a842 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -1,4 +1,6 @@
+import io.papermc.paperweight.tasks.BaseTask
 import io.papermc.paperweight.util.*
+import java.nio.file.Files
 
 plugins {
     java
@@ -19,12 +21,14 @@ dependencies {
           Scanning takes about 1-2 seconds so adding this speeds up the server start.
      */
     implementation("org.apache.logging.log4j:log4j-core:2.14.1") // Paper - implementation
+    annotationProcessor("org.apache.logging.log4j:log4j-core:2.14.1") // Paper - Needed to generate meta for our Log4j plugins
     // Paper end
     implementation("org.apache.logging.log4j:log4j-iostreams:2.17.1") // Paper
     implementation("org.ow2.asm:asm:9.3")
     implementation("org.ow2.asm:asm-commons:9.3") // Paper - ASM event executor generation
     implementation("org.spongepowered:configurate-yaml:4.1.2") // Paper - config files
     implementation("commons-lang:commons-lang:2.6")
+    implementation("net.fabricmc:mapping-io:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation
     runtimeOnly("org.xerial:sqlite-jdbc:3.36.0.3")
     runtimeOnly("mysql:mysql-connector-java:8.0.29")
     runtimeOnly("com.lmax:disruptor:3.4.4") // Paper
@@ -104,6 +108,45 @@ tasks.check {
 }
 // Paper end
 
+// Paper start - include reobf mappings in jar for stacktrace deobfuscation
+abstract class IncludeMappings : BaseTask() {
+    @get:InputFile
+    abstract val inputJar: RegularFileProperty
+
+    @get:InputFile
+    abstract val mappings: RegularFileProperty
+
+    @get:OutputFile
+    abstract val outputJar: RegularFileProperty
+
+    override fun init() {
+        super.init()
+        outputJar.convention(defaultOutput())
+    }
+
+    @TaskAction
+    private fun addMappings() {
+        outputJar.get().asFile.parentFile.mkdirs()
+        inputJar.get().asFile.copyTo(outputJar.get().asFile, overwrite = true)
+        outputJar.get().path.openZip().use { fs ->
+            val dir = fs.getPath("META-INF/mappings/")
+            Files.createDirectories(dir)
+            val target = dir.resolve("reobf.tiny")
+            Files.copy(mappings.path, target)
+        }
+    }
+}
+
+val includeMappings = tasks.register<IncludeMappings>("includeMappings") {
+    inputJar.set(tasks.fixJarForReobf.flatMap { it.outputJar })
+    mappings.set(tasks.reobfJar.flatMap { it.mappingsFile })
+}
+
+tasks.reobfJar {
+    inputJar.set(includeMappings.flatMap { it.outputJar })
+}
+// Paper end - include reobf mappings in jar for stacktrace deobfuscation
+
 tasks.test {
     exclude("org/bukkit/craftbukkit/inventory/ItemStack*Test.class")
 }
diff --git a/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java
index 0bb4aaa546939b67a5d22865190f30478a9337c1..d3e619655382e50e9ac9323ed942502d85c9599c 100644
--- a/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java
+++ b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java
@@ -91,7 +91,7 @@ public class SyncLoadFinder {
 
                 final JsonArray traces = new JsonArray();
 
-                for (StackTraceElement element : pair.getFirst().stacktrace) {
+                for (StackTraceElement element : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(pair.getFirst().stacktrace)) {
                     traces.add(String.valueOf(element));
                 }
 
diff --git a/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java b/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java
new file mode 100644
index 0000000000000000000000000000000000000000..c701ef3c287f62aa0ebfbdbd6da6ed82a1c7793c
--- /dev/null
+++ b/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java
@@ -0,0 +1,38 @@
+package io.papermc.paper.logging;
+
+import io.papermc.paper.util.StacktraceDeobfuscator;
+import org.apache.logging.log4j.core.Core;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.impl.Log4jLogEvent;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+@Plugin(
+    name = "StacktraceDeobfuscatingRewritePolicy",
+    category = Core.CATEGORY_NAME,
+    elementType = "rewritePolicy",
+    printObject = true
+)
+public final class StacktraceDeobfuscatingRewritePolicy implements RewritePolicy {
+    private StacktraceDeobfuscatingRewritePolicy() {
+    }
+
+    @Override
+    public @NonNull LogEvent rewrite(final @NonNull LogEvent rewrite) {
+        final Throwable thrown = rewrite.getThrown();
+        if (thrown != null) {
+            StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(thrown);
+            return new Log4jLogEvent.Builder(rewrite)
+                .setThrownProxy(null)
+                .build();
+        }
+        return rewrite;
+    }
+
+    @PluginFactory
+    public static @NonNull StacktraceDeobfuscatingRewritePolicy createPolicy() {
+        return new StacktraceDeobfuscatingRewritePolicy();
+    }
+}
diff --git a/src/main/java/io/papermc/paper/util/ObfHelper.java b/src/main/java/io/papermc/paper/util/ObfHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..b8b17d046f836c8652ab094db00ab1af84971b2c
--- /dev/null
+++ b/src/main/java/io/papermc/paper/util/ObfHelper.java
@@ -0,0 +1,146 @@
+package io.papermc.paper.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import net.fabricmc.mappingio.MappingReader;
+import net.fabricmc.mappingio.format.MappingFormat;
+import net.fabricmc.mappingio.tree.MappingTree;
+import net.fabricmc.mappingio.tree.MemoryMappingTree;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.checkerframework.framework.qual.DefaultQualifier;
+
+@DefaultQualifier(NonNull.class)
+public enum ObfHelper {
+    INSTANCE;
+
+    public static final String MOJANG_PLUS_YARN_NAMESPACE = "mojang+yarn";
+    public static final String SPIGOT_NAMESPACE = "spigot";
+
+    private final @Nullable Map<String, ClassMapping> mappingsByObfName;
+    private final @Nullable Map<String, ClassMapping> mappingsByMojangName;
+
+    ObfHelper() {
+        final @Nullable Set<ClassMapping> maps = loadMappingsIfPresent();
+        if (maps != null) {
+            this.mappingsByObfName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::obfName, map -> map));
+            this.mappingsByMojangName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::mojangName, map -> map));
+        } else {
+            this.mappingsByObfName = null;
+            this.mappingsByMojangName = null;
+        }
+    }
+
+    public @Nullable Map<String, ClassMapping> mappingsByObfName() {
+        return this.mappingsByObfName;
+    }
+
+    public @Nullable Map<String, ClassMapping> mappingsByMojangName() {
+        return this.mappingsByMojangName;
+    }
+
+    /**
+     * Attempts to get the obf name for a given class by its Mojang name. Will
+     * return the input string if mappings are not present.
+     *
+     * @param fullyQualifiedMojangName fully qualified class name (dotted)
+     * @return mapped or original fully qualified (dotted) class name
+     */
+    public String reobfClassName(final String fullyQualifiedMojangName) {
+        if (this.mappingsByMojangName == null) {
+            return fullyQualifiedMojangName;
+        }
+
+        final ClassMapping map = this.mappingsByMojangName.get(fullyQualifiedMojangName);
+        if (map == null) {
+            return fullyQualifiedMojangName;
+        }
+
+        return map.obfName();
+    }
+
+    /**
+     * Attempts to get the Mojang name for a given class by its obf name. Will
+     * return the input string if mappings are not present.
+     *
+     * @param fullyQualifiedObfName fully qualified class name (dotted)
+     * @return mapped or original fully qualified (dotted) class name
+     */
+    public String deobfClassName(final String fullyQualifiedObfName) {
+        if (this.mappingsByObfName == null) {
+            return fullyQualifiedObfName;
+        }
+
+        final ClassMapping map = this.mappingsByObfName.get(fullyQualifiedObfName);
+        if (map == null) {
+            return fullyQualifiedObfName;
+        }
+
+        return map.mojangName();
+    }
+
+    private static @Nullable Set<ClassMapping> loadMappingsIfPresent() {
+        try (final @Nullable InputStream mappingsInputStream = ObfHelper.class.getClassLoader().getResourceAsStream("META-INF/mappings/reobf.tiny")) {
+            if (mappingsInputStream == null) {
+                return null;
+            }
+            final MemoryMappingTree tree = new MemoryMappingTree();
+            MappingReader.read(new InputStreamReader(mappingsInputStream, StandardCharsets.UTF_8), MappingFormat.TINY_2, tree);
+            final Set<ClassMapping> classes = new HashSet<>();
+
+            final StringPool pool = new StringPool();
+            for (final MappingTree.ClassMapping cls : tree.getClasses()) {
+                final Map<String, String> methods = new HashMap<>();
+
+                for (final MappingTree.MethodMapping methodMapping : cls.getMethods()) {
+                    methods.put(
+                        pool.string(methodKey(
+                            methodMapping.getName(SPIGOT_NAMESPACE),
+                            methodMapping.getDesc(SPIGOT_NAMESPACE)
+                        )),
+                        pool.string(methodMapping.getName(MOJANG_PLUS_YARN_NAMESPACE))
+                    );
+                }
+
+                final ClassMapping map = new ClassMapping(
+                    cls.getName(SPIGOT_NAMESPACE).replace('/', '.'),
+                    cls.getName(MOJANG_PLUS_YARN_NAMESPACE).replace('/', '.'),
+                    Map.copyOf(methods)
+                );
+                classes.add(map);
+            }
+
+            return Set.copyOf(classes);
+        } catch (final IOException ex) {
+            System.err.println("Failed to load mappings for stacktrace deobfuscation.");
+            ex.printStackTrace();
+            return null;
+        }
+    }
+
+    public static String methodKey(final String obfName, final String obfDescriptor) {
+        return obfName + obfDescriptor;
+    }
+
+    private static final class StringPool {
+        private final Map<String, String> pool = new HashMap<>();
+
+        public String string(final String string) {
+            return this.pool.computeIfAbsent(string, Function.identity());
+        }
+    }
+
+    public record ClassMapping(
+        String obfName,
+        String mojangName,
+        Map<String, String> methodsByObf
+    ) {}
+}
diff --git a/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java
new file mode 100644
index 0000000000000000000000000000000000000000..eb910d4abf91488fa7cf1f5d47e0ee916c47f512
--- /dev/null
+++ b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java
@@ -0,0 +1,163 @@
+package io.papermc.paper.util;
+
+import io.papermc.paper.configuration.GlobalConfiguration;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.checkerframework.framework.qual.DefaultQualifier;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+@DefaultQualifier(NonNull.class)
+public enum StacktraceDeobfuscator {
+    INSTANCE;
+
+    private final Map<Class<?>, Map<String, IntList>> lineMapCache = Collections.synchronizedMap(new LinkedHashMap<>(128, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(final Map.Entry<Class<?>, Map<String, IntList>> eldest) {
+            return this.size() > 127;
+        }
+    });
+
+    public void deobfuscateThrowable(final Throwable throwable) {
+        if (GlobalConfiguration.get() != null && !GlobalConfiguration.get().logging.deobfuscateStacktraces) { // handle null as true
+            return;
+        }
+
+        throwable.setStackTrace(this.deobfuscateStacktrace(throwable.getStackTrace()));
+        final Throwable cause = throwable.getCause();
+        if (cause != null) {
+            this.deobfuscateThrowable(cause);
+        }
+        for (final Throwable suppressed : throwable.getSuppressed()) {
+            this.deobfuscateThrowable(suppressed);
+        }
+    }
+
+    public StackTraceElement[] deobfuscateStacktrace(final StackTraceElement[] traceElements) {
+        if (GlobalConfiguration.get() != null && !GlobalConfiguration.get().logging.deobfuscateStacktraces) { // handle null as true
+            return traceElements;
+        }
+
+        final @Nullable Map<String, ObfHelper.ClassMapping> mappings = ObfHelper.INSTANCE.mappingsByObfName();
+        if (mappings == null || traceElements.length == 0) {
+            return traceElements;
+        }
+        final StackTraceElement[] result = new StackTraceElement[traceElements.length];
+        for (int i = 0; i < traceElements.length; i++) {
+            final StackTraceElement element = traceElements[i];
+
+            final String className = element.getClassName();
+            final String methodName = element.getMethodName();
+
+            final ObfHelper.ClassMapping classMapping = mappings.get(className);
+            if (classMapping == null) {
+                result[i] = element;
+                continue;
+            }
+
+            final Class<?> clazz;
+            try {
+                clazz = Class.forName(className);
+            } catch (final ClassNotFoundException ex) {
+                throw new RuntimeException(ex);
+            }
+            final @Nullable String methodKey = this.determineMethodForLine(clazz, element.getLineNumber());
+            final @Nullable String mappedMethodName = methodKey == null ? null : classMapping.methodsByObf().get(methodKey);
+
+            result[i] = new StackTraceElement(
+                element.getClassLoaderName(),
+                element.getModuleName(),
+                element.getModuleVersion(),
+                classMapping.mojangName(),
+                mappedMethodName != null ? mappedMethodName : methodName,
+                sourceFileName(classMapping.mojangName()),
+                element.getLineNumber()
+            );
+        }
+        return result;
+    }
+
+    private @Nullable String determineMethodForLine(final Class<?> clazz, final int lineNumber) {
+        final Map<String, IntList> lineMap = this.lineMapCache.computeIfAbsent(clazz, StacktraceDeobfuscator::buildLineMap);
+        for (final var entry : lineMap.entrySet()) {
+            final String methodKey = entry.getKey();
+            final IntList lines = entry.getValue();
+            for (int i = 0, linesSize = lines.size(); i < linesSize; i++) {
+                final int num = lines.getInt(i);
+                if (num == lineNumber) {
+                    return methodKey;
+                }
+            }
+        }
+        return null;
+    }
+
+    private static String sourceFileName(final String fullClassName) {
+        final int dot = fullClassName.lastIndexOf('.');
+        final String className = dot == -1
+            ? fullClassName
+            : fullClassName.substring(dot + 1);
+        final String rootClassName = className.split("\\$")[0];
+        return rootClassName + ".java";
+    }
+
+    private static Map<String, IntList> buildLineMap(final Class<?> key) {
+        final Map<String, IntList> lineMap = new HashMap<>();
+        final class LineCollectingMethodVisitor extends MethodVisitor {
+            private final IntList lines = new IntArrayList();
+            private final String name;
+            private final String descriptor;
+
+            LineCollectingMethodVisitor(String name, String descriptor) {
+                super(Opcodes.ASM9);
+                this.name = name;
+                this.descriptor = descriptor;
+            }
+
+            @Override
+            public void visitLineNumber(int line, Label start) {
+                super.visitLineNumber(line, start);
+                this.lines.add(line);
+            }
+
+            @Override
+            public void visitEnd() {
+                super.visitEnd();
+                lineMap.put(ObfHelper.methodKey(this.name, this.descriptor), this.lines);
+            }
+        }
+        final ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
+            @Override
+            public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+                return new LineCollectingMethodVisitor(name, descriptor);
+            }
+        };
+        try {
+            final @Nullable InputStream inputStream = StacktraceDeobfuscator.class.getClassLoader()
+                .getResourceAsStream(key.getName().replace('.', '/') + ".class");
+            if (inputStream == null) {
+                throw new IllegalStateException("Could not find class file: " + key.getName());
+            }
+            final byte[] classData;
+            try (inputStream) {
+                classData = inputStream.readAllBytes();
+            }
+            final ClassReader reader = new ClassReader(classData);
+            reader.accept(classVisitor, 0);
+        } catch (final IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        return lineMap;
+    }
+}
diff --git a/src/main/java/io/papermc/paper/util/TraceUtil.java b/src/main/java/io/papermc/paper/util/TraceUtil.java
index 2d5494d2813b773e60ddba6790b750a9a08f21f8..0b210bdf7c1f5962afbd44195af6f84f625635e3 100644
--- a/src/main/java/io/papermc/paper/util/TraceUtil.java
+++ b/src/main/java/io/papermc/paper/util/TraceUtil.java
@@ -6,13 +6,20 @@ public final class TraceUtil {
 
     public static void dumpTraceForThread(Thread thread, String reason) {
         Bukkit.getLogger().warning(thread.getName() + ": " + reason);
-        StackTraceElement[] trace = thread.getStackTrace();
+        StackTraceElement[] trace = StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(thread.getStackTrace());
         for (StackTraceElement traceElement : trace) {
             Bukkit.getLogger().warning("\tat " + traceElement);
         }
     }
 
     public static void dumpTraceForThread(String reason) {
-        new Throwable(reason).printStackTrace();
+        final Throwable throwable = new Throwable(reason);
+        StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(throwable);
+        throwable.printStackTrace();
+    }
+
+    public static void printStackTrace(Throwable thr) {
+        StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(thr);
+        thr.printStackTrace();
     }
 }
diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
index 30a58229aa6dac5039511d0c0df5f2912ea7de9f..abe37c7c3c6f5ab73afd738ec78f06d7e4d2ed96 100644
--- a/src/main/java/net/minecraft/CrashReport.java
+++ b/src/main/java/net/minecraft/CrashReport.java
@@ -32,6 +32,7 @@ public class CrashReport {
     private final SystemReport systemReport = new SystemReport();
 
     public CrashReport(String message, Throwable cause) {
+        io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(cause); // Paper
         this.title = message;
         this.exception = cause;
         this.systemReport.setDetail("CraftBukkit Information", new org.bukkit.craftbukkit.CraftCrashReport()); // CraftBukkit
diff --git a/src/main/java/net/minecraft/CrashReportCategory.java b/src/main/java/net/minecraft/CrashReportCategory.java
index f114d5dab86aa2cdd59c78406c9d82f9caededca..99fa9f1952ee7ed79b223ff210a658e4b119b3e4 100644
--- a/src/main/java/net/minecraft/CrashReportCategory.java
+++ b/src/main/java/net/minecraft/CrashReportCategory.java
@@ -104,6 +104,7 @@ public class CrashReportCategory {
         } else {
             this.stackTrace = new StackTraceElement[stackTraceElements.length - 3 - ignoredCallCount];
             System.arraycopy(stackTraceElements, 3 + ignoredCallCount, this.stackTrace, 0, this.stackTrace.length);
+            this.stackTrace = io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(this.stackTrace); // Paper
             return this.stackTrace.length;
         }
     }
diff --git a/src/main/java/net/minecraft/network/Connection.java b/src/main/java/net/minecraft/network/Connection.java
index 593ea68037b467797aeeaee331a0349f7d57d800..bbedcdb71a326b3286805d0081e71c54a4312622 100644
--- a/src/main/java/net/minecraft/network/Connection.java
+++ b/src/main/java/net/minecraft/network/Connection.java
@@ -62,13 +62,13 @@ public class Connection extends SimpleChannelInboundHandler<Packet<?>> {
     });
     public static final AttributeKey<ConnectionProtocol> ATTRIBUTE_PROTOCOL = AttributeKey.valueOf("protocol");
     public static final LazyLoadedValue<NioEventLoopGroup> NETWORK_WORKER_GROUP = new LazyLoadedValue<>(() -> {
-        return new NioEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Client IO #%d").setDaemon(true).build());
+        return new NioEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Client IO #%d").setDaemon(true).setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(LOGGER)).build()); // Paper
     });
     public static final LazyLoadedValue<EpollEventLoopGroup> NETWORK_EPOLL_WORKER_GROUP = new LazyLoadedValue<>(() -> {
-        return new EpollEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Epoll Client IO #%d").setDaemon(true).build());
+        return new EpollEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Epoll Client IO #%d").setDaemon(true).setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(LOGGER)).build()); // Paper
     });
     public static final LazyLoadedValue<DefaultEventLoopGroup> LOCAL_WORKER_GROUP = new LazyLoadedValue<>(() -> {
-        return new DefaultEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Local Client IO #%d").setDaemon(true).build());
+        return new DefaultEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Local Client IO #%d").setDaemon(true).setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(LOGGER)).build()); // Paper
     });
     private final PacketFlow receiving;
     private final Queue<Connection.PacketHolder> queue = Queues.newConcurrentLinkedQueue();
@@ -193,7 +193,7 @@ public class Connection extends SimpleChannelInboundHandler<Packet<?>> {
 
             }
         }
-        if (net.minecraft.server.MinecraftServer.getServer().isDebugging()) throwable.printStackTrace(); // Spigot
+        if (net.minecraft.server.MinecraftServer.getServer().isDebugging()) io.papermc.paper.util.TraceUtil.printStackTrace(throwable); // Spigot // Paper
     }
 
     protected void channelRead0(ChannelHandlerContext channelhandlercontext, Packet<?> packet) {
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
index 6fba140877e7369cdb7933ec225572c6b153e3a8..e3a62579067209c447f2fdcb76b2a11e489a376b 100644
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
@@ -200,6 +200,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
         org.spigotmc.SpigotConfig.registerCommands();
         // Spigot end
         // Paper start
+        io.papermc.paper.util.ObfHelper.INSTANCE.getClass(); // Paper - load mappings for stacktrace deobf and etc.
         paperConfigurations.initializeGlobalConfiguration();
         paperConfigurations.initializeWorldDefaultsConfiguration();
         org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash);
diff --git a/src/main/java/net/minecraft/server/level/ServerLevel.java b/src/main/java/net/minecraft/server/level/ServerLevel.java
index 652d30ce735aa265db848ca73a48d7b0b143103b..35439d857801d56494cc457d116fbcbb12561bcd 100644
--- a/src/main/java/net/minecraft/server/level/ServerLevel.java
+++ b/src/main/java/net/minecraft/server/level/ServerLevel.java
@@ -218,7 +218,9 @@ public class ServerLevel extends Level implements WorldGenLevel {
     public final UUID uuid;
     public boolean hasPhysicsEvent = true; // Paper
     public static Throwable getAddToWorldStackTrace(Entity entity) {
-        return new Throwable(entity + " Added to world at " + new java.util.Date());
+        final Throwable thr = new Throwable(entity + " Added to world at " + new java.util.Date());
+        io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(thr);
+        return thr;
     }
 
     @Override public LevelChunk getChunkIfLoaded(int x, int z) { // Paper - this was added in world too but keeping here for NMS ABI
@@ -1289,7 +1291,7 @@ public class ServerLevel extends Level implements WorldGenLevel {
         if (entity.isRemoved()) {
             // Paper start
             if (DEBUG_ENTITIES) {
-                new Throwable("Tried to add entity " + entity + " but it was marked as removed already").printStackTrace(); // CraftBukkit
+                io.papermc.paper.util.TraceUtil.dumpTraceForThread("Tried to add entity " + entity + " but it was marked as removed already"); // CraftBukkit
                 getAddToWorldStackTrace(entity).printStackTrace();
             }
             // Paper end
diff --git a/src/main/java/net/minecraft/server/network/ServerConnectionListener.java b/src/main/java/net/minecraft/server/network/ServerConnectionListener.java
index a24ef433d0c9d06b86fd612978cfd6d877036791..1b38326c9a709536dc4cccf9af93aede98a1a782 100644
--- a/src/main/java/net/minecraft/server/network/ServerConnectionListener.java
+++ b/src/main/java/net/minecraft/server/network/ServerConnectionListener.java
@@ -54,10 +54,10 @@ public class ServerConnectionListener {
 
     private static final Logger LOGGER = LogUtils.getLogger();
     public static final LazyLoadedValue<NioEventLoopGroup> SERVER_EVENT_GROUP = new LazyLoadedValue<>(() -> {
-        return new NioEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Server IO #%d").setDaemon(true).build());
+        return new NioEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Server IO #%d").setDaemon(true).setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(LOGGER)).build()); // Paper
     });
     public static final LazyLoadedValue<EpollEventLoopGroup> SERVER_EPOLL_EVENT_GROUP = new LazyLoadedValue<>(() -> {
-        return new EpollEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Epoll Server IO #%d").setDaemon(true).build());
+        return new EpollEventLoopGroup(0, (new ThreadFactoryBuilder()).setNameFormat("Netty Epoll Server IO #%d").setDaemon(true).setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(LOGGER)).build()); // Paper
     });
     final MinecraftServer server;
     public volatile boolean running;
diff --git a/src/main/java/net/minecraft/server/network/ServerLoginPacketListenerImpl.java b/src/main/java/net/minecraft/server/network/ServerLoginPacketListenerImpl.java
index 930da83ff754bab6b6b6fff651df417baa76c46f..c3027735725c68eeb1b42a42ebb8646f3a35a518 100644
--- a/src/main/java/net/minecraft/server/network/ServerLoginPacketListenerImpl.java
+++ b/src/main/java/net/minecraft/server/network/ServerLoginPacketListenerImpl.java
@@ -204,7 +204,7 @@ public class ServerLoginPacketListenerImpl implements TickablePacketListener, Se
                 MutableComponent ichatmutablecomponent = Component.translatable("multiplayer.disconnect.invalid_player_data");
                 // Paper start
                 if (MinecraftServer.getServer().isDebugging()) {
-                    exception.printStackTrace();
+                    io.papermc.paper.util.TraceUtil.printStackTrace(exception);
                 }
                 // Paper end
 
diff --git a/src/main/java/net/minecraft/server/players/OldUsersConverter.java b/src/main/java/net/minecraft/server/players/OldUsersConverter.java
index 6599f874d9f97e9ef4862039ecad7277bbc5fd91..7edd4b88eb0476f0630630bc4681e859bd145b2b 100644
--- a/src/main/java/net/minecraft/server/players/OldUsersConverter.java
+++ b/src/main/java/net/minecraft/server/players/OldUsersConverter.java
@@ -364,7 +364,7 @@ public class OldUsersConverter {
                         try {
                             root = NbtIo.readCompressed(new java.io.FileInputStream(file5));
                         } catch (Exception exception) {
-                            exception.printStackTrace();
+                            io.papermc.paper.util.TraceUtil.printStackTrace(exception); // Paper
                             ServerInternalException.reportInternalException(exception); // Paper
                         }
 
@@ -378,7 +378,7 @@ public class OldUsersConverter {
                             try {
                                 NbtIo.writeCompressed(root, new java.io.FileOutputStream(file2));
                             } catch (Exception exception) {
-                                exception.printStackTrace();
+                                io.papermc.paper.util.TraceUtil.printStackTrace(exception); // Paper
                                 ServerInternalException.reportInternalException(exception); // Paper
                             }
                        }
diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
index 7e1f0c9f760d6f5cfb0138542252c8469534e152..623046a5a0ba36619d43d8e55cfee3c06d09b62b 100644
--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
@@ -640,7 +640,7 @@ public class LevelChunk extends ChunkAccess {
                     + " (" + getBlockState(blockposition) + ") where there was no entity tile!\n" +
                     "Chunk coordinates: " + (this.chunkPos.x * 16) + "," + (this.chunkPos.z * 16) +
                     "\nWorld: " + level.getLevel().dimension().location());
-            e.printStackTrace();
+            io.papermc.paper.util.TraceUtil.printStackTrace(e);
             ServerInternalException.reportInternalException(e);
             // Paper end
             // CraftBukkit end
diff --git a/src/main/java/org/bukkit/craftbukkit/scheduler/CraftAsyncScheduler.java b/src/main/java/org/bukkit/craftbukkit/scheduler/CraftAsyncScheduler.java
index 3c1992e212a6d6f1db4d5b807b38d71913619fc0..9c1aff17aabd062640e3f451a2ef8c50a7c62f10 100644
--- a/src/main/java/org/bukkit/craftbukkit/scheduler/CraftAsyncScheduler.java
+++ b/src/main/java/org/bukkit/craftbukkit/scheduler/CraftAsyncScheduler.java
@@ -40,9 +40,9 @@ public class CraftAsyncScheduler extends CraftScheduler {
 
     private final ThreadPoolExecutor executor = new ThreadPoolExecutor(
             4, Integer.MAX_VALUE,30L, TimeUnit.SECONDS, new SynchronousQueue<>(),
-            new ThreadFactoryBuilder().setNameFormat("Craft Scheduler Thread - %1$d").build());
+            new ThreadFactoryBuilder().setNameFormat("Craft Scheduler Thread - %1$d").setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(net.minecraft.server.MinecraftServer.LOGGER)).build()); // Paper
     private final Executor management = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
-            .setNameFormat("Craft Async Scheduler Management Thread").build());
+            .setNameFormat("Craft Async Scheduler Management Thread").setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(net.minecraft.server.MinecraftServer.LOGGER)).build()); // Paper
     private final List<CraftTask> temp = new ArrayList<>();
 
     CraftAsyncScheduler() {
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index 383c52c62f49b17db2fbf58009d6ea132d124bea..e0a71bfc1498a517456b21747ab6ef3f1067a3f3 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -105,7 +105,7 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
                     log.log( Level.SEVERE, "During the run of the server, a plugin set an excessive velocity on an entity" );
                     log.log( Level.SEVERE, "This may be the cause of the issue, or it may be entirely unrelated" );
                     log.log( Level.SEVERE, org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getMessage());
-                    for ( StackTraceElement stack : org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getStackTrace() )
+                    for ( StackTraceElement stack : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getStackTrace()) ) // Paper
                     {
                         log.log( Level.SEVERE, "\t\t" + stack );
                     }
@@ -193,7 +193,7 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
         }
         log.log( Level.SEVERE, "\tStack:" );
         //
-        for ( StackTraceElement stack : thread.getStackTrace() )
+        for ( StackTraceElement stack : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(thread.getStackTrace()) ) // Paper
         {
             log.log( Level.SEVERE, "\t\t" + stack );
         }
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index d285dbec16272db6b8a71865e05924ad66087407..1a05d23ff886b015fb9396f119822c678a47ec6f 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -30,10 +30,14 @@
             <DefaultRolloverStrategy max="1000"/>
         </RollingRandomAccessFile>
         <Async name="Async">
+            <AppenderRef ref="rewrite"/>
+        </Async>
+        <Rewrite name="rewrite">
+            <StacktraceDeobfuscatingRewritePolicy />
             <AppenderRef ref="File"/>
             <AppenderRef ref="TerminalConsole" level="info"/>
             <AppenderRef ref="ServerGuiConsole" level="info"/>
-        </Async>
+        </Rewrite>
     </Appenders>
     <Loggers>
         <Root level="info">