diff --git a/src/main/java/net/minestom/server/network/socket/Server.java b/src/main/java/net/minestom/server/network/socket/Server.java
index f603b7178..63ee8600a 100644
--- a/src/main/java/net/minestom/server/network/socket/Server.java
+++ b/src/main/java/net/minestom/server/network/socket/Server.java
@@ -87,13 +87,6 @@ public final class Server {
                     MinecraftServer.getExceptionManager().handleException(e);
                 }
             }
-            try {
-                if (socketAddress instanceof UnixDomainSocketAddress unixDomainSocketAddress) {
-                    Files.deleteIfExists(unixDomainSocketAddress.getPath());
-                }
-            } catch (IOException e) {
-                MinecraftServer.getExceptionManager().handleException(e);
-            }
         }, "Ms-entrypoint").start();
     }
 
@@ -105,6 +98,9 @@ public final class Server {
         this.stop = true;
         try {
             this.serverSocket.close();
+            if (socketAddress instanceof UnixDomainSocketAddress unixDomainSocketAddress) {
+                Files.deleteIfExists(unixDomainSocketAddress.getPath());
+            }
         } catch (IOException e) {
             MinecraftServer.getExceptionManager().handleException(e);
         }
diff --git a/src/test/java/net/minestom/server/network/socket/ServerAddressTest.java b/src/test/java/net/minestom/server/network/socket/ServerAddressTest.java
index b5361451a..7011003f1 100644
--- a/src/test/java/net/minestom/server/network/socket/ServerAddressTest.java
+++ b/src/test/java/net/minestom/server/network/socket/ServerAddressTest.java
@@ -37,8 +37,6 @@ public class ServerAddressTest {
 
         assertDoesNotThrow(server::start);
         assertDoesNotThrow(server::stop);
-
-        Thread.sleep(250);
         assertFalse(Files.exists(address.getPath()), "The socket file should be deleted");
     }
 }