diff --git a/sponge/src/main/java/me/lucko/luckperms/api/sponge/ContextData.java b/sponge/src/main/java/me/lucko/luckperms/api/sponge/ContextData.java
index b6030a435..2f6ded9ed 100644
--- a/sponge/src/main/java/me/lucko/luckperms/api/sponge/ContextData.java
+++ b/sponge/src/main/java/me/lucko/luckperms/api/sponge/ContextData.java
@@ -36,7 +36,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 public class ContextData {
-    private final LuckPermsUserSubject parent;
 
     @Getter
     private final Map<String, String> context;
@@ -47,7 +46,6 @@ public class ContextData {
     private final PermissionCalculator calculator;
 
     public ContextData(LuckPermsUserSubject parent, Map<String, String> context, LuckPermsService service) {
-        this.parent = parent;
         this.context = context;
 
         Set<Context> contexts = context.entrySet().stream().map(e -> new Context(e.getKey(), e.getValue())).collect(Collectors.toSet());
diff --git a/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsSubject.java b/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsSubject.java
index aa4b0deb0..957ec5996 100644
--- a/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsSubject.java
+++ b/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsSubject.java
@@ -50,10 +50,10 @@ public class LuckPermsSubject implements Subject {
     }
 
     @Getter
-    private final PermissionHolder holder;
-    private final LuckPermsSubjectData enduringData;
-    private final LuckPermsSubjectData transientData;
-    protected final LuckPermsService service;
+    private PermissionHolder holder;
+    private LuckPermsSubjectData enduringData;
+    private LuckPermsSubjectData transientData;
+    protected LuckPermsService service;
 
     LuckPermsSubject(PermissionHolder holder, LuckPermsService service) {
         this.holder = holder;
@@ -62,6 +62,13 @@ public class LuckPermsSubject implements Subject {
         this.service = service;
     }
 
+    public void deprovision() {
+        holder = null;
+        enduringData = null;
+        transientData = null;
+        service = null;
+    }
+
     void objectSave(PermissionHolder t) {
         service.getPlugin().doAsync(() -> {
             if (t instanceof User) {
diff --git a/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsUserSubject.java b/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsUserSubject.java
index 9d45b5633..4e8ff58d6 100644
--- a/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsUserSubject.java
+++ b/sponge/src/main/java/me/lucko/luckperms/api/sponge/LuckPermsUserSubject.java
@@ -45,10 +45,10 @@ public class LuckPermsUserSubject extends LuckPermsSubject {
     }
 
     @Getter
-    private final User user;
+    private User user;
 
     @Getter
-    private final Map<Map<String, String>, ContextData> contextData;
+    private Map<Map<String, String>, ContextData> contextData;
 
     private LuckPermsUserSubject(User user, LuckPermsService service) {
         super(user, service);
@@ -57,6 +57,15 @@ public class LuckPermsUserSubject extends LuckPermsSubject {
         contextData = new ConcurrentHashMap<>();
     }
 
+    @Override
+    public void deprovision() {
+        /* For some reason, Sponge holds onto User instances in a cache, which in turn, prevents LuckPerms data from being GCed.
+           As well as unloading, we also remove all references to the User instances. */
+        super.deprovision();
+        user = null;
+        contextData = null;
+    }
+
     @Override
     public Tristate getPermissionValue(@NonNull Set<Context> contexts, @NonNull String permission) {
         Map<String, String> context = contexts.stream().collect(Collectors.toMap(Context::getKey, Context::getValue));
diff --git a/sponge/src/main/java/me/lucko/luckperms/api/sponge/collections/UserCollection.java b/sponge/src/main/java/me/lucko/luckperms/api/sponge/collections/UserCollection.java
index 7a14f3563..99d3b2c42 100644
--- a/sponge/src/main/java/me/lucko/luckperms/api/sponge/collections/UserCollection.java
+++ b/sponge/src/main/java/me/lucko/luckperms/api/sponge/collections/UserCollection.java
@@ -92,7 +92,9 @@ public class UserCollection implements SubjectCollection {
      * @param uuid the internal uuid of the user
      */
     public void unload(UUID uuid) {
-        users.remove(uuid);
+        if (users.containsKey(uuid)) {
+            users.remove(uuid).deprovision();
+        }
     }
 
     @Override