diff --git a/src/main/java/com/sekwah/advancedportals/bukkit/listeners/Listeners.java b/src/main/java/com/sekwah/advancedportals/bukkit/listeners/Listeners.java
index d7f1aa1..d2ce617 100644
--- a/src/main/java/com/sekwah/advancedportals/bukkit/listeners/Listeners.java
+++ b/src/main/java/com/sekwah/advancedportals/bukkit/listeners/Listeners.java
@@ -22,7 +22,9 @@ import org.bukkit.event.player.*;
 import org.bukkit.inventory.ItemStack;
 import org.bukkit.metadata.FixedMetadataValue;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 
 public class Listeners implements Listener {
@@ -48,6 +50,10 @@ public class Listeners implements Listener {
         }
 
         plugin.getServer().getPluginManager().registerEvents(this, plugin);
+
+        int cleanPeriod = config.getConfig().getInt("CleanUpPeriod", 120);
+        int period = 20 * 60 * cleanPeriod;
+        Bukkit.getScheduler().scheduleSyncRepeatingTask(plugin, new CooldownDataRemovalTask(), period, period);
     }
 
     @SuppressWarnings("deprecation")
@@ -77,7 +83,6 @@ public class Listeners implements Listener {
     @EventHandler
     public void onJoinEvent(PlayerJoinEvent event) {
         Portal.joinCooldown.put(event.getPlayer().getName(), System.currentTimeMillis());
-
         /*
          * if (plugin.PlayerDestiMap.containsKey(event.getPlayer())) { String desti =
          * plugin.PlayerDestiMap.get(event.getPlayer());
@@ -137,6 +142,65 @@ public class Listeners implements Listener {
         }
     }
 
+    class CooldownDataRemovalTask implements Runnable {
+
+        private int removed;
+
+        @Override
+        public void run() {
+            boolean canRemove = true;
+            while (canRemove) {
+                canRemove = Portal.cooldown.entrySet().removeIf(e -> {
+                    HashMap<String, Long> cds = e.getValue();
+                    if (cds == null) {
+                        removed++;
+                        return true;
+                    } else {
+                        cds.entrySet().removeIf(entry -> shouldRemovePortalCooldown(entry));
+                        if (cds.isEmpty()) {
+                            removed++;
+                            return true;
+                        }
+                    }
+                    return false;
+                });
+            }
+
+            // Make sure maps are never too big than they need to be
+            if (removed > 16) {
+                resizeMaps();
+                removed = 0;
+            }
+        }
+
+        private boolean shouldRemovePortalCooldown(Map.Entry<String, Long> entry) {
+            String portalName = entry.getKey();
+            AdvancedPortal portal = Portal.getPortal(portalName);
+            if (portal != null) {
+                long portalCD = entry.getValue();
+                int diff = (int) ((System.currentTimeMillis() - portalCD) / 1000);
+                int portalCooldown = -1;
+                try {
+                    portalCooldown = Integer.parseInt(portal.getArg("cooldowndelay"));
+                    return diff >= portalCooldown; // cooldown expired
+                } catch (Exception exc) {
+                    return true;
+                }
+            }
+            return true;
+        }
+
+        private void resizeMaps() {
+            HashMap<String, HashMap<String, Long>> newCooldowns = new HashMap<String, HashMap<String, Long>>(Math.max(Portal.cooldown.size() * 2, 10));
+            newCooldowns.putAll(Portal.cooldown);
+            Portal.cooldown = newCooldowns;
+
+            HashMap<String, Long> newJoinCooldowns = new HashMap<String, Long>(Math.max(Portal.joinCooldown.size() * 2, 10));
+            newJoinCooldowns.putAll(Portal.joinCooldown);
+            Portal.joinCooldown = newJoinCooldowns;
+        }
+    }
+
     // These are here because java 7 can only take finals straight into a runnable
     class RemoveLavaData implements Runnable {
 
diff --git a/src/main/java/com/sekwah/advancedportals/bukkit/portals/Portal.java b/src/main/java/com/sekwah/advancedportals/bukkit/portals/Portal.java
index b0aa29d..9be90bb 100644
--- a/src/main/java/com/sekwah/advancedportals/bukkit/portals/Portal.java
+++ b/src/main/java/com/sekwah/advancedportals/bukkit/portals/Portal.java
@@ -377,6 +377,15 @@ public class Portal {
         loadPortals();
     }
 
+    public static AdvancedPortal getPortal(String portalName) {
+        for (AdvancedPortal portalElement : Portal.portals) {
+            if (portalElement.getName().equals(portalName)) {
+                return portalElement;
+            }
+        }
+        return null;
+    }
+
     public static boolean portalExists(String portalName) {
 
         String posX = portalData.getConfig().getString(portalName + ".pos1.X");