diff --git a/src/pkg/proxy/secret/manager.go b/src/pkg/proxy/secret/manager.go
index 289223171..b335f55ba 100644
--- a/src/pkg/proxy/secret/manager.go
+++ b/src/pkg/proxy/secret/manager.go
@@ -2,12 +2,18 @@ package secret
 
 import (
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/lib/log"
 )
 
-const defaultExpiration = 15 * time.Second
+const (
+	defaultExpiration        = 15 * time.Second
+	defaultGCInterval        = 10 * time.Second
+	defaultCap        uint64 = 1024 * 1024
+)
 
 type targetRepository struct {
 	name      string
@@ -25,14 +31,40 @@ type Manager interface {
 	Verify(secret, repository string) bool
 }
 
+type flag struct {
+	v uint32
+}
+
+func (f *flag) grab() bool {
+	return atomic.CompareAndSwapUint32(&f.v, 0, 1)
+}
+
+func (f *flag) release() {
+	atomic.StoreUint32(&f.v, 0)
+}
+
 type mgr struct {
-	m   *sync.Map
-	exp time.Duration
+	gcFlag         *flag
+	gcScheduleFlag *flag
+	// the minimal interval for gc
+	gcInterval time.Duration
+	lastGC     time.Time
+	// the size of the map, it must be read and write via atomic
+	size uint64
+	// the capacity of the map, if the size is above the cap the gc will be triggered
+	cap  uint64
+	m    *sync.Map
+	lock sync.Mutex
+	exp  time.Duration
 }
 
 func (man *mgr) Generate(rn string) string {
+	if atomic.LoadUint64(&man.size) > man.cap {
+		man.gc()
+	}
 	s := utils.GenerateRandomStringWithLen(8)
 	man.m.Store(s, targetRepository{name: rn, expiresAt: time.Now().Add(man.exp)})
+	atomic.AddUint64(&man.size, 1)
 	return s
 }
 
@@ -43,12 +75,59 @@ func (man *mgr) Verify(sec, rn string) bool {
 	}
 	p, ok := v.(targetRepository)
 	if ok && p.name == rn {
-		defer man.m.Delete(sec)
+		defer man.delete(sec)
 		return p.expiresAt.After(time.Now())
 	}
 	return false
 }
 
+func (man *mgr) delete(sec string) {
+	if _, ok := man.m.Load(sec); ok {
+		man.lock.Lock()
+		defer man.lock.Unlock()
+		if _, ok := man.m.Load(sec); ok {
+			man.m.Delete(sec)
+			atomic.AddUint64(&man.size, ^uint64(0))
+		}
+
+	}
+}
+
+// gc removes the expired entries so it's possible that after running gc the size is still larger than cap
+// If that happens it will try to start a go routine to run another gc
+func (man *mgr) gc() {
+	if !man.gcFlag.grab() {
+		log.Debugf("There is GC in progress, skip")
+		return
+	}
+	defer func() {
+		if atomic.LoadUint64(&man.size) > man.cap && man.gcScheduleFlag.grab() {
+			log.Debugf("Size is still larger than cap, schedule a gc in next cycle")
+			go func() {
+				time.Sleep(man.gcInterval)
+				man.gcScheduleFlag.release()
+				man.gc()
+			}()
+		}
+		man.gcFlag.release()
+	}()
+	if time.Now().Before(man.lastGC.Add(man.gcInterval)) {
+		log.Debugf("Skip too frequent GC, last one: %v, ", man.lastGC)
+		return
+	}
+	log.Debugf("Running GC on secret map...")
+	man.m.Range(func(k, v interface{}) bool {
+		repoV, ok := v.(targetRepository)
+		if ok && repoV.expiresAt.Before(time.Now()) {
+			log.Debugf("Removed expire secret: %s, repo: %s", k, repoV.name)
+			man.delete(k.(string))
+		}
+		return true
+	})
+	man.lastGC = time.Now()
+	log.Debugf("GC on secret map finished.")
+}
+
 var (
 	defaultManager Manager
 	once           sync.Once
@@ -57,14 +136,18 @@ var (
 // GetManager returns the default manager which is a singleton in the package
 func GetManager() Manager {
 	once.Do(func() {
-		defaultManager = createManager(defaultExpiration)
+		defaultManager = createManager(defaultExpiration, defaultCap, defaultGCInterval)
 	})
 	return defaultManager
 }
 
-func createManager(d time.Duration) Manager {
+func createManager(d time.Duration, c uint64, interval time.Duration) Manager {
 	return &mgr{
-		m:   &sync.Map{},
-		exp: d,
+		m:              &sync.Map{},
+		exp:            d,
+		cap:            c,
+		gcInterval:     interval,
+		gcFlag:         &flag{},
+		gcScheduleFlag: &flag{},
 	}
 }
diff --git a/src/pkg/proxy/secret/manager_test.go b/src/pkg/proxy/secret/manager_test.go
index 83ed00aae..0e07ee0f2 100644
--- a/src/pkg/proxy/secret/manager_test.go
+++ b/src/pkg/proxy/secret/manager_test.go
@@ -1,6 +1,8 @@
 package secret
 
 import (
+	"fmt"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -24,10 +26,28 @@ func TestManger(t *testing.T) {
 }
 
 func TestExpiration(t *testing.T) {
-	manager := createManager(1 * time.Second)
+	manager := createManager(1*time.Second, defaultCap, defaultGCInterval)
 	rn1 := "project1/golang"
 	s := manager.Generate(rn1)
 	// Sleep till the secret expires
 	time.Sleep(2 * time.Second)
 	assert.False(t, manager.Verify(s, rn1))
 }
+
+func TestGC(t *testing.T) {
+	manager := createManager(1*time.Second, 10, 1*time.Second).(*mgr)
+	for i := 0; i < 10; i++ {
+		rn := fmt.Sprintf("project%d/golang", i)
+		manager.Generate(rn)
+	}
+	time.Sleep(2 * time.Second)
+	assert.Equal(t, uint64(10), manager.size)
+	for i := 0; i < 1000; i++ {
+		rn := fmt.Sprintf("project%d/redis", i)
+		manager.Generate(rn)
+	}
+	assert.Equal(t, uint64(1000), atomic.LoadUint64(&manager.size))
+	time.Sleep(4 * time.Second)
+	assert.Equal(t, uint64(0), atomic.LoadUint64(&manager.size))
+
+}