From 840aa86dfac85e51916a164f309333879095fc82 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Thu, 16 Jul 2020 17:15:36 +0800 Subject: [PATCH 1/2] Provide secret manager for proxy cache project This commit provides the secret manager for proxy cache. The secret is used for pushing blobs to local when it's proxied from remote registry. Each secret can be used only once and has a relatively short expiration time. Signed-off-by: Daniel Jiang --- src/pkg/proxy/secret/manager.go | 70 ++++++++++++++++++++++++++++ src/pkg/proxy/secret/manager_test.go | 33 +++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 src/pkg/proxy/secret/manager.go create mode 100644 src/pkg/proxy/secret/manager_test.go diff --git a/src/pkg/proxy/secret/manager.go b/src/pkg/proxy/secret/manager.go new file mode 100644 index 000000000..289223171 --- /dev/null +++ b/src/pkg/proxy/secret/manager.go @@ -0,0 +1,70 @@ +package secret + +import ( + "sync" + "time" + + "github.com/goharbor/harbor/src/common/utils" +) + +const defaultExpiration = 15 * time.Second + +type targetRepository struct { + name string + expiresAt time.Time +} + +// Manager generates and verifies the secret for repositories under proxy project +// The secret normally is used for authorizing a request trying to push artifact to a project +// A secret can be used only once and expires in a short period of time. +// As the request will be sent to 127.0.0.1 so the secret will live in one process. +type Manager interface { + // Generate generates a secret for the given repository, sample value for repository: "library/ubuntu" + Generate(repository string) string + // Verify verifies the secret against repo name, after the verification the secret should be invalid + Verify(secret, repository string) bool +} + +type mgr struct { + m *sync.Map + exp time.Duration +} + +func (man *mgr) Generate(rn string) string { + s := utils.GenerateRandomStringWithLen(8) + man.m.Store(s, targetRepository{name: rn, expiresAt: time.Now().Add(man.exp)}) + return s +} + +func (man *mgr) Verify(sec, rn string) bool { + v, ok := man.m.Load(sec) + if !ok { + return false + } + p, ok := v.(targetRepository) + if ok && p.name == rn { + defer man.m.Delete(sec) + return p.expiresAt.After(time.Now()) + } + return false +} + +var ( + defaultManager Manager + once sync.Once +) + +// GetManager returns the default manager which is a singleton in the package +func GetManager() Manager { + once.Do(func() { + defaultManager = createManager(defaultExpiration) + }) + return defaultManager +} + +func createManager(d time.Duration) Manager { + return &mgr{ + m: &sync.Map{}, + exp: d, + } +} diff --git a/src/pkg/proxy/secret/manager_test.go b/src/pkg/proxy/secret/manager_test.go new file mode 100644 index 000000000..83ed00aae --- /dev/null +++ b/src/pkg/proxy/secret/manager_test.go @@ -0,0 +1,33 @@ +package secret + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestManger(t *testing.T) { + manager := GetManager() + rn1 := "project1/golang" + assert.False(t, manager.Verify("whatever", rn1)) + s1 := manager.Generate(rn1) + s2 := manager.Generate(rn1) + assert.False(t, s1 == s2) + + assert.False(t, manager.Verify(s1, "project1/donotexist")) + assert.True(t, manager.Verify(s1, rn1)) + // A secret can be used only once. + assert.False(t, manager.Verify(s1, rn1)) + manager2 := GetManager() + assert.Equal(t, manager2, manager) +} + +func TestExpiration(t *testing.T) { + manager := createManager(1 * time.Second) + rn1 := "project1/golang" + s := manager.Generate(rn1) + // Sleep till the secret expires + time.Sleep(2 * time.Second) + assert.False(t, manager.Verify(s, rn1)) +} From 14203169bf81c598c7a4c06da6cedf87477279d3 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Mon, 20 Jul 2020 14:23:49 +0800 Subject: [PATCH 2/2] Add GC mechanism to secret manager When Generate is called and the size is larger than cap, GC will be triggered. Signed-off-by: Daniel Jiang --- src/pkg/proxy/secret/manager.go | 99 +++++++++++++++++++++++++--- src/pkg/proxy/secret/manager_test.go | 22 ++++++- 2 files changed, 112 insertions(+), 9 deletions(-) diff --git a/src/pkg/proxy/secret/manager.go b/src/pkg/proxy/secret/manager.go index 289223171..b335f55ba 100644 --- a/src/pkg/proxy/secret/manager.go +++ b/src/pkg/proxy/secret/manager.go @@ -2,12 +2,18 @@ package secret import ( "sync" + "sync/atomic" "time" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/log" ) -const defaultExpiration = 15 * time.Second +const ( + defaultExpiration = 15 * time.Second + defaultGCInterval = 10 * time.Second + defaultCap uint64 = 1024 * 1024 +) type targetRepository struct { name string @@ -25,14 +31,40 @@ type Manager interface { Verify(secret, repository string) bool } +type flag struct { + v uint32 +} + +func (f *flag) grab() bool { + return atomic.CompareAndSwapUint32(&f.v, 0, 1) +} + +func (f *flag) release() { + atomic.StoreUint32(&f.v, 0) +} + type mgr struct { - m *sync.Map - exp time.Duration + gcFlag *flag + gcScheduleFlag *flag + // the minimal interval for gc + gcInterval time.Duration + lastGC time.Time + // the size of the map, it must be read and write via atomic + size uint64 + // the capacity of the map, if the size is above the cap the gc will be triggered + cap uint64 + m *sync.Map + lock sync.Mutex + exp time.Duration } func (man *mgr) Generate(rn string) string { + if atomic.LoadUint64(&man.size) > man.cap { + man.gc() + } s := utils.GenerateRandomStringWithLen(8) man.m.Store(s, targetRepository{name: rn, expiresAt: time.Now().Add(man.exp)}) + atomic.AddUint64(&man.size, 1) return s } @@ -43,12 +75,59 @@ func (man *mgr) Verify(sec, rn string) bool { } p, ok := v.(targetRepository) if ok && p.name == rn { - defer man.m.Delete(sec) + defer man.delete(sec) return p.expiresAt.After(time.Now()) } return false } +func (man *mgr) delete(sec string) { + if _, ok := man.m.Load(sec); ok { + man.lock.Lock() + defer man.lock.Unlock() + if _, ok := man.m.Load(sec); ok { + man.m.Delete(sec) + atomic.AddUint64(&man.size, ^uint64(0)) + } + + } +} + +// gc removes the expired entries so it's possible that after running gc the size is still larger than cap +// If that happens it will try to start a go routine to run another gc +func (man *mgr) gc() { + if !man.gcFlag.grab() { + log.Debugf("There is GC in progress, skip") + return + } + defer func() { + if atomic.LoadUint64(&man.size) > man.cap && man.gcScheduleFlag.grab() { + log.Debugf("Size is still larger than cap, schedule a gc in next cycle") + go func() { + time.Sleep(man.gcInterval) + man.gcScheduleFlag.release() + man.gc() + }() + } + man.gcFlag.release() + }() + if time.Now().Before(man.lastGC.Add(man.gcInterval)) { + log.Debugf("Skip too frequent GC, last one: %v, ", man.lastGC) + return + } + log.Debugf("Running GC on secret map...") + man.m.Range(func(k, v interface{}) bool { + repoV, ok := v.(targetRepository) + if ok && repoV.expiresAt.Before(time.Now()) { + log.Debugf("Removed expire secret: %s, repo: %s", k, repoV.name) + man.delete(k.(string)) + } + return true + }) + man.lastGC = time.Now() + log.Debugf("GC on secret map finished.") +} + var ( defaultManager Manager once sync.Once @@ -57,14 +136,18 @@ var ( // GetManager returns the default manager which is a singleton in the package func GetManager() Manager { once.Do(func() { - defaultManager = createManager(defaultExpiration) + defaultManager = createManager(defaultExpiration, defaultCap, defaultGCInterval) }) return defaultManager } -func createManager(d time.Duration) Manager { +func createManager(d time.Duration, c uint64, interval time.Duration) Manager { return &mgr{ - m: &sync.Map{}, - exp: d, + m: &sync.Map{}, + exp: d, + cap: c, + gcInterval: interval, + gcFlag: &flag{}, + gcScheduleFlag: &flag{}, } } diff --git a/src/pkg/proxy/secret/manager_test.go b/src/pkg/proxy/secret/manager_test.go index 83ed00aae..0e07ee0f2 100644 --- a/src/pkg/proxy/secret/manager_test.go +++ b/src/pkg/proxy/secret/manager_test.go @@ -1,6 +1,8 @@ package secret import ( + "fmt" + "sync/atomic" "testing" "time" @@ -24,10 +26,28 @@ func TestManger(t *testing.T) { } func TestExpiration(t *testing.T) { - manager := createManager(1 * time.Second) + manager := createManager(1*time.Second, defaultCap, defaultGCInterval) rn1 := "project1/golang" s := manager.Generate(rn1) // Sleep till the secret expires time.Sleep(2 * time.Second) assert.False(t, manager.Verify(s, rn1)) } + +func TestGC(t *testing.T) { + manager := createManager(1*time.Second, 10, 1*time.Second).(*mgr) + for i := 0; i < 10; i++ { + rn := fmt.Sprintf("project%d/golang", i) + manager.Generate(rn) + } + time.Sleep(2 * time.Second) + assert.Equal(t, uint64(10), manager.size) + for i := 0; i < 1000; i++ { + rn := fmt.Sprintf("project%d/redis", i) + manager.Generate(rn) + } + assert.Equal(t, uint64(1000), atomic.LoadUint64(&manager.size)) + time.Sleep(4 * time.Second) + assert.Equal(t, uint64(0), atomic.LoadUint64(&manager.size)) + +}