Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-06-30 00:35:07 +02:00
Code Issues Projects Releases Wiki Activity

add retry for blob & manifest deletion in GC (#15235)

Browse Source 
Fixes #15197

To handle the blob & manifest deletion error, add 1 minute retry.

Signed-off-by: Wang Yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2021-07-01 18:49:40 +08:00 committed by GitHub
parent 4e1f6633af
commit 29ccdff766
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/jobservice/job/impl/gc/garbage_collection.go
View File

@ -15,6 +15,7 @@
package gc
import (
"github.com/goharbor/harbor/src/lib"
"os"
"time"
@ -270,9 +271,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
}
// for manifest, it has to delete the revisions folder of each repository
gc.logger.Infof("delete manifest from storage: %s", blob.Digest)
if err := ignoreNotFound(func() error {
return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
}); err != nil {
if err := lib.RetryUntil(func() error {
return ignoreNotFound(func() error {
return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
})
}, lib.RetryCallback(func(err error, sleep time.Duration) {
gc.logger.Infof("failed to exec DeleteManifest retry after %s : %v", sleep, err)
})); err != nil {
if err := ignoreNotFound(func() error {
return gc.markDeleteFailed(ctx, blob)
}); err != nil {
@ -294,9 +299,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
// for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record.
if !blob.IsForeignLayer() {
gc.logger.Infof("delete blob from storage: %s", blob.Digest)
if err := ignoreNotFound(func() error {
return gc.registryCtlClient.DeleteBlob(blob.Digest)
}); err != nil {
if err := lib.RetryUntil(func() error {
return ignoreNotFound(func() error {
return gc.registryCtlClient.DeleteBlob(blob.Digest)
})
}, lib.RetryCallback(func(err error, sleep time.Duration) {
gc.logger.Infof("failed to exec DeleteBlob retry after %s : %v", sleep, err)
})); err != nil {
if err := ignoreNotFound(func() error {
return gc.markDeleteFailed(ctx, blob)
}); err != nil {

11
src/jobservice/job/impl/gc/util.go
View File

@ -2,9 +2,11 @@ package gc
import (
"fmt"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/gomodule/redigo/redis"
"time"
)
// delKeys ...
@ -50,10 +52,11 @@ func v2DeleteManifest(repository, digest string) error {
if !exist {
return nil
}
if err := registry.Cli.DeleteManifest(repository, digest); err != nil {
return err
}
return nil
return lib.RetryUntil(func() error {
return registry.Cli.DeleteManifest(repository, digest)
}, lib.RetryCallback(func(err error, sleep time.Duration) {
fmt.Printf("failed to exec DeleteManifest retry after %s : %v\n", sleep, err)
}))
}
// ignoreNotFound ignores the NotFoundErr error