mirror of
https://github.com/goharbor/harbor.git
synced 2024-09-30 22:37:43 +02:00
add retry for blob & manifest deletion in GC (#15235)
Fixes #15197 To handle the blob & manifest deletion error, add 1 minute retry. Signed-off-by: Wang Yan <wangyan@vmware.com>
This commit is contained in:
parent
4e1f6633af
commit
29ccdff766
@ -15,6 +15,7 @@
|
|||||||
package gc
|
package gc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"github.com/goharbor/harbor/src/lib"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -270,9 +271,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
|
|||||||
}
|
}
|
||||||
// for manifest, it has to delete the revisions folder of each repository
|
// for manifest, it has to delete the revisions folder of each repository
|
||||||
gc.logger.Infof("delete manifest from storage: %s", blob.Digest)
|
gc.logger.Infof("delete manifest from storage: %s", blob.Digest)
|
||||||
if err := ignoreNotFound(func() error {
|
if err := lib.RetryUntil(func() error {
|
||||||
return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
|
return ignoreNotFound(func() error {
|
||||||
}); err != nil {
|
return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
|
||||||
|
})
|
||||||
|
}, lib.RetryCallback(func(err error, sleep time.Duration) {
|
||||||
|
gc.logger.Infof("failed to exec DeleteManifest retry after %s : %v", sleep, err)
|
||||||
|
})); err != nil {
|
||||||
if err := ignoreNotFound(func() error {
|
if err := ignoreNotFound(func() error {
|
||||||
return gc.markDeleteFailed(ctx, blob)
|
return gc.markDeleteFailed(ctx, blob)
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@ -294,9 +299,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
|
|||||||
// for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record.
|
// for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record.
|
||||||
if !blob.IsForeignLayer() {
|
if !blob.IsForeignLayer() {
|
||||||
gc.logger.Infof("delete blob from storage: %s", blob.Digest)
|
gc.logger.Infof("delete blob from storage: %s", blob.Digest)
|
||||||
if err := ignoreNotFound(func() error {
|
if err := lib.RetryUntil(func() error {
|
||||||
return gc.registryCtlClient.DeleteBlob(blob.Digest)
|
return ignoreNotFound(func() error {
|
||||||
}); err != nil {
|
return gc.registryCtlClient.DeleteBlob(blob.Digest)
|
||||||
|
})
|
||||||
|
}, lib.RetryCallback(func(err error, sleep time.Duration) {
|
||||||
|
gc.logger.Infof("failed to exec DeleteBlob retry after %s : %v", sleep, err)
|
||||||
|
})); err != nil {
|
||||||
if err := ignoreNotFound(func() error {
|
if err := ignoreNotFound(func() error {
|
||||||
return gc.markDeleteFailed(ctx, blob)
|
return gc.markDeleteFailed(ctx, blob)
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
@ -2,9 +2,11 @@ package gc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/goharbor/harbor/src/lib"
|
||||||
"github.com/goharbor/harbor/src/lib/errors"
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
"github.com/goharbor/harbor/src/pkg/registry"
|
"github.com/goharbor/harbor/src/pkg/registry"
|
||||||
"github.com/gomodule/redigo/redis"
|
"github.com/gomodule/redigo/redis"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// delKeys ...
|
// delKeys ...
|
||||||
@ -50,10 +52,11 @@ func v2DeleteManifest(repository, digest string) error {
|
|||||||
if !exist {
|
if !exist {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := registry.Cli.DeleteManifest(repository, digest); err != nil {
|
return lib.RetryUntil(func() error {
|
||||||
return err
|
return registry.Cli.DeleteManifest(repository, digest)
|
||||||
}
|
}, lib.RetryCallback(func(err error, sleep time.Duration) {
|
||||||
return nil
|
fmt.Printf("failed to exec DeleteManifest retry after %s : %v\n", sleep, err)
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
// ignoreNotFound ignores the NotFoundErr error
|
// ignoreNotFound ignores the NotFoundErr error
|
||||||
|
Loading…
Reference in New Issue
Block a user