From 29ccdff766541aadc0d4eeae6e18676a955ccc23 Mon Sep 17 00:00:00 2001
From: Wang Yan <wangyan@vmware.com>
Date: Thu, 1 Jul 2021 18:49:40 +0800
Subject: [PATCH] add retry for blob & manifest deletion in GC (#15235)

Fixes #15197

To handle the blob & manifest deletion error, add 1 minute retry.

Signed-off-by: Wang Yan <wangyan@vmware.com>
---
 .../job/impl/gc/garbage_collection.go         | 21 +++++++++++++------
 src/jobservice/job/impl/gc/util.go            | 11 ++++++----
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/jobservice/job/impl/gc/garbage_collection.go b/src/jobservice/job/impl/gc/garbage_collection.go
index 07d38b48a..94ed29bff 100644
--- a/src/jobservice/job/impl/gc/garbage_collection.go
+++ b/src/jobservice/job/impl/gc/garbage_collection.go
@@ -15,6 +15,7 @@
 package gc
 
 import (
+	"github.com/goharbor/harbor/src/lib"
 	"os"
 	"time"
 
@@ -270,9 +271,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
 				}
 				// for manifest, it has to delete the revisions folder of each repository
 				gc.logger.Infof("delete manifest from storage: %s", blob.Digest)
-				if err := ignoreNotFound(func() error {
-					return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
-				}); err != nil {
+				if err := lib.RetryUntil(func() error {
+					return ignoreNotFound(func() error {
+						return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest)
+					})
+				}, lib.RetryCallback(func(err error, sleep time.Duration) {
+					gc.logger.Infof("failed to exec DeleteManifest retry after %s : %v", sleep, err)
+				})); err != nil {
 					if err := ignoreNotFound(func() error {
 						return gc.markDeleteFailed(ctx, blob)
 					}); err != nil {
@@ -294,9 +299,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
 		// for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record.
 		if !blob.IsForeignLayer() {
 			gc.logger.Infof("delete blob from storage: %s", blob.Digest)
-			if err := ignoreNotFound(func() error {
-				return gc.registryCtlClient.DeleteBlob(blob.Digest)
-			}); err != nil {
+			if err := lib.RetryUntil(func() error {
+				return ignoreNotFound(func() error {
+					return gc.registryCtlClient.DeleteBlob(blob.Digest)
+				})
+			}, lib.RetryCallback(func(err error, sleep time.Duration) {
+				gc.logger.Infof("failed to exec DeleteBlob retry after %s : %v", sleep, err)
+			})); err != nil {
 				if err := ignoreNotFound(func() error {
 					return gc.markDeleteFailed(ctx, blob)
 				}); err != nil {
diff --git a/src/jobservice/job/impl/gc/util.go b/src/jobservice/job/impl/gc/util.go
index f021df2f5..e6466f1e0 100644
--- a/src/jobservice/job/impl/gc/util.go
+++ b/src/jobservice/job/impl/gc/util.go
@@ -2,9 +2,11 @@ package gc
 
 import (
 	"fmt"
+	"github.com/goharbor/harbor/src/lib"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/pkg/registry"
 	"github.com/gomodule/redigo/redis"
+	"time"
 )
 
 // delKeys ...
@@ -50,10 +52,11 @@ func v2DeleteManifest(repository, digest string) error {
 	if !exist {
 		return nil
 	}
-	if err := registry.Cli.DeleteManifest(repository, digest); err != nil {
-		return err
-	}
-	return nil
+	return lib.RetryUntil(func() error {
+		return registry.Cli.DeleteManifest(repository, digest)
+	}, lib.RetryCallback(func(err error, sleep time.Duration) {
+		fmt.Printf("failed to exec DeleteManifest retry after %s : %v\n", sleep, err)
+	}))
 }
 
 // ignoreNotFound ignores the NotFoundErr error