From abe728325bbff6bf5763a2991e49bbb54d4d9fee Mon Sep 17 00:00:00 2001
From: Daniel Jiang <jiangd@vmware.com>
Date: Tue, 6 Nov 2018 14:02:44 +0800
Subject: [PATCH] Wait for manifest in notification handler

There's an issue in registry 2.6.x, that when the webhook is sent the
manifest of the image may not be written.
For details: https://github.com/docker/distribution/issues/2625

This will cause issue in "scan on push" or replication.
This commit mitigates the issue by adding retries in notification
handler.

Signed-off-by: Daniel Jiang <jiangd@vmware.com>
---
 .../service/notifications/registry/handler.go |  4 +++
 src/core/utils/job.go                         |  5 +++-
 src/core/utils/utils.go                       | 28 +++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go
index 55003a7d4..a30141c7f 100644
--- a/src/core/service/notifications/registry/handler.go
+++ b/src/core/service/notifications/registry/handler.go
@@ -106,6 +106,10 @@ func (n *NotificationHandler) Post() {
 					log.Errorf("Error happens when adding repository: %v", err)
 				}
 			}()
+			if !coreutils.WaitForManifestReady(repository, tag, 5) {
+				log.Errorf("Manifest for image %s:%s is not ready, skip the follow up actions.", repository, tag)
+				return
+			}
 
 			go func() {
 				image := repository + ":" + tag
diff --git a/src/core/utils/job.go b/src/core/utils/job.go
index 60558031e..b157c49a1 100644
--- a/src/core/utils/job.go
+++ b/src/core/utils/job.go
@@ -94,7 +94,10 @@ func TriggerImageScan(repository string, tag string) error {
 	if err != nil {
 		return err
 	}
-	digest, _, err := repoClient.ManifestExist(tag)
+	digest, exist, err := repoClient.ManifestExist(tag)
+	if !exist {
+		return fmt.Errorf("unable to perform scan: the manifest of image %s:%s does not exist", repository, tag)
+	}
 	if err != nil {
 		log.Errorf("Failed to get Manifest for %s:%s", repository, tag)
 		return err
diff --git a/src/core/utils/utils.go b/src/core/utils/utils.go
index 1c85383da..5959c4514 100644
--- a/src/core/utils/utils.go
+++ b/src/core/utils/utils.go
@@ -17,7 +17,9 @@ package utils
 
 import (
 	"net/http"
+	"time"
 
+	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	"github.com/goharbor/harbor/src/core/config"
@@ -42,3 +44,29 @@ func NewRepositoryClientForUI(username, repository string) (*registry.Repository
 	}
 	return registry.NewRepository(repository, endpoint, client)
 }
+
+// WaitForManifestReady implements exponential sleeep to wait until manifest is ready in registry.
+// This is a workaround for https://github.com/docker/distribution/issues/2625
+func WaitForManifestReady(repository string, tag string, maxRetry int) bool {
+	// The initial wait interval, hard-coded to 50ms
+	interval := 50 * time.Millisecond
+	repoClient, err := NewRepositoryClientForUI("harbor-core", repository)
+	if err != nil {
+		log.Errorf("Failed to create repo client.")
+		return false
+	}
+	for i := 0; i < maxRetry; i++ {
+		_, exist, err := repoClient.ManifestExist(tag)
+		if err != nil {
+			log.Errorf("Unexpected error when checking manifest existence, image:  %s:%s, error: %v", repository, tag, err)
+			continue
+		}
+		if exist {
+			return true
+		}
+		log.Warningf("manifest for image %s:%s is not ready, retry after %v", repository, tag, interval)
+		time.Sleep(interval)
+		interval = interval * 2
+	}
+	return false
+}