Merge pull request #9599 from wy65701436/fix-quota-migration

Fix quota sync crash on getting the unknown mainfest

src/core/api/internal.go

@@ -172,7 +172,8 @@ func (ia *InternalAPI) SyncQuota() {
 			cfgMgr.Save()
 		}()
 		log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.")
-		err := quota.Sync(ia.ProjectMgr, false)
+		// As the sync function ignores all of duplicate error, it's safe to enable persist DB.
+		err := quota.Sync(ia.ProjectMgr, true)
 		if err != nil {
 			log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err)
 			return

src/core/api/quota/migrator.go

@@ -22,7 +22,6 @@ import (
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/promgr"
 	"github.com/goharbor/harbor/src/pkg/types"
-	"math/rand"
 	"strconv"
 	"time"
 )
@@ -80,7 +79,6 @@ func Register(name string, adapter Instance) {
 
 // Sync ...
 func Sync(pm promgr.ProjectManager, populate bool) error {
-	rand.Seed(time.Now().UnixNano())
 	totalUsage := make(map[string][]ProjectUsage)
 	for name, instanceFunc := range adapters {
 		if !config.WithChartMuseum() {

src/core/api/quota/registry/registry.go

@@ -86,6 +86,11 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 			repoMap[pro.Name] = repos
 		}
 	}
+	repoMap, err = rm.appendEmptyProject(repoMap)
+	if err != nil {
+		log.Errorf("fail to add empty projects: %v", err)
+		return nil, err
+	}
 
 	wg.Add(len(repoMap))
 	errChan := make(chan error, 1)
@@ -143,6 +148,24 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 	return projects, nil
 }
 
+// As catalog api cannot list the empty projects in harbor, here it needs to append the empty projects into repo infor
+// so that quota syncer can add 0 usage into quota usage.
+func (rm *Migrator) appendEmptyProject(repoMap map[string][]string) (map[string][]string, error) {
+	var withEmptyProjects map[string][]string
+	all, err := dao.GetProjects(nil)
+	if err != nil {
+		return withEmptyProjects, err
+	}
+	withEmptyProjects = repoMap
+	for _, pro := range all {
+		_, exist := repoMap[pro.Name]
+		if !exist {
+			withEmptyProjects[pro.Name] = []string{}
+		}
+	}
+	return withEmptyProjects, nil
+}
+
 // Usage ...
 // registry needs to merge the shard blobs of different repositories.
 func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
@@ -389,7 +412,10 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
 		})
 		if err != nil {
 			log.Error(err)
-			return quota.RepoData{}, err
+			// To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it.
+			// Let the sync process pass, but the 'Unknown manifest' will not be counted into size and count of quota usage.
+			// User still can view there images with size 0 in harbor.
+			continue
 		}
 		manifest, desc, err := registry.UnMarshal(mediaType, payload)
 		if err != nil {