Update codes per review comments

Signed-off-by: wang yan <wangyan@vmware.com>

docs/swagger.yaml

@@ -2409,6 +2409,20 @@ paths:
           $ref: '#/responses/UnsupportedMediaType'
         '500':
           description: Unexpected internal errors.
+  /internal/syncquota:
+    post:
+      summary: Sync quota from registry/chart to DB.
+      description: |
+        This endpoint is for syncing quota usage of registry/chart with database.
+      tags:
+      - Products
+      responses:
+        '200':
+          description: Sync repositories successfully.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User does not have permission of system admin role.
   /systeminfo:
     get:
       summary: Get general system info

make/migrations/postgresql/0010_1.9.0_schema.up.sql

@@ -86,6 +86,7 @@ CREATE TABLE quota_usage
   UNIQUE (reference, reference_id)
 );
 
+/* only set quota and usage for 'library', and let the sync quota handling others. */
 INSERT INTO quota (reference, reference_id, hard, creation_time, update_time)
 SELECT 'project',
        CAST(project_id AS VARCHAR),
@@ -93,7 +94,7 @@ SELECT 'project',
        NOW(),
        NOW()
 FROM project
-WHERE deleted = 'f';
+WHERE name = 'library' and deleted = 'f';
 
 INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time)
 SELECT id,

src/core/api/harborapi_test.go

@@ -35,6 +35,7 @@ import (
 	testutils "github.com/goharbor/harbor/src/common/utils/test"
 	api_models "github.com/goharbor/harbor/src/core/api/models"
 	apimodels "github.com/goharbor/harbor/src/core/api/models"
+	quota "github.com/goharbor/harbor/src/core/api/quota"
 	_ "github.com/goharbor/harbor/src/core/auth/db"
 	_ "github.com/goharbor/harbor/src/core/auth/ldap"
 	"github.com/goharbor/harbor/src/core/config"
@@ -203,12 +204,17 @@ func init() {
 	beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put")
 
 	beego.Router("/api/internal/switchquota", &InternalAPI{}, "put:SwitchQuota")
+	beego.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota")
 
 	// syncRegistry
 	if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
 		log.Fatalf("failed to sync repositories from registry: %v", err)
 	}
 
+	if err := quota.Sync(config.GlobalProjectMgr, false); err != nil {
+		log.Fatalf("failed to sync quota from backend: %v", err)
+	}
+
 	// Init user Info
 	admin = &usrInfo{adminName, adminPwd}
 	unknownUsr = &usrInfo{"unknown", "unknown"}

src/core/api/internal.go

@@ -28,6 +28,8 @@ import (
 	"strconv"
 
 	quota "github.com/goharbor/harbor/src/core/api/quota"
+
+	comcfg "github.com/goharbor/harbor/src/common/config"
 )
 
 // InternalAPI handles request of harbor admin...
@@ -154,9 +156,25 @@ func (ia *InternalAPI) ensureQuota() error {
 
 // SyncQuota ...
 func (ia *InternalAPI) SyncQuota() {
-	err := quota.Sync(ia.ProjectMgr, false)
-	if err != nil {
-		ia.SendInternalServerError(err)
-		return
+	cur := config.ReadOnly()
+	cfgMgr := comcfg.NewDBCfgManager()
+	if cur != true {
+		cfgMgr.Set(common.ReadOnly, true)
 	}
+	// For api call, to avoid the timeout, it should be asynchronous
+	go func() {
+		defer func() {
+			if cur != true {
+				cfgMgr.Set(common.ReadOnly, false)
+			}
+		}()
+		log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.")
+		err := quota.Sync(ia.ProjectMgr, false)
+		if err != nil {
+			log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err)
+			return
+		}
+		log.Info("success to sync quota(API).")
+	}()
+	return
 }

src/core/api/internal_test.go

@@ -54,3 +54,36 @@ func TestSwitchQuota(t *testing.T) {
 	}
 	runCodeCheckingCases(t, cases...)
 }
+
+// cannot verify the real scenario here
+func TestSyncQuota(t *testing.T) {
+	cases := []*codeCheckingCase{
+		// 401
+		{
+			request: &testingRequest{
+				method: http.MethodPost,
+				url:    "/api/internal/syncquota",
+			},
+			code: http.StatusUnauthorized,
+		},
+		// 200
+		{
+			request: &testingRequest{
+				method:     http.MethodPost,
+				url:        "/api/internal/syncquota",
+				credential: sysAdmin,
+			},
+			code: http.StatusOK,
+		},
+		// 403
+		{
+			request: &testingRequest{
+				url:        "/api/internal/syncquota",
+				method:     http.MethodPost,
+				credential: nonSysAdmin,
+			},
+			code: http.StatusForbidden,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}

src/core/api/quota/chart/chart.go

@@ -103,19 +103,22 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 
 			ctr, err := chartController()
 			if err != nil {
-				log.Error(err)
+				errChan <- err
+				return
 			}
 
 			chartInfo, err := ctr.ListCharts(project.Name)
 			if err != nil {
-				log.Error(err)
+				errChan <- err
+				return
 			}
 
 			// repo
 			for _, chart := range chartInfo {
 				chartVersions, err := ctr.GetChart(project.Name, chart.Name)
 				if err != nil {
-					log.Error(err)
+					errChan <- err
+					continue
 				}
 				for _, chart := range chartVersions {
 					af := &models.Artifact{
@@ -143,7 +146,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 		}(project)
 	}
 
-	wg.Done()
+	wg.Wait()
 	close(infoChan)
 
 	<-done

src/core/api/quota/migrator.go

@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package models
+package api
 
 import (
-	"fmt"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/quota"
@@ -75,23 +74,21 @@ func Register(name string, adapter Instance) {
 
 // Sync ...
 func Sync(pm promgr.ProjectManager, populate bool) error {
-	for name := range adapters {
+	for name, instanceFunc := range adapters {
 		if !config.WithChartMuseum() {
 			if name == "chart" {
 				continue
 			}
 		}
-		instanceFunc, ok := adapters[name]
-		if !ok {
-			err := fmt.Errorf("quota migtator: unknown adapter name %q", name)
-			return err
-		}
 		adapter := instanceFunc(pm)
 		data, err := adapter.Dump()
 		if err != nil {
 			return err
 		}
 		usage, err := adapter.Usage(data)
+		if err != nil {
+			return err
+		}
 		if err := ensureQuota(usage); err != nil {
 			return err
 		}

src/core/api/quota/registry/registry.go

@@ -383,6 +383,14 @@ func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
 			DigestBlob: desc.Digest.String(),
 		}
 		afnbs = append(afnbs, afnb)
+		// add manifest as a blob.
+		blob := &models.Blob{
+			Digest:       desc.Digest.String(),
+			ContentType:  desc.MediaType,
+			Size:         desc.Size,
+			CreationTime: time.Now(),
+		}
+		blobs = append(blobs, blob)
 		for _, layer := range manifest.References() {
 			afnb := &models.ArtifactAndBlob{
 				DigestAF:   desc.Digest.String(),

src/core/main.go

@@ -17,16 +17,12 @@ package main
 import (
 	"encoding/gob"
 	"fmt"
-	"os"
-	"os/signal"
-	"strconv"
-	"syscall"
-
 	"github.com/astaxie/beego"
 	_ "github.com/astaxie/beego/session/redis"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/job"
 	"github.com/goharbor/harbor/src/common/models"
+	common_quota "github.com/goharbor/harbor/src/common/quota"
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/api"
@@ -34,6 +30,10 @@ import (
 	_ "github.com/goharbor/harbor/src/core/auth/db"
 	_ "github.com/goharbor/harbor/src/core/auth/ldap"
 	_ "github.com/goharbor/harbor/src/core/auth/uaa"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
 
 	quota "github.com/goharbor/harbor/src/core/api/quota"
 	_ "github.com/goharbor/harbor/src/core/api/quota/chart"
@@ -46,6 +46,7 @@ import (
 	"github.com/goharbor/harbor/src/core/service/token"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/scheduler"
+	"github.com/goharbor/harbor/src/pkg/types"
 	"github.com/goharbor/harbor/src/replication"
 )
 
@@ -92,16 +93,48 @@ func quotaSync() error {
 		return err
 	}
 
-	// upgrade from old version
-	if len(projects) > 1 && len(usages) == 1 {
+	// The condition handles these two cases:
+	// 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB.
+	// 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully.
+	if len(projects) != len(usages) {
 		log.Info("Start to sync quota data .....")
 		if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
-			log.Errorf("Error happened when syncing quota usage data, %v", err)
+			log.Errorf("Fail to sync quota data, %v", err)
 			return err
 		}
 		log.Info("Success to sync quota data .....")
+		return nil
 	}
 
+	// Only has one project without usage
+	zero := common_quota.ResourceList{
+		common_quota.ResourceCount:   0,
+		common_quota.ResourceStorage: 0,
+	}
+	if len(projects) == 1 && len(usages) == 1 {
+		totalRepo, err := dao.GetTotalOfRepositories()
+		if totalRepo == 0 {
+			return nil
+		}
+		refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64)
+		if err != nil {
+			log.Error(err)
+			return err
+		}
+		usedRes, err := types.NewResourceList(usages[0].Used)
+		if err != nil {
+			log.Error(err)
+			return err
+		}
+		if types.Equals(usedRes, zero) && refID == projects[0].ProjectID {
+			log.Info("Start to sync quota data .....")
+			if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
+				log.Errorf("Fail to sync quota data, %v", err)
+				return err
+			}
+			log.Info("Success to sync quota data .....")
+		}
+	}
 	return nil
 }