Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-24 09:38:09 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8624 from wy65701436/quota-migration-dev

Browse Source 
Add quota sync api to sync quota data with backend storage
This commit is contained in:
Wang Yan 2019-08-16 17:34:54 +08:00 committed by GitHub
commit 42bb5e1143
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1024 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/swagger.yaml
View File

@ -2409,6 +2409,20 @@ paths:
$ref: '#/responses/UnsupportedMediaType'
'500':
description: Unexpected internal errors.
/internal/syncquota:
post:
summary: Sync quota from registry/chart to DB.
description: |
This endpoint is for syncing quota usage of registry/chart with database.
tags:
- Products
responses:
'200':
description: Sync repositories successfully.
'401':
description: User need to log in first.
'403':
description: User does not have permission of system admin role.
/systeminfo:
get:
summary: Get general system info

3
make/migrations/postgresql/0010_1.9.0_schema.up.sql
View File

@ -86,6 +86,7 @@ CREATE TABLE quota_usage
UNIQUE (reference, reference_id)
);
/* only set quota and usage for 'library', and let the sync quota handling others. */
INSERT INTO quota (reference, reference_id, hard, creation_time, update_time)
SELECT 'project',
CAST(project_id AS VARCHAR),
@ -93,7 +94,7 @@ SELECT 'project',
NOW(),
NOW()
FROM project
WHERE deleted = 'f';
WHERE name = 'library' and deleted = 'f';
INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time)
SELECT id,

1
src/common/dao/artifact.go
View File

@ -58,6 +58,7 @@ func UpdateArtifactPullTime(af *models.Artifact) error {
// DeleteArtifact ...
func DeleteArtifact(id int64) error {
_, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete()
return err
}

6
src/core/api/harborapi_test.go
View File

@ -35,6 +35,7 @@ import (
testutils "github.com/goharbor/harbor/src/common/utils/test"
api_models "github.com/goharbor/harbor/src/core/api/models"
apimodels "github.com/goharbor/harbor/src/core/api/models"
quota "github.com/goharbor/harbor/src/core/api/quota"
_ "github.com/goharbor/harbor/src/core/auth/db"
_ "github.com/goharbor/harbor/src/core/auth/ldap"
"github.com/goharbor/harbor/src/core/config"
@ -203,12 +204,17 @@ func init() {
beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put")
beego.Router("/api/internal/switchquota", &InternalAPI{}, "put:SwitchQuota")
beego.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota")
// syncRegistry
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
log.Fatalf("failed to sync repositories from registry: %v", err)
}
if err := quota.Sync(config.GlobalProjectMgr, false); err != nil {
log.Fatalf("failed to sync quota from backend: %v", err)
}
// Init user Info
admin = &usrInfo{adminName, adminPwd}
unknownUsr = &usrInfo{"unknown", "unknown"}

31
src/core/api/health.go
View File

@ -34,8 +34,9 @@ import (
)
var (
timeout = 60 * time.Second
healthCheckerRegistry = map[string]health.Checker{}
timeout = 60 * time.Second
// HealthCheckerRegistry ...
HealthCheckerRegistry = map[string]health.Checker{}
)
type overallHealthStatus struct {
@ -67,11 +68,11 @@ type HealthAPI struct {
func (h *HealthAPI) CheckHealth() {
var isHealthy healthy = true
components := []*componentHealthStatus{}
c := make(chan *componentHealthStatus, len(healthCheckerRegistry))
for name, checker := range healthCheckerRegistry {
c := make(chan *componentHealthStatus, len(HealthCheckerRegistry))
for name, checker := range HealthCheckerRegistry {
go check(name, checker, timeout, c)
}
for i := 0; i < len(healthCheckerRegistry); i++ {
for i := 0; i < len(HealthCheckerRegistry); i++ {
componentStatus := <-c
if len(componentStatus.Error) != 0 {
isHealthy = false
@ -290,21 +291,21 @@ func redisHealthChecker() health.Checker {
}
func registerHealthCheckers() {
healthCheckerRegistry["core"] = coreHealthChecker()
healthCheckerRegistry["portal"] = portalHealthChecker()
healthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
healthCheckerRegistry["registry"] = registryHealthChecker()
healthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
healthCheckerRegistry["database"] = databaseHealthChecker()
healthCheckerRegistry["redis"] = redisHealthChecker()
HealthCheckerRegistry["core"] = coreHealthChecker()
HealthCheckerRegistry["portal"] = portalHealthChecker()
HealthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
HealthCheckerRegistry["registry"] = registryHealthChecker()
HealthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
HealthCheckerRegistry["database"] = databaseHealthChecker()
HealthCheckerRegistry["redis"] = redisHealthChecker()
if config.WithChartMuseum() {
healthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
HealthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
}
if config.WithClair() {
healthCheckerRegistry["clair"] = clairHealthChecker()
HealthCheckerRegistry["clair"] = clairHealthChecker()
}
if config.WithNotary() {
healthCheckerRegistry["notary"] = notaryHealthChecker()
HealthCheckerRegistry["notary"] = notaryHealthChecker()
}
}

16
src/core/api/health_test.go
View File

@ -92,9 +92,9 @@ func fakeHealthChecker(healthy bool) health.Checker {
}
func TestCheckHealth(t *testing.T) {
// component01: healthy, component02: healthy => status: healthy
healthCheckerRegistry = map[string]health.Checker{}
healthCheckerRegistry["component01"] = fakeHealthChecker(true)
healthCheckerRegistry["component02"] = fakeHealthChecker(true)
HealthCheckerRegistry = map[string]health.Checker{}
HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
HealthCheckerRegistry["component02"] = fakeHealthChecker(true)
status := map[string]interface{}{}
err := handleAndParse(&testingRequest{
method: http.MethodGet,
@ -104,9 +104,9 @@ func TestCheckHealth(t *testing.T) {
assert.Equal(t, "healthy", status["status"].(string))
// component01: healthy, component02: unhealthy => status: unhealthy
healthCheckerRegistry = map[string]health.Checker{}
healthCheckerRegistry["component01"] = fakeHealthChecker(true)
healthCheckerRegistry["component02"] = fakeHealthChecker(false)
HealthCheckerRegistry = map[string]health.Checker{}
HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
HealthCheckerRegistry["component02"] = fakeHealthChecker(false)
status = map[string]interface{}{}
err = handleAndParse(&testingRequest{
method: http.MethodGet,
@ -128,7 +128,7 @@ func TestDatabaseHealthChecker(t *testing.T) {
}
func TestRegisterHealthCheckers(t *testing.T) {
healthCheckerRegistry = map[string]health.Checker{}
HealthCheckerRegistry = map[string]health.Checker{}
registerHealthCheckers()
assert.NotNil(t, healthCheckerRegistry["core"])
assert.NotNil(t, HealthCheckerRegistry["core"])
}

40
src/core/api/internal.go
View File

@ -19,12 +19,17 @@ import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/quota"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/pkg/errors"
"strconv"
quota "github.com/goharbor/harbor/src/core/api/quota"
comcfg "github.com/goharbor/harbor/src/common/config"
)
// InternalAPI handles request of harbor admin...
@ -132,14 +137,14 @@ func (ia *InternalAPI) ensureQuota() error {
pCount = pCount + int64(count)
}
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10))
quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10))
if err != nil {
logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err)
continue
}
used := quota.ResourceList{
quota.ResourceStorage: pSize,
quota.ResourceCount: pCount,
used := common_quota.ResourceList{
common_quota.ResourceStorage: pSize,
common_quota.ResourceCount: pCount,
}
if err := quotaMgr.EnsureQuota(used); err != nil {
logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err)
@ -148,3 +153,28 @@ func (ia *InternalAPI) ensureQuota() error {
}
return nil
}
// SyncQuota ...
func (ia *InternalAPI) SyncQuota() {
cur := config.ReadOnly()
cfgMgr := comcfg.NewDBCfgManager()
if cur != true {
cfgMgr.Set(common.ReadOnly, true)
}
// For api call, to avoid the timeout, it should be asynchronous
go func() {
defer func() {
if cur != true {
cfgMgr.Set(common.ReadOnly, false)
}
}()
log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.")
err := quota.Sync(ia.ProjectMgr, false)
if err != nil {
log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err)
return
}
log.Info("success to sync quota(API).")
}()
return
}

33
src/core/api/internal_test.go
View File

@ -54,3 +54,36 @@ func TestSwitchQuota(t *testing.T) {
}
runCodeCheckingCases(t, cases...)
}
// cannot verify the real scenario here
func TestSyncQuota(t *testing.T) {
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/internal/syncquota",
},
code: http.StatusUnauthorized,
},
// 200
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/internal/syncquota",
credential: sysAdmin,
},
code: http.StatusOK,
},
// 403
{
request: &testingRequest{
url: "/api/internal/syncquota",
method: http.MethodPost,
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
}
runCodeCheckingCases(t, cases...)
}

226
src/core/api/quota/chart/chart.go Normal file
View File

@ -0,0 +1,226 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"fmt"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/pkg/errors"
"net/url"
"strings"
"sync"
)
// Migrator ...
type Migrator struct {
pm promgr.ProjectManager
}
// NewChartMigrator returns a new RegistryMigrator.
func NewChartMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
migrator := Migrator{
pm: pm,
}
return &migrator
}
var (
controller *chartserver.Controller
controllerErr error
controllerOnce sync.Once
)
// Ping ...
func (rm *Migrator) Ping() error {
return api.HealthCheckerRegistry["chartmuseum"].Check()
}
// Dump ...
// Depends on DB to dump chart data, as chart cannot get all of namespaces.
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
var (
projects []quota.ProjectInfo
wg sync.WaitGroup
err error
)
all, err := dao.GetProjects(nil)
if err != nil {
return nil, err
}
wg.Add(len(all))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
project, ok := result.(quota.ProjectInfo)
if ok {
projects = append(projects, project)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of project")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for _, project := range all {
go func(project *models.Project) {
defer wg.Done()
var repos []quota.RepoData
ctr, err := chartController()
if err != nil {
errChan <- err
return
}
chartInfo, err := ctr.ListCharts(project.Name)
if err != nil {
errChan <- err
return
}
// repo
for _, chart := range chartInfo {
var afs []*models.Artifact
chartVersions, err := ctr.GetChart(project.Name, chart.Name)
if err != nil {
errChan <- err
continue
}
for _, chart := range chartVersions {
af := &models.Artifact{
PID: project.ProjectID,
Repo: chart.Name,
Tag: chart.Version,
Digest: chart.Digest,
Kind: "Chart",
}
afs = append(afs, af)
}
repoData := quota.RepoData{
Name: project.Name,
Afs: afs,
}
repos = append(repos, repoData)
}
projectInfo := quota.ProjectInfo{
Name: project.Name,
Repos: repos,
}
infoChan <- projectInfo
}(project)
}
wg.Wait()
close(infoChan)
<-done
if err != nil {
return nil, err
}
return projects, nil
}
// Usage ...
// Chart will not cover size.
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
var pros []quota.ProjectUsage
for _, project := range projects {
var count int64
// usage count
for _, repo := range project.Repos {
count = count + int64(len(repo.Afs))
}
proUsage := quota.ProjectUsage{
Project: project.Name,
Used: common_quota.ResourceList{
common_quota.ResourceCount: count,
common_quota.ResourceStorage: 0,
},
}
pros = append(pros, proUsage)
}
return pros, nil
}
// Persist ...
// Chart will not persist data into db.
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
return nil
}
func chartController() (*chartserver.Controller, error) {
controllerOnce.Do(func() {
addr, err := config.GetChartMuseumEndpoint()
if err != nil {
controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error())
return
}
addr = strings.TrimSuffix(addr, "/")
url, err := url.Parse(addr)
if err != nil {
controllerErr = errors.New("endpoint URL of chart storage server is malformed")
return
}
ctr, err := chartserver.NewController(url)
if err != nil {
controllerErr = errors.New("failed to initialize chart API controller")
}
controller = ctr
log.Debugf("Chart storage server is set to %s", url.String())
log.Info("API controller for chart repository server is successfully initialized")
})
return controller, controllerErr
}
func init() {
quota.Register("chart", NewChartMigrator)
}

173
src/core/api/quota/migrator.go Normal file
View File

@ -0,0 +1,173 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/types"
"strconv"
)
// QuotaMigrator ...
type QuotaMigrator interface {
// Ping validates and wait for backend service ready.
Ping() error
// Dump exports all data from backend service, registry, chartmuseum
Dump() ([]ProjectInfo, error)
// Usage computes the quota usage of all the projects
Usage([]ProjectInfo) ([]ProjectUsage, error)
// Persist record the data to DB, artifact, artifact_blob and blob tabel.
Persist([]ProjectInfo) error
}
// ProjectInfo ...
type ProjectInfo struct {
Name string
Repos []RepoData
}
// RepoData ...
type RepoData struct {
Name string
Afs []*models.Artifact
Afnbs []*models.ArtifactAndBlob
Blobs []*models.Blob
}
// ProjectUsage ...
type ProjectUsage struct {
Project string
Used quota.ResourceList
}
// Instance ...
type Instance func(promgr.ProjectManager) QuotaMigrator
var adapters = make(map[string]Instance)
// Register ...
func Register(name string, adapter Instance) {
if adapter == nil {
panic("quota: Register adapter is nil")
}
if _, ok := adapters[name]; ok {
panic("quota: Register called twice for adapter " + name)
}
adapters[name] = adapter
}
// Sync ...
func Sync(pm promgr.ProjectManager, populate bool) error {
totalUsage := make(map[string][]ProjectUsage)
for name, instanceFunc := range adapters {
if !config.WithChartMuseum() {
if name == "chart" {
continue
}
}
adapter := instanceFunc(pm)
if err := adapter.Ping(); err != nil {
return err
}
data, err := adapter.Dump()
if err != nil {
return err
}
usage, err := adapter.Usage(data)
if err != nil {
return err
}
totalUsage[name] = usage
if populate {
if err := adapter.Persist(data); err != nil {
return err
}
}
}
merged := mergeUsage(totalUsage)
if err := ensureQuota(merged); err != nil {
return err
}
return nil
}
// mergeUsage merges the usage of adapters
func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage {
if !config.WithChartMuseum() {
return total["registry"]
}
regUsgs := total["registry"]
chartUsgs := total["chart"]
var mergedUsage []ProjectUsage
temp := make(map[string]quota.ResourceList)
for _, regUsg := range regUsgs {
_, exist := temp[regUsg.Project]
if !exist {
temp[regUsg.Project] = regUsg.Used
mergedUsage = append(mergedUsage, ProjectUsage{
Project: regUsg.Project,
Used: regUsg.Used,
})
}
}
for _, chartUsg := range chartUsgs {
var usedTemp quota.ResourceList
_, exist := temp[chartUsg.Project]
if !exist {
usedTemp = chartUsg.Used
} else {
usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used)
}
temp[chartUsg.Project] = usedTemp
mergedUsage = append(mergedUsage, ProjectUsage{
Project: chartUsg.Project,
Used: usedTemp,
})
}
return mergedUsage
}
// ensureQuota updates the quota and quota usage in the data base.
func ensureQuota(usages []ProjectUsage) error {
var pid int64
for _, usage := range usages {
project, err := dao.GetProjectByName(usage.Project)
if err != nil {
log.Error(err)
return err
}
pid = project.ProjectID
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(pid, 10))
if err != nil {
log.Errorf("Error occurred when to new quota manager %v", err)
return err
}
if err := quotaMgr.EnsureQuota(usage.Used); err != nil {
log.Errorf("cannot ensure quota for the project: %d, err: %v", pid, err)
return err
}
}
return nil
}

433
src/core/api/quota/registry/registry.go Normal file
View File

@ -0,0 +1,433 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/pkg/errors"
"strings"
"sync"
"time"
)
// Migrator ...
type Migrator struct {
pm promgr.ProjectManager
}
// NewRegistryMigrator returns a new Migrator.
func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
migrator := Migrator{
pm: pm,
}
return &migrator
}
// Ping ...
func (rm *Migrator) Ping() error {
return api.HealthCheckerRegistry["registry"].Check()
}
// Dump ...
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
var (
projects []quota.ProjectInfo
wg sync.WaitGroup
err error
)
reposInRegistry, err := api.Catalog()
if err != nil {
return nil, err
}
// repoMap : map[project_name : []repo list]
repoMap := make(map[string][]string)
for _, item := range reposInRegistry {
projectName := strings.Split(item, "/")[0]
pro, err := rm.pm.Get(projectName)
if err != nil {
log.Errorf("failed to get project %s: %v", projectName, err)
continue
}
_, exist := repoMap[pro.Name]
if !exist {
repoMap[pro.Name] = []string{item}
} else {
repos := repoMap[pro.Name]
repos = append(repos, item)
repoMap[pro.Name] = repos
}
}
wg.Add(len(repoMap))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
project, ok := result.(quota.ProjectInfo)
if ok {
projects = append(projects, project)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of project")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for project, repos := range repoMap {
go func(project string, repos []string) {
defer wg.Done()
info, err := infoOfProject(project, repos)
if err != nil {
errChan <- err
return
}
infoChan <- info
}(project, repos)
}
wg.Wait()
close(infoChan)
// wait for all of project info
<-done
if err != nil {
return nil, err
}
return projects, nil
}
// Usage ...
// registry needs to merge the shard blobs of different repositories.
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
var pros []quota.ProjectUsage
for _, project := range projects {
var size, count int64
var blobs = make(map[string]int64)
// usage count
for _, repo := range project.Repos {
count = count + int64(len(repo.Afs))
// Because that there are some shared blobs between repositories, it needs to remove the duplicate items.
for _, blob := range repo.Blobs {
_, exist := blobs[blob.Digest]
if !exist {
blobs[blob.Digest] = blob.Size
}
}
}
// size
for _, item := range blobs {
size = size + item
}
proUsage := quota.ProjectUsage{
Project: project.Name,
Used: common_quota.ResourceList{
common_quota.ResourceCount: count,
common_quota.ResourceStorage: size,
},
}
pros = append(pros, proUsage)
}
return pros, nil
}
// Persist ...
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
for _, project := range projects {
for _, repo := range project.Repos {
if err := persistAf(repo.Afs); err != nil {
return err
}
if err := persistAfnbs(repo.Afnbs); err != nil {
return err
}
if err := persistBlob(repo.Blobs); err != nil {
return err
}
}
}
if err := persistPB(projects); err != nil {
return err
}
return nil
}
func persistAf(afs []*models.Artifact) error {
if len(afs) != 0 {
for _, af := range afs {
_, err := dao.AddArtifact(af)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistAfnbs(afnbs []*models.ArtifactAndBlob) error {
if len(afnbs) != 0 {
for _, afnb := range afnbs {
_, err := dao.AddArtifactNBlob(afnb)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistBlob(blobs []*models.Blob) error {
if len(blobs) != 0 {
for _, blob := range blobs {
_, err := dao.AddBlob(blob)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistPB(projects []quota.ProjectInfo) error {
for _, project := range projects {
var blobs = make(map[string]int64)
var blobsOfPro []*models.Blob
for _, repo := range project.Repos {
for _, blob := range repo.Blobs {
_, exist := blobs[blob.Digest]
if exist {
continue
}
blobs[blob.Digest] = blob.Size
blobInDB, err := dao.GetBlob(blob.Digest)
if err != nil {
log.Error(err)
return err
}
if blobInDB != nil {
blobsOfPro = append(blobsOfPro, blobInDB)
}
}
}
pro, err := dao.GetProjectByName(project.Name)
if err != nil {
log.Error(err)
return err
}
_, err = dao.AddBlobsToProject(pro.ProjectID, blobsOfPro...)
if err != nil {
log.Error(err)
return err
}
}
return nil
}
func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error) {
var (
repos []quota.RepoData
wg sync.WaitGroup
err error
)
wg.Add(len(repoList))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
pro, err := dao.GetProjectByName(project)
if err != nil {
log.Error(err)
return quota.ProjectInfo{}, err
}
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
repoData, ok := result.(quota.RepoData)
if ok {
repos = append(repos, repoData)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of repo")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for _, repo := range repoList {
go func(pid int64, repo string) {
defer func() {
wg.Done()
}()
info, err := infoOfRepo(pid, repo)
if err != nil {
errChan <- err
return
}
infoChan <- info
}(pro.ProjectID, repo)
}
wg.Wait()
close(infoChan)
<-done
if err != nil {
return quota.ProjectInfo{}, err
}
return quota.ProjectInfo{
Name: project,
Repos: repos,
}, nil
}
func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo)
if err != nil {
return quota.RepoData{}, err
}
tags, err := repoClient.ListTag()
if err != nil {
return quota.RepoData{}, err
}
var afnbs []*models.ArtifactAndBlob
var afs []*models.Artifact
var blobs []*models.Blob
for _, tag := range tags {
_, mediaType, payload, err := repoClient.PullManifest(tag, []string{
schema1.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
})
if err != nil {
log.Error(err)
return quota.RepoData{}, err
}
manifest, desc, err := registry.UnMarshal(mediaType, payload)
if err != nil {
log.Error(err)
return quota.RepoData{}, err
}
// self
afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(),
DigestBlob: desc.Digest.String(),
}
afnbs = append(afnbs, afnb)
// add manifest as a blob.
blob := &models.Blob{
Digest: desc.Digest.String(),
ContentType: desc.MediaType,
Size: desc.Size,
CreationTime: time.Now(),
}
blobs = append(blobs, blob)
for _, layer := range manifest.References() {
afnb := &models.ArtifactAndBlob{
DigestAF: desc.Digest.String(),
DigestBlob: layer.Digest.String(),
}
afnbs = append(afnbs, afnb)
blob := &models.Blob{
Digest: layer.Digest.String(),
ContentType: layer.MediaType,
Size: layer.Size,
CreationTime: time.Now(),
}
blobs = append(blobs, blob)
}
af := &models.Artifact{
PID: pid,
Repo: strings.Split(repo, "/")[1],
Tag: tag,
Digest: desc.Digest.String(),
Kind: "Docker-Image",
CreationTime: time.Now(),
}
afs = append(afs, af)
}
return quota.RepoData{
Name: repo,
Afs: afs,
Afnbs: afnbs,
Blobs: blobs,
}, nil
}
func init() {
quota.Register("registry", NewRegistryMigrator)
}

5
src/core/api/utils.go
View File

@ -38,7 +38,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
log.Infof("Start syncing repositories from registry to DB... ")
reposInRegistry, err := catalog()
reposInRegistry, err := Catalog()
if err != nil {
log.Error(err)
return err
@ -105,7 +105,8 @@ func SyncRegistry(pm promgr.ProjectManager) error {
return nil
}
func catalog() ([]string, error) {
// Catalog ...
func Catalog() ([]string, error) {
repositories := []string{}
rc, err := initRegistryClient()

79
src/core/main.go
View File

@ -17,16 +17,12 @@ package main
import (
"encoding/gob"
"fmt"
"os"
"os/signal"
"strconv"
"syscall"
"github.com/astaxie/beego"
_ "github.com/astaxie/beego/session/redis"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
@ -34,6 +30,15 @@ import (
_ "github.com/goharbor/harbor/src/core/auth/db"
_ "github.com/goharbor/harbor/src/core/auth/ldap"
_ "github.com/goharbor/harbor/src/core/auth/uaa"
"os"
"os/signal"
"strconv"
"syscall"
quota "github.com/goharbor/harbor/src/core/api/quota"
_ "github.com/goharbor/harbor/src/core/api/quota/chart"
_ "github.com/goharbor/harbor/src/core/api/quota/registry"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/middlewares"
@ -41,6 +46,7 @@ import (
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/goharbor/harbor/src/replication"
)
@ -74,6 +80,64 @@ func updateInitPassword(userID int, password string) error {
return nil
}
// Quota migration
func quotaSync() error {
usages, err := dao.ListQuotaUsages()
if err != nil {
log.Errorf("list quota usage error, %v", err)
return err
}
projects, err := dao.GetProjects(nil)
if err != nil {
log.Errorf("list project error, %v", err)
return err
}
// The condition handles these two cases:
// 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB.
// 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully.
if len(projects) != len(usages) {
log.Info("Start to sync quota data .....")
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
log.Errorf("Fail to sync quota data, %v", err)
return err
}
log.Info("Success to sync quota data .....")
return nil
}
// Only has one project without usage
zero := common_quota.ResourceList{
common_quota.ResourceCount: 0,
common_quota.ResourceStorage: 0,
}
if len(projects) == 1 && len(usages) == 1 {
totalRepo, err := dao.GetTotalOfRepositories()
if totalRepo == 0 {
return nil
}
refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64)
if err != nil {
log.Error(err)
return err
}
usedRes, err := types.NewResourceList(usages[0].Used)
if err != nil {
log.Error(err)
return err
}
if types.Equals(usedRes, zero) && refID == projects[0].ProjectID {
log.Info("Start to sync quota data .....")
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
log.Errorf("Fail to sync quota data, %v", err)
return err
}
log.Info("Success to sync quota data .....")
}
}
return nil
}
func gracefulShutdown(closing chan struct{}) {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
@ -174,6 +238,9 @@ func main() {
log.Fatalf("init proxy error, %v", err)
}
// go proxy.StartProxy()
if err := quotaSync(); err != nil {
log.Fatalf("quota migration error, %v", err)
}
beego.Run()
}

1
src/core/router.go
View File

@ -135,6 +135,7 @@ func initRouters() {
beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry")
beego.Router("/api/internal/renameadmin", &api.InternalAPI{}, "post:RenameAdmin")
beego.Router("/api/internal/switchquota", &api.InternalAPI{}, "put:SwitchQuota")
beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota")
// external service that hosted on harbor process:
beego.Router("/service/notifications", &registry.NotificationHandler{})